diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5baf55..9de1af4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,71 +5,108 @@ on: branches: [main] pull_request: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" + jobs: - test: - name: pytest (py${{ matrix.python-version }}) + base: + name: base py${{ matrix.python-version }} / ${{ matrix.dependency-set }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + include: + - python-version: "3.12" + dependency-set: minimum + numpy-spec: "numpy==2.2.0" + - python-version: "3.12" + dependency-set: latest + numpy-spec: "numpy>=2.2,<3" + - python-version: "3.13" + dependency-set: minimum + numpy-spec: "numpy==2.2.0" + - python-version: "3.13" + dependency-set: latest + numpy-spec: "numpy>=2.2,<3" + - python-version: "3.14" + dependency-set: minimum + numpy-spec: "numpy==2.3.2" + - python-version: "3.14" + dependency-set: latest + numpy-spec: "numpy>=2.3.2,<3" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip - - name: Install package with all extras + - name: Install the base distribution run: | python -m pip install --upgrade pip - pip install -e ".[all]" pytest - - name: Run tests - # test/pde requires a working torch runtime; revisit when pde is - # re-validated (see IMPROVEMENT_PLAN.md phase 3). - run: pytest -q --ignore=test/pde + python -m pip install "${{ matrix.numpy-spec }}" + python -m pip install . "pytest>=8.4,<10" + - name: Verify installed-package origin + working-directory: ${{ runner.temp }} + env: + RISKLABAI_REPOSITORY_ROOT: ${{ github.workspace }} + run: python -B -c "import os; from pathlib import Path; import RiskLabAI; origin = Path(RiskLabAI.__file__).resolve(); repository = Path(os.environ['RISKLABAI_REPOSITORY_ROOT']).resolve(); assert not origin.is_relative_to(repository), (origin, repository); assert RiskLabAI.__version__ == '3.0.0'; print(origin)" + - name: Run the installed base package tests + working-directory: ${{ runner.temp }} + run: python -B -m pytest -q -p no:cacheprovider -c "$GITHUB_WORKSPACE/pyproject.toml" "$GITHUB_WORKSPACE/test" - minimal-install: - name: base install is importable + optional: + name: optional features py${{ matrix.python-version }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} cache: pip - - name: Install base package only (no extras) + - name: Install feature-specific optional groups run: | python -m pip install --upgrade pip - pip install . - - name: Import every public sub-package + python -m pip install ".[speed,pde,synth,hpo,plot,symbolic,profile,simulation,changepoints,test]" + - name: Verify changepoint availability policy run: | - python -c " - import RiskLabAI - print('version:', RiskLabAI.__version__) - for sub in ['backtest', 'cluster', 'controller', 'data', - 'ensemble', 'features', 'hpc', 'optimization', 'utils']: - getattr(RiskLabAI, sub) - print('ok:', sub) - " - - format: - name: black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - run: pip install "black==26.5.1" - - run: black --check RiskLabAI test + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + python -B -c "import importlib.util; assert importlib.util.find_spec('ruptures') is None" + else + python -B -c "import ruptures" + fi + - name: Verify installed-package origin + working-directory: ${{ runner.temp }} + env: + RISKLABAI_REPOSITORY_ROOT: ${{ github.workspace }} + run: python -B -c "import os; from pathlib import Path; import RiskLabAI; origin = Path(RiskLabAI.__file__).resolve(); repository = Path(os.environ['RISKLABAI_REPOSITORY_ROOT']).resolve(); assert not origin.is_relative_to(repository), (origin, repository); assert RiskLabAI.__version__ == '3.0.0'; print(origin)" + - name: Run the installed optional-feature tests + working-directory: ${{ runner.temp }} + run: python -B -m pytest -q -p no:cacheprovider -c "$GITHUB_WORKSPACE/pyproject.toml" "$GITHUB_WORKSPACE/test" - lint: - name: ruff + quality: + name: static quality runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" - - run: pip install "ruff==0.15.17" - - run: ruff check RiskLabAI test + python-version: "3.14" + - name: Install pinned quality tools + run: python -m pip install "black==26.5.1" "ruff==0.15.17" + - name: Black + run: black --check src/RiskLabAI/causal_factor_analysis test/causal_factor_analysis + - name: Ruff + run: ruff check src/RiskLabAI/causal_factor_analysis test/causal_factor_analysis diff --git a/LICENSE b/LICENSE index 421ea80..587ef53 100644 --- a/LICENSE +++ b/LICENSE @@ -1,12 +1,8 @@ BSD 3-Clause License -Copyright (c) 2022, RiskLab AI (risklab.ai) +Copyright (c) 2022-2026, Hamid Arian All rights reserved. -RiskLabAI includes codes and algorithms from Marcos López de Prado's books -"Advances in Financial Machine Learning" and -"Machine Learning for Asset Managers". - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f572cee --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,16 @@ +include LICENSE +include README.md +include RELEASE_NOTES_NEXT_MAJOR.md +include PUBLIC_API.json +include TEST_INVENTORY.json +include PACKAGE_FILES.json +include PACKAGE_INSPECTION.md +include .github/workflows/ci.yml +recursive-include src *.py +recursive-include test *.py +recursive-include docs *.md +recursive-include examples *.py +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude .DS_Store +global-exclude Thumbs.db diff --git a/PACKAGE_FILES.json b/PACKAGE_FILES.json new file mode 100644 index 0000000..9f4dd1c --- /dev/null +++ b/PACKAGE_FILES.json @@ -0,0 +1,788 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_INTENDED_PACKAGE_FILES", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "artifact_types": [ + "wheel", + "source_distribution" + ], + "candidate_file_count": 216, + "candidate_files": [ + ".github/workflows/ci.yml", + "LICENSE", + "MANIFEST.in", + "PACKAGE_FILES.json", + "PACKAGE_IDENTITY.json", + "PACKAGE_INSPECTION.md", + "PUBLIC_API.json", + "README.md", + "RELEASE_NOTES_NEXT_MAJOR.md", + "TEST_INVENTORY.json", + "docs/causal_factor_analysis.md", + "docs/compatibility.md", + "examples/causal_factor_analysis_quickstart.py", + "pyproject.toml", + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py", + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "local_control_files_excluded_from_artifacts": [ + "PACKAGE_IDENTITY.json" + ], + "release_source_file_count": 215, + "release_source_files": [ + ".github/workflows/ci.yml", + "LICENSE", + "MANIFEST.in", + "PACKAGE_FILES.json", + "PACKAGE_INSPECTION.md", + "PUBLIC_API.json", + "README.md", + "RELEASE_NOTES_NEXT_MAJOR.md", + "TEST_INVENTORY.json", + "docs/causal_factor_analysis.md", + "docs/compatibility.md", + "examples/causal_factor_analysis_quickstart.py", + "pyproject.toml", + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py", + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "runtime_source_file_count": 124, + "runtime_source_files": [ + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py" + ], + "test_file_count": 78, + "test_files": [ + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "wheel_python_source_members": [ + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py" + ], + "generated_metadata_members": { + "policy": "Generated package-manager metadata is not pre-enumerated by filename here; the future artifact inspector must validate its normalized field set and reject undeclared payload files." + } +} diff --git a/PACKAGE_INSPECTION.md b/PACKAGE_INSPECTION.md new file mode 100644 index 0000000..fceac72 --- /dev/null +++ b/PACKAGE_INSPECTION.md @@ -0,0 +1,64 @@ +# Future Python package inspection specification + +This specification applies only after a separate human authorization creates a +temporary RiskLabAI 3.0.0 wheel and source distribution. It does not authorize +building, installing, uploading, publishing, tagging, or releasing anything. + +## Inputs + +The inspector must use the final, hash-locked source tree and compare it with +`PACKAGE_FILES.json`, `PUBLIC_API.json`, `TEST_INVENTORY.json`, and +`pyproject.toml`. It must reject a changed input, an undecodable file, an +unresolved link, or a concurrent mutation before considering an artifact. + +## Archive safety and membership + +For both archive formats, reject absolute paths, drive-qualified paths, +backslashes, traversal components, duplicate paths, case-fold or Unicode +normalization collisions, control characters, trailing dots or spaces, +alternate data streams, links, device files, and unexpected native binaries. + +The wheel may contain only the 124 declared Python runtime modules plus the +standard metadata directory created for RiskLabAI 3.0.0. Tests, examples, +local readiness controls, caches, bytecode, credentials, and undeclared data +must not be present in the wheel. The source distribution must contain exactly +the 215 release-source members declared in `PACKAGE_FILES.json`, including the +governed `.github/workflows/ci.yml`, plus only the +backend-generated metadata members that the inspection record enumerates and +hashes explicitly. + +## Metadata + +The normalized package metadata must agree exactly with `pyproject.toml`: +name RiskLabAI, version 3.0.0, Python 3.12 through 3.14, BSD-3-Clause, the +declared maintainer and contact, repository, issue, and documentation links, +all required dependency clauses, and every optional group. No dynamic version, +entry point, plugin, executable script, or undeclared dependency is permitted. + +Every wheel record must have a valid cryptographic digest and size, except its +own permitted empty digest fields. The license and long description must match +the reviewed source bytes. + +## Runtime checks + +In fresh, isolated environments for every supported Python and valid NumPy +lane, verify the exact module origins, `RiskLabAI.__version__ == "3.0.0"`, the +13 root exports, all module and package exports in `PUBLIC_API.json`, and the +complete 57-feature causal API. For every module that declares `__all__`, +require every listed name to be bound on that installed module and resolvable by +an explicit import; an advertised but unbound name rejects the artifact. Run +the exact package-scoped test inventory and the valid optional-feature lanes. +Confirm that optional groups do not become base dependencies and that +changepoints alone is unavailable on Python 3.14. Run installed-package tests +outside the source checkout and reject any import whose resolved path remains +inside the repository. Tests that launch child interpreters must give those +processes clean temporary working directories and apply the same origin +rejection. + +## Final review + +Scan every archive member name and every decoded public text member against the +release hygiene denylist. Record archive hashes, member hashes, metadata, +runtime origins, test results, supported environments, and every deviation. +Any deviation keeps the artifact rejected. Human release, publication, and +version-control authority remain separate decisions after inspection. diff --git a/PUBLIC_API.json b/PUBLIC_API.json new file mode 100644 index 0000000..026794f --- /dev/null +++ b/PUBLIC_API.json @@ -0,0 +1,4480 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_PUBLIC_API_INVENTORY", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "inventory_rule": { + "root_and_package_exports": "Literal __all__ when present; otherwise imported public bindings in package initializers.", + "module_definitions": "Every non-underscore top-level function or class and every public uppercase constant is frozen conservatively." + }, + "root_exports": [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils" + ], + "runtime_module_count": 124, + "runtime_modules": [ + { + "path": "src/RiskLabAI/__init__.py", + "module": "RiskLabAI", + "module_kind": "package", + "sha256": "70093B30E9B7B92F555E40FB0DF936092A661BC88CA032549F1982624147A4AC", + "size": 1095, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/_deprecation.py", + "module": "RiskLabAI._deprecation", + "module_kind": "module", + "sha256": "F44F03C24759B46E69B0BD0241B51FF34454A9AAA1CD9A0498364284D5FB9498", + "size": 3889, + "defined_public_names": [ + "deprecated_alias", + "deprecated_class", + "warn_deprecated_name" + ], + "definitions": [ + { + "name": "deprecated_alias", + "kind": "function", + "signature": "(new_func: _F, old_name: str, *, removed_in: str)" + }, + { + "name": "deprecated_class", + "kind": "function", + "signature": "(new_cls: type, old_name: str, *, removed_in: str)" + }, + { + "name": "warn_deprecated_name", + "kind": "function", + "signature": "(old_name: str, new_name: str, *, removed_in: str)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/__init__.py", + "module": "RiskLabAI.backtest", + "module_kind": "package", + "sha256": "BC1D1DB5707902C0C61EFBB1DB1CDC88101388B065BCA30FE7A8CA621667058C", + "size": 6172, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "module": "RiskLabAI.backtest.advanced_bet_sizing", + "module_kind": "module", + "sha256": "E50F8275DC235F7DA777146D0715B945BAA3C7B91AEDA1FBDAA210D44C4A099F", + "size": 10017, + "defined_public_names": [ + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction" + ], + "definitions": [ + { + "name": "kelly_bet_fraction", + "kind": "function", + "signature": "(probability, payout_ratio: float=1.0, max_fraction: float=1.0)" + }, + { + "name": "expected_calibration_error", + "kind": "function", + "signature": "(probabilities, outcomes, n_bins: int=10)" + }, + { + "name": "PlattCalibrator", + "kind": "class", + "signature": "(self)" + }, + { + "name": "distributionally_robust_kelly_fraction", + "kind": "function", + "signature": "(probability_estimate, n_observations: int, payout_ratio: float=1.0, max_fraction: float=1.0, confidence_z: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "module": "RiskLabAI.backtest.backtest_overfitting_simulation", + "module_kind": "module", + "sha256": "F435D84F28A7ACFB3237D329BE0FF82B7F60B1729810E4B8936A324CAA8EF595", + "size": 61737, + "defined_public_names": [ + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "backtest_overfitting_simulation_results", + "expected_shortfall", + "financial_features_backtest_overfitting_simulation", + "format_cpu_info", + "generate_random_data", + "get_cpu_info", + "measure_all_cv_computational_requirements", + "measure_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "noised_backtest_overfitting_simulation", + "overall_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "sharpe_ratio", + "sortino_ratio", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation" + ], + "definitions": [ + { + "name": "financial_features_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, noise_scale: float=0.0, random_state: Optional[int]=None)" + }, + { + "name": "backtest_overfitting_simulation_results", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], cross_validators: dict[str, Any], noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "overall_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "temporal_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, overfitting_partitions_length: int, n_jobs: int=1)" + }, + { + "name": "time_temporal_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, overfitting_partitions_duration: str='A', n_jobs: int=1)" + }, + { + "name": "varying_embargo_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, embargo_values: list[float], n_jobs: int=1)" + }, + { + "name": "sharpe_ratio", + "kind": "function", + "signature": "(returns, risk_free_rate=0)" + }, + { + "name": "sortino_ratio", + "kind": "function", + "signature": "(returns, risk_free_rate=0)" + }, + { + "name": "expected_shortfall", + "kind": "function", + "signature": "(returns, step_risk_free_rate, confidence_level=0.05)" + }, + { + "name": "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, n_jobs: int=1)" + }, + { + "name": "backtest_overfitting_simulation_model_complexity", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, Any], step_risk_free_rate: float, n_jobs: int=1)" + }, + { + "name": "noised_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scales: list[float], random_state: int=None, n_jobs: int=1)" + }, + { + "name": "overall_novel_methods_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "get_cpu_info", + "kind": "function", + "signature": "()" + }, + { + "name": "format_cpu_info", + "kind": "function", + "signature": "(cpu_info)" + }, + { + "name": "generate_random_data", + "kind": "function", + "signature": "(n_samples: int, n_features: int)" + }, + { + "name": "measure_computational_requirements", + "kind": "function", + "signature": "(cross_validator, model, data, target, weights, n_jobs: int=1)" + }, + { + "name": "measure_all_cv_computational_requirements", + "kind": "function", + "signature": "(cross_validators: dict[str, Any], n_samples: int=40 * 252, n_features: int=22, n_jobs: int=1, n_repeats: int=30)" + }, + { + "name": "measure_cpcv_parallelization", + "kind": "function", + "signature": "(n_samples: int=40 * 252, n_features: int=22, n_repeats: int=30, n_jobs_list: list[int]=_DEFAULT_N_JOBS_LIST)" + }, + { + "name": "measure_cpcv_scalability", + "kind": "function", + "signature": "(sample_sizes: list[int], feature_sizes: list[int], n_repeats: int=1, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_statistics.py", + "module": "RiskLabAI.backtest.backtest_statistics", + "module_kind": "module", + "sha256": "D6365C332C46BE6E6A176B39D3CC70DA170301362A974609613FA370D710E643", + "size": 9157, + "defined_public_names": [ + "bet_timing", + "calculate_hhi", + "calculate_hhi_concentration", + "calculate_holding_period", + "compute_drawdowns_time_under_water", + "sharpe_ratio" + ], + "definitions": [ + { + "name": "sharpe_ratio", + "kind": "function", + "signature": "(returns: np.ndarray, risk_free_rate: float=0.0)" + }, + { + "name": "bet_timing", + "kind": "function", + "signature": "(target_positions: pd.Series)" + }, + { + "name": "calculate_holding_period", + "kind": "function", + "signature": "(target_positions: pd.Series)" + }, + { + "name": "calculate_hhi", + "kind": "function", + "signature": "(bet_returns: pd.Series)" + }, + { + "name": "calculate_hhi_concentration", + "kind": "function", + "signature": "(returns: pd.Series)" + }, + { + "name": "compute_drawdowns_time_under_water", + "kind": "function", + "signature": "(pnl_series: pd.Series, dollars: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "module": "RiskLabAI.backtest.backtest_synthetic_data", + "module_kind": "module", + "sha256": "F67B8239A8E5F9EF5BA73F84938614F211C71197CD87AAFCFE10A8EDEBABC627", + "size": 3424, + "defined_public_names": [ + "synthetic_back_testing" + ], + "definitions": [ + { + "name": "synthetic_back_testing", + "kind": "function", + "signature": "(forecast: float, half_life: float, sigma: float, n_iteration: int=100000, maximum_holding_period: int=100, profit_taking_range: np.ndarray=_DEFAULT_PROFIT_TAKING_RANGE, stop_loss_range: np.ndarray=_DEFAULT_STOP_LOSS_RANGE, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/bet_sizing.py", + "module": "RiskLabAI.backtest.bet_sizing", + "module_kind": "module", + "sha256": "40704B327DF86A3901B684D6085DDC503947B5641E081BFCBB9B80CBBE1BE889", + "size": 15577, + "defined_public_names": [ + "average_bet_sizes", + "avg_active_signals", + "bet_size_sigmoid", + "compute_sigmoid_width", + "discrete_signal", + "generate_signal", + "inverse_price", + "limit_price", + "mp_avg_active_signals", + "probability_bet_size", + "strategy_bet_sizing", + "target_position" + ], + "definitions": [ + { + "name": "probability_bet_size", + "kind": "function", + "signature": "(probabilities: np.ndarray, sides: np.ndarray)" + }, + { + "name": "average_bet_sizes", + "kind": "function", + "signature": "(price_dates: np.ndarray, start_dates: np.ndarray, end_dates: np.ndarray, bet_sizes: np.ndarray)" + }, + { + "name": "strategy_bet_sizing", + "kind": "function", + "signature": "(price_timestamps: pd.Index, times: pd.Series, sides: pd.Series, probabilities: pd.Series)" + }, + { + "name": "avg_active_signals", + "kind": "function", + "signature": "(signals: pd.DataFrame, n_threads: int)" + }, + { + "name": "mp_avg_active_signals", + "kind": "function", + "signature": "(signals: pd.DataFrame, molecule: list)" + }, + { + "name": "discrete_signal", + "kind": "function", + "signature": "(signal: pd.Series, step_size: float)" + }, + { + "name": "generate_signal", + "kind": "function", + "signature": "(events: pd.DataFrame, step_size: float, probability: pd.Series, prediction: pd.Series, n_classes: int, n_threads: int)" + }, + { + "name": "bet_size_sigmoid", + "kind": "function", + "signature": "(w: float, x: float)" + }, + { + "name": "target_position", + "kind": "function", + "signature": "(w: float, f: float, actual_price: float, maximum_position_size: int)" + }, + { + "name": "inverse_price", + "kind": "function", + "signature": "(f: float, w: float, m: float)" + }, + { + "name": "limit_price", + "kind": "function", + "signature": "(target_position_size: int, current_position: int, f: float, w: float, maximum_position_size: int)" + }, + { + "name": "compute_sigmoid_width", + "kind": "function", + "signature": "(x: float, m: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/multiple_testing.py", + "module": "RiskLabAI.backtest.multiple_testing", + "module_kind": "module", + "sha256": "8B867AE2529D3A354CE0847D9C08E90DDE29E9E20042C0888683EBB98B89AAA8", + "size": 7841, + "defined_public_names": [ + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + "holm_adjusted_p_values", + "sharpe_ratio_p_values" + ], + "definitions": [ + { + "name": "sharpe_ratio_p_values", + "kind": "function", + "signature": "(sharpe_ratios: np.ndarray, number_of_returns: int)" + }, + { + "name": "holm_adjusted_p_values", + "kind": "function", + "signature": "(p_values: np.ndarray)" + }, + { + "name": "benjamini_hochberg_yekutieli_adjusted_p_values", + "kind": "function", + "signature": "(p_values: np.ndarray)" + }, + { + "name": "haircut_sharpe_ratios", + "kind": "function", + "signature": "(sharpe_ratios: np.ndarray, number_of_returns: int, method: str='holm', significance_level: float=0.05)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/ou_trading_rules.py", + "module": "RiskLabAI.backtest.ou_trading_rules", + "module_kind": "module", + "sha256": "D48DFA47F2C98EF791E66AA7724C29153813F7742CD244E88FAAFD6062231742", + "size": 12532, + "defined_public_names": [ + "fit_ornstein_uhlenbeck", + "hit_upper_probability", + "mean_exit_time", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "stationary_std", + "theta_from_half_life" + ], + "definitions": [ + { + "name": "theta_from_half_life", + "kind": "function", + "signature": "(half_life: float)" + }, + { + "name": "stationary_std", + "kind": "function", + "signature": "(theta: float, sigma: float)" + }, + { + "name": "hit_upper_probability", + "kind": "function", + "signature": "(entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float)" + }, + { + "name": "mean_exit_time", + "kind": "function", + "signature": "(entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float)" + }, + { + "name": "ou_rule_metrics", + "kind": "function", + "signature": "(profit_take: float, stop_loss: float, theta: float, sigma: float, entry_gap: float, cost: float=0.0)" + }, + { + "name": "optimal_ou_trading_rule", + "kind": "function", + "signature": "(theta: float, sigma: float, entry_gap: float, cost: float=0.0, bounds: tuple=(0.25, 4.0))" + }, + { + "name": "fit_ornstein_uhlenbeck", + "kind": "function", + "signature": "(series: np.ndarray, dt: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "module": "RiskLabAI.backtest.probabilistic_sharpe_ratio", + "module_kind": "module", + "sha256": "3CD56F7C02E0D42CE04E395737DB6039FAEA617B367E6BFBC2F0A74768FA7B49", + "size": 4538, + "defined_public_names": [ + "benchmark_sharpe_ratio", + "probabilistic_sharpe_ratio" + ], + "definitions": [ + { + "name": "probabilistic_sharpe_ratio", + "kind": "function", + "signature": "(observed_sharpe_ratio: float, benchmark_sharpe_ratio: float, number_of_returns: int, skewness_of_returns: float=0.0, kurtosis_of_returns: float=3.0, return_test_statistic: bool=False)" + }, + { + "name": "benchmark_sharpe_ratio", + "kind": "function", + "signature": "(sharpe_ratio_estimates: list[float])" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "module": "RiskLabAI.backtest.probability_of_backtest_overfitting", + "module_kind": "module", + "sha256": "C14BB1E1179AAC1D61D56ACF6BF282BF2B9593B143A43E357DC7D9181C7DA373", + "size": 5085, + "defined_public_names": [ + "performance_evaluation", + "probability_of_backtest_overfitting" + ], + "definitions": [ + { + "name": "performance_evaluation", + "kind": "function", + "signature": "(train_partition: np.ndarray, test_partition: np.ndarray, n_strategies: int, metric: Callable[[np.ndarray, float], float], risk_free_return: float)" + }, + { + "name": "probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, risk_free_return: float=0.0, metric: Optional[Callable[[np.ndarray, float], float]]=None, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/robust_statistics.py", + "module": "RiskLabAI.backtest.robust_statistics", + "module_kind": "module", + "sha256": "6144EAD8326C07605EF783AFC41AC49884A4DB3BD56F6D3DA8CFBE93AD3C0F29", + "size": 11865, + "defined_public_names": [ + "conditional_expected_drawdown", + "sharpe_difference_test" + ], + "definitions": [ + { + "name": "conditional_expected_drawdown", + "kind": "function", + "signature": "(returns: np.ndarray, horizon: int, alpha: float=0.9)" + }, + { + "name": "sharpe_difference_test", + "kind": "function", + "signature": "(returns_a: np.ndarray, returns_b: np.ndarray, method: str='ledoit_wolf', block_length: int | None=None, n_boot: int=1000, bandwidth: int | None=None, random_state: int | None=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/sharpe_inference.py", + "module": "RiskLabAI.backtest.sharpe_inference", + "module_kind": "module", + "sha256": "CB3DE49BA2230604868ECAB11B4AA5C1C55AE4BCF47FF6ACD8FE2C71B4839E2C", + "size": 7799, + "defined_public_names": [ + "lplz_sharpe_inference", + "newey_west_automatic_lag", + "newey_west_long_run_variance", + "sharpe_ratio_influence_function" + ], + "definitions": [ + { + "name": "sharpe_ratio_influence_function", + "kind": "function", + "signature": "(returns: np.ndarray)" + }, + { + "name": "newey_west_long_run_variance", + "kind": "function", + "signature": "(series: np.ndarray, lag: int)" + }, + { + "name": "newey_west_automatic_lag", + "kind": "function", + "signature": "(number_of_returns: int)" + }, + { + "name": "lplz_sharpe_inference", + "kind": "function", + "signature": "(returns: np.ndarray, confidence_level: float=0.95, lag: Optional[int]=None, null_sharpe_ratio: float=0.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/strategy_risk.py", + "module": "RiskLabAI.backtest.strategy_risk", + "module_kind": "module", + "sha256": "00D0E6574F81E68051A95CD762158E7BA52BB8109D8E47FB723C02C06C476447", + "size": 10670, + "defined_public_names": [ + "bin_frequency", + "binomial_sharpe_ratio", + "calculate_strategy_risk", + "failure_probability", + "implied_precision", + "mix_gaussians", + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic" + ], + "definitions": [ + { + "name": "sharpe_ratio_trials", + "kind": "function", + "signature": "(p: float, n_run: int)" + }, + { + "name": "target_sharpe_ratio_symbolic", + "kind": "function", + "signature": "()" + }, + { + "name": "implied_precision", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, frequency: float, target_sharpe_ratio: float)" + }, + { + "name": "bin_frequency", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, precision: float, target_sharpe_ratio: float)" + }, + { + "name": "binomial_sharpe_ratio", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, frequency: float, probability: float)" + }, + { + "name": "mix_gaussians", + "kind": "function", + "signature": "(mu1: float, mu2: float, sigma1: float, sigma2: float, probability: float, n_obs: int)" + }, + { + "name": "failure_probability", + "kind": "function", + "signature": "(returns: np.ndarray, frequency: float, target_sharpe_ratio: float)" + }, + { + "name": "calculate_strategy_risk", + "kind": "function", + "signature": "(mu1: float, mu2: float, sigma1: float, sigma2: float, probability: float, n_obs: int, frequency: float, target_sharpe_ratio: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/test_set_overfitting.py", + "module": "RiskLabAI.backtest.test_set_overfitting", + "module_kind": "module", + "sha256": "2C813D80E85ED2717F4C3B991244B23DEE1253807C8FFA9EA71A36F37E9DDED0", + "size": 8865, + "defined_public_names": [ + "estimated_sharpe_ratio_z_statistics", + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "strategy_type1_error_probability", + "strategy_type2_error_probability", + "theta_for_type2_error" + ], + "definitions": [ + { + "name": "expected_max_sharpe_ratio", + "kind": "function", + "signature": "(n_trials: int, mean_sharpe_ratio: float, std_sharpe_ratio: float)" + }, + { + "name": "generate_max_sharpe_ratios", + "kind": "function", + "signature": "(n_sims: int, n_trials_list: list[int], std_sharpe_ratio: float, mean_sharpe_ratio: float)" + }, + { + "name": "mean_std_error", + "kind": "function", + "signature": "(n_sims0: int, n_sims1: int, n_trials: list[int], std_sharpe_ratio: float=1.0, mean_sharpe_ratio: float=0.0)" + }, + { + "name": "estimated_sharpe_ratio_z_statistics", + "kind": "function", + "signature": "(sharpe_ratio: float, t: int, true_sharpe_ratio: float=0.0, skew: float=0.0, kurt: int=3)" + }, + { + "name": "strategy_type1_error_probability", + "kind": "function", + "signature": "(z: float, k: int=1)" + }, + { + "name": "theta_for_type2_error", + "kind": "function", + "signature": "(sharpe_ratio: float, t: int, true_sharpe_ratio: float, skew: float=0.0, kurt: int=3)" + }, + { + "name": "strategy_type2_error_probability", + "kind": "function", + "signature": "(alpha_k: float, k: int, theta: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/__init__.py", + "module": "RiskLabAI.backtest.validation", + "module_kind": "package", + "sha256": "88C99E37BE96E2BD23FD54ED0491C962F2BDEDD696E97F9B2627A410D1F8660F", + "size": 1896, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.adaptive_combinatorial_purged", + "module_kind": "module", + "sha256": "B466BD0C43297F0654467E1A91B135CCB7958CD856A1A45BAFBCDBE372F7FAA0", + "size": 18395, + "defined_public_names": [ + "AdaptiveCombinatorialPurged" + ], + "definitions": [ + { + "name": "AdaptiveCombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0, n_subsplits: int=3, external_feature: Union[pd.Series, dict[str, pd.Series]]=None, lower_quantile: float=0.25, upper_quantile: float=0.75, subtract_border_adjustments: bool=True)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.bagged_combinatorial_purged", + "module_kind": "module", + "sha256": "A881EDEBB301C8C5184248AA37EDCCE94159A9314BA5369026E0C423C5C73B12", + "size": 9472, + "defined_public_names": [ + "BaggedCombinatorialPurged" + ], + "definitions": [ + { + "name": "BaggedCombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0, classifier: bool=True, n_estimators: int=10, max_samples: float=1.0, max_features: float=1.0, bootstrap: bool=True, bootstrap_features: bool=False, random_state: int=None)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.combinatorial_purged", + "module_kind": "module", + "sha256": "8CC827C13CE4DE02A70C5A942A663CC2B704BD0EF5B251CDE78F59CDB9F375DB", + "size": 14461, + "defined_public_names": [ + "CombinatorialPurged" + ], + "definitions": [ + { + "name": "CombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "module": "RiskLabAI.backtest.validation.cross_validator_controller", + "module_kind": "module", + "sha256": "FE635E694640176AA96872AAE26046E4707C0F5FBE8F8AE79E2197BC0A4D6C51", + "size": 1233, + "defined_public_names": [ + "CrossValidatorController" + ], + "definitions": [ + { + "name": "CrossValidatorController", + "kind": "class", + "signature": "(self, validator_type: str, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "module": "RiskLabAI.backtest.validation.cross_validator_factory", + "module_kind": "module", + "sha256": "EAEECAE7A1F73789F72BA9E5FBF03FE48CC20487AAFF5404613B82F41DE96C77", + "size": 2650, + "defined_public_names": [ + "CrossValidatorFactory" + ], + "definitions": [ + { + "name": "CrossValidatorFactory", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "module": "RiskLabAI.backtest.validation.cross_validator_interface", + "module_kind": "module", + "sha256": "A6FB69106047B94AF0E7ECCDE88FBDCCB5178BE08ED8EE8E03B04564F8E50A7E", + "size": 7967, + "defined_public_names": [ + "CrossValidator" + ], + "definitions": [ + { + "name": "CrossValidator", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/kfold.py", + "module": "RiskLabAI.backtest.validation.kfold", + "module_kind": "module", + "sha256": "BBA92A01B9EA320C89DB7CEBFED147EDBDDAE9CB7FE2D550FF2F2ECFE7ACC15B", + "size": 12431, + "defined_public_names": [ + "KFold" + ], + "definitions": [ + { + "name": "KFold", + "kind": "class", + "signature": "(self, n_splits: int, shuffle: bool=False, random_seed: int=None)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "module": "RiskLabAI.backtest.validation.leakage_aware_hpo", + "module_kind": "module", + "sha256": "A8D16874C56CFDB725B92906F5A08A1605DA93B263EDA210B3B561310E188410", + "size": 9891, + "defined_public_names": [ + "deflated_sharpe_gate", + "leakage_aware_hpo" + ], + "definitions": [ + { + "name": "leakage_aware_hpo", + "kind": "function", + "signature": "(suggest_params: Callable, estimator_factory: Callable, x, y, times, n_trials: int=50, sampler: str='tpe', n_splits: int=5, embargo: float=0.0, scoring: Callable | None=None, random_state: int=0)" + }, + { + "name": "deflated_sharpe_gate", + "kind": "function", + "signature": "(out_of_sample_returns: np.ndarray, n_trials: int, trial_sharpe_std: float, threshold: float=0.5)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "module": "RiskLabAI.backtest.validation.path_adaptive_cpcv", + "module_kind": "module", + "sha256": "4A1BCEDEA466B9C05165C282740CC1EDBB96174629CC377276432C3325C2E3B6", + "size": 10541, + "defined_public_names": [ + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes" + ], + "definitions": [ + { + "name": "estimate_volatility_regimes", + "kind": "function", + "signature": "(performances: np.ndarray, n_regimes: int=2, window: int | None=None)" + }, + { + "name": "adaptive_probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, target_fraction: float=0.25, risk_free_return: float=0.0, metric: Callable[[np.ndarray, float], float] | None=None, n_regimes: int=2)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "module": "RiskLabAI.backtest.validation.path_bagged_cpcv", + "module_kind": "module", + "sha256": "CB9FDF560D59DD7E035DD194DC1F78EDB392A9072ADE4959F6D0B55D3EFD5B83", + "size": 5682, + "defined_public_names": [ + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices" + ], + "definitions": [ + { + "name": "moving_block_bootstrap_indices", + "kind": "function", + "signature": "(n_observations: int, block_size: int, rng: np.random.Generator)" + }, + { + "name": "bagged_probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, n_bag: int=30, block_size: Optional[int]=None, risk_free_return: float=0.0, metric: Optional[Callable[[np.ndarray, float], float]]=None, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/purged_kfold.py", + "module": "RiskLabAI.backtest.validation.purged_kfold", + "module_kind": "module", + "sha256": "8E2385391624CFB31E3B9C97B0D8E27B774E1985DF6ED5997F59A1D49ACFF902", + "size": 19267, + "defined_public_names": [ + "PurgedKFold" + ], + "definitions": [ + { + "name": "PurgedKFold", + "kind": "class", + "signature": "(self, n_splits: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/walk_forward.py", + "module": "RiskLabAI.backtest.validation.walk_forward", + "module_kind": "module", + "sha256": "47C0786EAF7B5F4D774B24537ADEEDC9E80AAC3604FA3D14C52A17864FEC9B7A", + "size": 6474, + "defined_public_names": [ + "WalkForward" + ], + "definitions": [ + { + "name": "WalkForward", + "kind": "class", + "signature": "(self, n_splits: int=5, max_train_size: Optional[int]=None, gap: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/__init__.py", + "module": "RiskLabAI.causal_factor_analysis", + "module_kind": "package", + "sha256": "6B6153F1D4C87E8FA6DEDF123CFD27DA3AAD23ED1DEDE271684DFC93E71A3F3C", + "size": 4290, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "module": "RiskLabAI.causal_factor_analysis.factor_mirage", + "module_kind": "module", + "sha256": "5E5C44C8F25E926E5C1528D623BAA6886548E32C1308DB65B5AFB907988C522B", + "size": 14673, + "defined_public_names": [ + "ColliderCoefficients", + "ColliderDiagnostics", + "StrategyPerformance", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient" + ], + "definitions": [ + { + "name": "StrategyPerformance", + "kind": "class", + "signature": null + }, + { + "name": "ColliderCoefficients", + "kind": "class", + "signature": null + }, + { + "name": "ColliderDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "confounder_undercontrolled_coefficient", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "confounder_factor_return", + "kind": "function", + "signature": "(x: Real, z: Real, beta: Real, gamma: Real, delta_estimated: Real, *, delta_realized: Optional[Real]=None)" + }, + { + "name": "confounder_forecast_return", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta_estimated: Real, *, delta_realized: Optional[Real]=None)" + }, + { + "name": "collider_overcontrolled_coefficients", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_factor_return", + "kind": "function", + "signature": "(x: Real, collider_proxy: Real, beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_forecast_return", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_model_diagnostics", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real, n_observations: Integral)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "module": "RiskLabAI.causal_factor_analysis.graph_identification", + "module_kind": "module", + "sha256": "E34CED76FDA172EB688B7F45D610F3297332AE43D24639860325CE38E0A180BE", + "size": 37121, + "defined_public_names": [ + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets" + ], + "definitions": [ + { + "name": "CausalDAG", + "kind": "class", + "signature": null + }, + { + "name": "PathEvidence", + "kind": "class", + "signature": null + }, + { + "name": "DSeparationEvidence", + "kind": "class", + "signature": null + }, + { + "name": "BackdoorAdjustmentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "FrontdoorAdjustmentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "InstrumentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "NodeRoleEvidence", + "kind": "class", + "signature": null + }, + { + "name": "d_separation", + "kind": "function", + "signature": "(dag: CausalDAG, left: Iterable[str], right: Iterable[str], conditioned: Iterable[str]=(), *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "check_backdoor_adjustment_set", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, adjustment_set: Iterable[str]=(), *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "minimal_backdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, *, max_candidates: Optional[int]=_DEFAULT_MAX_CANDIDATES)" + }, + { + "name": "check_frontdoor_adjustment_set", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, mediator_set: Iterable[str], *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "minimal_frontdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, *, max_candidates: Optional[int]=_DEFAULT_MAX_CANDIDATES)" + }, + { + "name": "check_instrument", + "kind": "function", + "signature": "(dag: CausalDAG, instrument: str, treatment: str, outcome: str, conditioned: Iterable[str]=())" + }, + { + "name": "causal_role_evidence", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, node: str, *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "module": "RiskLabAI.causal_factor_analysis.optimizer", + "module_kind": "module", + "sha256": "C83DFD2630E82E3D31D45E47E3895FD8926CB2FC67A66842C4C25BDEEF2379A5", + "size": 8835, + "defined_public_names": [ + "minimum_variance_factor_weights" + ], + "definitions": [ + { + "name": "minimum_variance_factor_weights", + "kind": "function", + "signature": "(covariance: ArrayLike, factor_exposures: ArrayLike, target_exposures: ArrayLike)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/protocol.py", + "module": "RiskLabAI.causal_factor_analysis.protocol", + "module_kind": "module", + "sha256": "97A78E100A01192A9C7511DC8C18CF54300F46BF36187A87E3FF607A5C91011B", + "size": 64712, + "defined_public_names": [ + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol" + ], + "definitions": [ + { + "name": "EventHorizon", + "kind": "class", + "signature": null + }, + { + "name": "FoldEvidence", + "kind": "class", + "signature": null + }, + { + "name": "ValidationEvidence", + "kind": "class", + "signature": null + }, + { + "name": "VariableSelectionStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalDiscoveryStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalAdjustmentSetStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalExplanatoryAndPredictivePowerStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalPortfolioConstructionStage", + "kind": "class", + "signature": null + }, + { + "name": "BacktestStage", + "kind": "class", + "signature": null + }, + { + "name": "MultipleTestingAdjustmentsStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalFactorProtocolReport", + "kind": "class", + "signature": null + }, + { + "name": "validate_causal_factor_protocol", + "kind": "function", + "signature": "(report: CausalFactorProtocolReport)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "module_kind": "module", + "sha256": "5AF8F882F798FF9D0AC8F3955A9A2C9533589AA328B929030CF16930FA28FA65", + "size": 9595, + "defined_public_names": [ + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "fork_population_diagnostics", + "fork_specification_experiment" + ], + "definitions": [ + { + "name": "PopulationRegressionDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "RegressionDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "SpecificationPopulationResult", + "kind": "class", + "signature": null + }, + { + "name": "SpecificationExperimentResult", + "kind": "class", + "signature": null + }, + { + "name": "fork_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "collider_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "confounded_mediator_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "fork_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + }, + { + "name": "collider_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + }, + { + "name": "confounded_mediator_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "module_kind": "module", + "sha256": "B3D980B969803C12F0F3C5A6C6C55C37D844827131F45924F53E629F02A1E980", + "size": 11832, + "defined_public_names": [ + "DifferenceInDifferencesEstimate", + "TreatmentEffectDecomposition", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "difference_in_differences", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "randomized_mean_difference", + "treatment_effect_decomposition" + ], + "definitions": [ + { + "name": "TreatmentEffectDecomposition", + "kind": "class", + "signature": null + }, + { + "name": "DifferenceInDifferencesEstimate", + "kind": "class", + "signature": null + }, + { + "name": "average_treatment_effect", + "kind": "function", + "signature": "(treated_interventional_mean: Real, control_interventional_mean: Real)" + }, + { + "name": "treatment_effect_decomposition", + "kind": "function", + "signature": "(observed_treated_mean: Real, observed_control_mean: Real, counterfactual_control_mean_for_treated: Real)" + }, + { + "name": "randomized_mean_difference", + "kind": "function", + "signature": "(treated_outcomes: Sequence[Real], control_outcomes: Sequence[Real])" + }, + { + "name": "difference_in_differences", + "kind": "function", + "signature": "(treated_before: Real, treated_after: Real, control_before: Real, control_after: Real)" + }, + { + "name": "backdoor_adjusted_expectation", + "kind": "function", + "signature": "(conditional_outcome_means: Sequence[Real], adjustment_probabilities: Sequence[Real])" + }, + { + "name": "backdoor_adjusted_average_treatment_effect", + "kind": "function", + "signature": "(treated_conditional_means: Sequence[Real], control_conditional_means: Sequence[Real], adjustment_probabilities: Sequence[Real])" + }, + { + "name": "frontdoor_adjusted_probability", + "kind": "function", + "signature": "(mediator_probabilities_given_treatment: Sequence[Real], outcome_probabilities_given_mediator_and_treatment: Sequence[Sequence[Real]], treatment_probabilities: Sequence[Real])" + }, + { + "name": "linear_instrumental_variable_effect", + "kind": "function", + "signature": "(outcome_instrument_covariance: Real, treatment_instrument_covariance: Real)" + } + ] + }, + { + "path": "src/RiskLabAI/cluster/__init__.py", + "module": "RiskLabAI.cluster", + "module_kind": "package", + "sha256": "1313FCD1DFC97508FB0ECAE567B46BAF5CE9997748C302ABACD0AA4606D931B3", + "size": 642, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/cluster/clustering.py", + "module": "RiskLabAI.cluster.clustering", + "module_kind": "module", + "sha256": "C9ADF94D36C3B1257D33E0DBC7D38243C4876BF1921B72A9092FE81A745610B2", + "size": 14275, + "defined_public_names": [ + "cluster_k_means_base", + "cluster_k_means_top", + "covariance_to_correlation", + "make_new_outputs", + "random_block_correlation", + "random_block_covariance", + "random_covariance_sub" + ], + "definitions": [ + { + "name": "covariance_to_correlation", + "kind": "function", + "signature": "(covariance: np.ndarray)" + }, + { + "name": "cluster_k_means_base", + "kind": "function", + "signature": "(correlation: pd.DataFrame, max_clusters: int=10, iterations: int=10, random_state: Optional[int]=None)" + }, + { + "name": "make_new_outputs", + "kind": "function", + "signature": "(correlation: pd.DataFrame, clusters_1: dict[int, list[str]], clusters_2: dict[int, list[str]])" + }, + { + "name": "cluster_k_means_top", + "kind": "function", + "signature": "(correlation: pd.DataFrame, max_clusters: Optional[int]=None, iterations: int=10, random_state: Optional[int]=None)" + }, + { + "name": "random_covariance_sub", + "kind": "function", + "signature": "(n_observations: int, n_columns: int, sigma: float, random_state: Optional[int]=None)" + }, + { + "name": "random_block_covariance", + "kind": "function", + "signature": "(n_columns: int, n_blocks: int, block_size_min: int=1, sigma: float=1.0, random_state: Optional[int]=None)" + }, + { + "name": "random_block_correlation", + "kind": "function", + "signature": "(n_columns: int, n_blocks: int, random_state: Optional[int]=None, block_size_min: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/controller/__init__.py", + "module": "RiskLabAI.controller", + "module_kind": "package", + "sha256": "0DED3F4FAB2B94562CC9337E7D1A170B6ACA95D4F4477C2A6730DF29ABE0A8FF", + "size": 73, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/controller/bars_initializer.py", + "module": "RiskLabAI.controller.bars_initializer", + "module_kind": "module", + "sha256": "6C45A2CACEB4BD9AAB6B08EA0EDD240E4144CF823FCE361C5297F2DC9FFF9022", + "size": 13700, + "defined_public_names": [ + "BarsInitializerController" + ], + "definitions": [ + { + "name": "BarsInitializerController", + "kind": "class", + "signature": "(self)" + } + ] + }, + { + "path": "src/RiskLabAI/controller/data_structure_controller.py", + "module": "RiskLabAI.controller.data_structure_controller", + "module_kind": "module", + "sha256": "4D4CAFBA0B0059A89398A1CD453F57A803E80466C93DC3AC4A0F11BB894DDF8C", + "size": 5914, + "defined_public_names": [ + "BAR_COLUMNS", + "Controller" + ], + "definitions": [ + { + "name": "Controller", + "kind": "class", + "signature": "(self)" + } + ] + }, + { + "path": "src/RiskLabAI/core/__init__.py", + "module": "RiskLabAI.core", + "module_kind": "package", + "sha256": "1F57377F2CDCD8958910376A2615F45A23FD96954FDE05FFBEDC8FDC20AD1DE8", + "size": 1933, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/core/_builtins.py", + "module": "RiskLabAI.core._builtins", + "module_kind": "module", + "sha256": "5D754E23EF41F580D0A4B5CB635DFA95A5A1AB20E3A500B1D4B9F9A75EB333A7", + "size": 7247, + "defined_public_names": [ + "BARS", + "BET_SIZERS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "PORTFOLIO_OPTIMIZERS", + "REGISTRIES", + "get_registry", + "list_components" + ], + "definitions": [ + { + "name": "get_registry", + "kind": "function", + "signature": "(family: str)" + }, + { + "name": "list_components", + "kind": "function", + "signature": "()" + } + ] + }, + { + "path": "src/RiskLabAI/core/base.py", + "module": "RiskLabAI.core.base", + "module_kind": "module", + "sha256": "47F3B61B748629A7AFBB1209A701A8F6614F2474EA9919B298B4574D3E757C5E", + "size": 6270, + "defined_public_names": [ + "BaseBetSizer", + "BaseLabeler", + "BasePortfolioOptimizer", + "Estimator" + ], + "definitions": [ + { + "name": "Estimator", + "kind": "class", + "signature": null + }, + { + "name": "BaseLabeler", + "kind": "class", + "signature": null + }, + { + "name": "BaseBetSizer", + "kind": "class", + "signature": null + }, + { + "name": "BasePortfolioOptimizer", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/core/registry.py", + "module": "RiskLabAI.core.registry", + "module_kind": "module", + "sha256": "2A48E47D33ED1E2EE074DEE4933C363884433226EB42CA4BFCA949E33AAC1016", + "size": 14263, + "defined_public_names": [ + "Registry" + ], + "definitions": [ + { + "name": "Registry", + "kind": "class", + "signature": "(self, name: str, *, base: type | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/__init__.py", + "module": "RiskLabAI.data", + "module_kind": "package", + "sha256": "BF1701C8581DC602A5D37E10BAE6CDB837D489BD8791781B5F50BA4D067876FD", + "size": 630, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/denoise/__init__.py", + "module": "RiskLabAI.data.denoise", + "module_kind": "package", + "sha256": "72E4069E071BF4ED278796A6FF14751B99ED9B64D71D0175F57150C1F00427F0", + "size": 844, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/denoise/denoising.py", + "module": "RiskLabAI.data.denoise.denoising", + "module_kind": "module", + "sha256": "4F78D615A8B429309E2840AFBE1429D6340231F5DE73BEAA93C4741545AAA06E", + "size": 15138, + "defined_public_names": [ + "corr_to_cov", + "cov_to_corr", + "denoise_cov", + "denoised_corr", + "denoised_corr2", + "find_max_eval", + "fit_kde", + "marcenko_pastur_pdf", + "optimal_portfolio", + "optimal_portfolio_denoised", + "pca" + ], + "definitions": [ + { + "name": "marcenko_pastur_pdf", + "kind": "function", + "signature": "(variance: float, q: float, num_points: int=1000)" + }, + { + "name": "fit_kde", + "kind": "function", + "signature": "(observations: np.ndarray, bandwidth: float=0.01, kernel: str='gaussian')" + }, + { + "name": "find_max_eval", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, q: float, bandwidth: float)" + }, + { + "name": "denoised_corr", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int)" + }, + { + "name": "denoised_corr2", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_factors: int, alpha: float=0.0)" + }, + { + "name": "pca", + "kind": "function", + "signature": "(matrix: np.ndarray)" + }, + { + "name": "cov_to_corr", + "kind": "function", + "signature": "(cov: np.ndarray)" + }, + { + "name": "corr_to_cov", + "kind": "function", + "signature": "(corr: np.ndarray, std: np.ndarray)" + }, + { + "name": "denoise_cov", + "kind": "function", + "signature": "(cov0: np.ndarray, q: float, bandwidth: float=0.01)" + }, + { + "name": "optimal_portfolio", + "kind": "function", + "signature": "(cov: np.ndarray, mu: Optional[np.ndarray]=None)" + }, + { + "name": "optimal_portfolio_denoised", + "kind": "function", + "signature": "(cov: np.ndarray, q: float, mu: Optional[np.ndarray]=None, bandwidth: float=0.01)" + } + ] + }, + { + "path": "src/RiskLabAI/data/denoise/nercome.py", + "module": "RiskLabAI.data.denoise.nercome", + "module_kind": "module", + "sha256": "81FEA0C543026F0C927DBC54CAC60DA5F3829DAFD84EAB788B59F7F8462BAFBB", + "size": 6386, + "defined_public_names": [ + "nercome_denoised_covariance" + ], + "definitions": [ + { + "name": "nercome_denoised_covariance", + "kind": "function", + "signature": "(returns: np.ndarray, n_splits: int=50, split_fraction: float=2.0 / 3.0, random_state: int | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/differentiation/__init__.py", + "module": "RiskLabAI.data.differentiation", + "module_kind": "package", + "sha256": "51D73513B8108EF8835E5113E337F336DD62CBC4006BBBD486B50623401B670B", + "size": 196, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "module": "RiskLabAI.data.differentiation.adaptive_differentiation", + "module_kind": "module", + "sha256": "BD56143C44222FBACDF490FA366063CA04559FD180FF4B62E667F9288139F610", + "size": 11374, + "defined_public_names": [ + "DEFAULT_ADF_MAXLAG", + "DEFAULT_ADF_SIGNIFICANCE", + "DEFAULT_CV_THRESHOLDS", + "DEFAULT_DELTA_GRID", + "adaptive_differencing_order", + "adaptive_fractional_difference", + "rescaled_range_hurst", + "wavelet_variance_hurst" + ], + "definitions": [ + { + "name": "wavelet_variance_hurst", + "kind": "function", + "signature": "(series: np.ndarray, wavelet: str='db2', skip_fine: int=1)" + }, + { + "name": "rescaled_range_hurst", + "kind": "function", + "signature": "(series: np.ndarray)" + }, + { + "name": "adaptive_differencing_order", + "kind": "function", + "signature": "(increments: np.ndarray)" + }, + { + "name": "adaptive_fractional_difference", + "kind": "function", + "signature": "(series: pd.Series, delta_grid: list | None=None, cv_thresholds: tuple=DEFAULT_CV_THRESHOLDS, adf_significance: float=DEFAULT_ADF_SIGNIFICANCE, adf_maxlag: int=DEFAULT_ADF_MAXLAG)" + } + ] + }, + { + "path": "src/RiskLabAI/data/differentiation/differentiation.py", + "module": "RiskLabAI.data.differentiation.differentiation", + "module_kind": "module", + "sha256": "4E9E45C90AEB0B8949E5F6F774D1AF20AA5973188F5FD23FFB4344E6EF5C5E9F", + "size": 11859, + "defined_public_names": [ + "calculate_weights_ffd", + "calculate_weights_std", + "find_optimal_ffd_simple", + "fractional_difference_fixed", + "fractional_difference_fixed_single", + "fractional_difference_std", + "fractionally_differentiated_log_price", + "plot_weights" + ], + "definitions": [ + { + "name": "calculate_weights_std", + "kind": "function", + "signature": "(degree: float, size: int)" + }, + { + "name": "calculate_weights_ffd", + "kind": "function", + "signature": "(degree: float, threshold: float=1e-05)" + }, + { + "name": "fractional_difference_std", + "kind": "function", + "signature": "(series: pd.DataFrame, degree: float, threshold: float=0.01)" + }, + { + "name": "fractional_difference_fixed", + "kind": "function", + "signature": "(series: pd.DataFrame, degree: float, threshold: float=1e-05)" + }, + { + "name": "fractional_difference_fixed_single", + "kind": "function", + "signature": "(series: pd.Series, degree: float, threshold: float=1e-05)" + }, + { + "name": "plot_weights", + "kind": "function", + "signature": "(degree_range: tuple[float, float], number_degrees: int, size: int, ax: Optional['plt.Axes']=None)" + }, + { + "name": "find_optimal_ffd_simple", + "kind": "function", + "signature": "(input_series: pd.DataFrame, p_value_threshold: float=0.05)" + }, + { + "name": "fractionally_differentiated_log_price", + "kind": "function", + "signature": "(input_series: pd.Series, threshold: float=1e-05, step: float=0.01, p_value_threshold: float=0.05)" + } + ] + }, + { + "path": "src/RiskLabAI/data/distance/__init__.py", + "module": "RiskLabAI.data.distance", + "module_kind": "package", + "sha256": "5BA3546F05199F587C42C2C8A4E80B541616EA3F80A93654B7F2E8D8F548F296", + "size": 102, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/distance/codependence.py", + "module": "RiskLabAI.data.distance.codependence", + "module_kind": "module", + "sha256": "DF6E672A5D25468FCE63B36C6AFAAF55B6AD14486E34D4CC42D84D7B43F278DA", + "size": 7241, + "defined_public_names": [ + "distance_correlation", + "ksg_mutual_information" + ], + "definitions": [ + { + "name": "ksg_mutual_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, k: int=4, random_state: int | None=0)" + }, + { + "name": "distance_correlation", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/data/distance/distance_metric.py", + "module": "RiskLabAI.data.distance.distance_metric", + "module_kind": "module", + "sha256": "89073FAA39E9FF195C7028BEA970410562745534AE52AAFD310C5499E4DC914D", + "size": 8073, + "defined_public_names": [ + "calculate_cross_entropy", + "calculate_distance", + "calculate_kullback_leibler_divergence", + "calculate_mutual_information", + "calculate_number_of_bins", + "calculate_variation_of_information", + "calculate_variation_of_information_extended" + ], + "definitions": [ + { + "name": "calculate_variation_of_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, bins: int, norm: bool=False)" + }, + { + "name": "calculate_number_of_bins", + "kind": "function", + "signature": "(num_observations: int, correlation: Optional[float]=None)" + }, + { + "name": "calculate_variation_of_information_extended", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, norm: bool=False)" + }, + { + "name": "calculate_mutual_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, norm: bool=False)" + }, + { + "name": "calculate_distance", + "kind": "function", + "signature": "(dependence: np.ndarray, metric: str='angular')" + }, + { + "name": "calculate_kullback_leibler_divergence", + "kind": "function", + "signature": "(p: np.ndarray, q: np.ndarray)" + }, + { + "name": "calculate_cross_entropy", + "kind": "function", + "signature": "(p: np.ndarray, q: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/data/labeling/__init__.py", + "module": "RiskLabAI.data.labeling", + "module_kind": "package", + "sha256": "CBDD997697143DFFD4287A26E568C257FDE2634BDBA325955438353D65DC86F7", + "size": 1144, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/labeling/financial_labels.py", + "module": "RiskLabAI.data.labeling.financial_labels", + "module_kind": "module", + "sha256": "A2471BEAFFB28F2339D05DAED1811C4384C702CC0602638230D0FC03A4BE6A9C", + "size": 4424, + "defined_public_names": [ + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning" + ], + "definitions": [ + { + "name": "calculate_t_value_linear_regression", + "kind": "function", + "signature": "(prices: pd.Series)" + }, + { + "name": "find_trend_using_trend_scanning", + "kind": "function", + "signature": "(molecule: pd.Index, close: pd.Series, span: tuple[int, int])" + } + ] + }, + { + "path": "src/RiskLabAI/data/labeling/labeling.py", + "module": "RiskLabAI.data.labeling.labeling", + "module_kind": "module", + "sha256": "36687544C68B3016B5E4CFDE12ADE104971F45F79D4AC9FEB99BEB298F8E1418", + "size": 14680, + "defined_public_names": [ + "cusum_filter_events_dynamic_threshold", + "daily_volatility_with_log_returns", + "meta_events", + "meta_labeling", + "symmetric_cusum_filter", + "triple_barrier", + "vertical_barrier" + ], + "definitions": [ + { + "name": "cusum_filter_events_dynamic_threshold", + "kind": "function", + "signature": "(prices: pd.Series, threshold: pd.Series)" + }, + { + "name": "symmetric_cusum_filter", + "kind": "function", + "signature": "(prices: pd.Series, threshold: float)" + }, + { + "name": "daily_volatility_with_log_returns", + "kind": "function", + "signature": "(close: pd.Series, span: int=100)" + }, + { + "name": "vertical_barrier", + "kind": "function", + "signature": "(close: pd.Series, time_events: pd.DatetimeIndex, number_days: int)" + }, + { + "name": "triple_barrier", + "kind": "function", + "signature": "(close: pd.Series, events: pd.DataFrame, ptsl: list[float], molecule: list[pd.Timestamp])" + }, + { + "name": "meta_events", + "kind": "function", + "signature": "(close: pd.Series, time_events: pd.DatetimeIndex, ptsl: list[float], target: pd.Series, return_min: float, num_threads: int, vertical_barrier_times: Optional[pd.Series]=None, side: Optional[pd.Series]=None)" + }, + { + "name": "meta_labeling", + "kind": "function", + "signature": "(events: pd.DataFrame, close: pd.Series)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/__init__.py", + "module": "RiskLabAI.data.structures", + "module_kind": "package", + "sha256": "F0E79621D4AF2B9E8C8BE1EBEF26EBE66536A5A3A50CF09E1A16BAEA94834398", + "size": 873, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_bars.py", + "module": "RiskLabAI.data.structures.abstract_bars", + "module_kind": "module", + "sha256": "F775F93AE69A634E22EA324A0AB8603E729786CB72A684741D2111E090DA6014", + "size": 7436, + "defined_public_names": [ + "AbstractBars" + ], + "definitions": [ + { + "name": "AbstractBars", + "kind": "class", + "signature": "(self, bar_type: str)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "module": "RiskLabAI.data.structures.abstract_imbalance_bars", + "module_kind": "module", + "sha256": "E7A54B451B412FD1F2463E59E8706131A1D7AD8C84D3FE6CC1E2B5A3DC8B1B99", + "size": 6210, + "defined_public_names": [ + "AbstractImbalanceBars" + ], + "definitions": [ + { + "name": "AbstractImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], window_size_for_expected_imbalance_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, analyse_thresholds: bool)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "module": "RiskLabAI.data.structures.abstract_information_driven_bars", + "module_kind": "module", + "sha256": "8601CD7E405A10F66BF6A82192E8EA261F9D71CA2A8C85E48FFA334EBC00D8DF", + "size": 4021, + "defined_public_names": [ + "AbstractInformationDrivenBars" + ], + "definitions": [ + { + "name": "AbstractInformationDrivenBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_run_bars.py", + "module": "RiskLabAI.data.structures.abstract_run_bars", + "module_kind": "module", + "sha256": "1B7E12ED9C1B2505F014FE5487CF1AFC938463198C8B40F7360ED1739E035EFD", + "size": 9943, + "defined_public_names": [ + "AbstractRunBars" + ], + "definitions": [ + { + "name": "AbstractRunBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], window_size_for_expected_imbalance_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, analyse_thresholds: bool)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "module": "RiskLabAI.data.structures.hardened_information_driven_bars", + "module_kind": "module", + "sha256": "48BCDD8B73D5DED1815505696C36D56528ECBAED3A295EDCE31DE56AB2B3B8EB", + "size": 6781, + "defined_public_names": [ + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars" + ], + "definitions": [ + { + "name": "HardenedExpectedImbalanceBars", + "kind": "class", + "signature": "(self, *args, min_ticks: int, max_ticks: int, **kwargs)" + }, + { + "name": "HardenedExpectedRunBars", + "kind": "class", + "signature": "(self, *args, min_ticks: int, max_ticks: int, **kwargs)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/imbalance_bars.py", + "module": "RiskLabAI.data.structures.imbalance_bars", + "module_kind": "module", + "sha256": "D355A01123732210788FAB14EDC8D68ABC6FA4E1035AC5BD636BD11DD4E210AC", + "size": 4741, + "defined_public_names": [ + "ExpectedImbalanceBars", + "FixedImbalanceBars" + ], + "definitions": [ + { + "name": "ExpectedImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: int=10000, initial_estimate_of_expected_n_ticks_in_bar: int=20000, window_size_for_expected_imbalance_estimation: int=10000, expected_ticks_number_bounds: Optional[tuple[float, float]]=None, analyse_thresholds: bool=False)" + }, + { + "name": "FixedImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, initial_estimate_of_expected_n_ticks_in_bar: int=20000, window_size_for_expected_imbalance_estimation: int=10000, analyse_thresholds: bool=False, window_size_for_expected_n_ticks_estimation: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/run_bars.py", + "module": "RiskLabAI.data.structures.run_bars", + "module_kind": "module", + "sha256": "7FE0F23DF5BE39B475073CA69FABD6FDE7126EC6E4289C2D84652D930D251900", + "size": 4515, + "defined_public_names": [ + "ExpectedRunBars", + "FixedRunBars" + ], + "definitions": [ + { + "name": "ExpectedRunBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int, expected_ticks_number_bounds: Optional[tuple[float, float]]=None, analyse_thresholds: bool=False)" + }, + { + "name": "FixedRunBars", + "kind": "class", + "signature": "(self, bar_type: str, initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int, analyse_thresholds: bool=False, window_size_for_expected_n_ticks_estimation: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/standard_bars.py", + "module": "RiskLabAI.data.structures.standard_bars", + "module_kind": "module", + "sha256": "C4142C10BEE79EF96E8A9DA31F1444EE062933D587630EB567E0A5CE3C81C362", + "size": 2669, + "defined_public_names": [ + "StandardBars" + ], + "definitions": [ + { + "name": "StandardBars", + "kind": "class", + "signature": "(self, bar_type: str, threshold: float=50000)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/time_bars.py", + "module": "RiskLabAI.data.structures.time_bars", + "module_kind": "module", + "sha256": "F9126A81DA595A4E1D87400164C09071833222CD9C1546D69164F643D0AC582F", + "size": 4544, + "defined_public_names": [ + "TimeBars" + ], + "definitions": [ + { + "name": "TimeBars", + "kind": "class", + "signature": "(self, resolution_type: str, resolution_units: int)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/__init__.py", + "module": "RiskLabAI.data.synthetic_data", + "module_kind": "package", + "sha256": "BE3B0444592D9B45C9439E57210EB713C761D4B21CE497609B2B63E955F70768", + "size": 857, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "module": "RiskLabAI.data.synthetic_data.drift_burst_hypothesis", + "module_kind": "module", + "sha256": "3A613F5FE50555654556236EF409BC4F7FB83252CDF3502CA60D46EB29F4BC4C", + "size": 3115, + "defined_public_names": [ + "drift_volatility_burst" + ], + "definitions": [ + { + "name": "drift_volatility_burst", + "kind": "function", + "signature": "(bubble_length: int, a_before: float, a_after: float, b_before: float, b_after: float, alpha: float, beta: float, explosion_filter_width: float=0.1)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/simulation.py", + "module": "RiskLabAI.data.synthetic_data.simulation", + "module_kind": "module", + "sha256": "09394D81D758211140D894B5CB4CB6289EACEC862638006C8A7D9A79FD16DC55", + "size": 3392, + "defined_public_names": [ + "form_block_matrix", + "form_true_matrix", + "random_cov", + "simulates_cov_mu" + ], + "definitions": [ + { + "name": "random_cov", + "kind": "function", + "signature": "(num_columns: int, num_factors: int)" + }, + { + "name": "form_block_matrix", + "kind": "function", + "signature": "(n_blocks: int, block_size: int, block_correlation: float)" + }, + { + "name": "form_true_matrix", + "kind": "function", + "signature": "(n_blocks: int, block_size: int, block_correlation: float)" + }, + { + "name": "simulates_cov_mu", + "kind": "function", + "signature": "(mu0: np.ndarray, cov0: np.ndarray, n_obs: int, shrink: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "module": "RiskLabAI.data.synthetic_data.synthetic_controlled_environment", + "module_kind": "module", + "sha256": "6D5DFCF4C4430A7BAE367A79CB156D3D6DF3D24457C1D60B68A1A4D556CF4AD4", + "size": 12391, + "defined_public_names": [ + "align_params_length", + "compute_log_returns", + "generate_prices_from_regimes", + "heston_merton_log_returns", + "parallel_generate_prices" + ], + "definitions": [ + { + "name": "compute_log_returns", + "kind": "function", + "signature": "(n_steps: int, mu_vector: np.ndarray, kappa_vector: np.ndarray, theta_vector: np.ndarray, xi_vector: np.ndarray, dw_stock: np.ndarray, dw_vol: np.ndarray, jump_comp: np.ndarray, poisson_var: np.ndarray, dt: float, sqrt_dt: float, lambda_vector: np.ndarray, m_vector: np.ndarray, v_vector: np.ndarray, regime_change: np.ndarray)" + }, + { + "name": "heston_merton_log_returns", + "kind": "function", + "signature": "(total_time: float, n_steps: int, mu_vector: np.ndarray, kappa_vector: np.ndarray, theta_vector: np.ndarray, xi_vector: np.ndarray, rho_vector: np.ndarray, lambda_vector: np.ndarray, m_vector: np.ndarray, v_vector: np.ndarray, regime_change: np.ndarray, random_state: Optional[int]=None)" + }, + { + "name": "align_params_length", + "kind": "function", + "signature": "(regime_params: RegimeParams)" + }, + { + "name": "generate_prices_from_regimes", + "kind": "function", + "signature": "(regimes: dict[str, RegimeParams], transition_matrix: np.ndarray, total_time: float, n_steps: int, random_state: Optional[int]=None)" + }, + { + "name": "parallel_generate_prices", + "kind": "function", + "signature": "(number_of_paths: int, regimes: dict[str, RegimeParams], transition_matrix: np.ndarray, total_time: float, n_steps: int, random_state: Optional[int]=None, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/data/weights/__init__.py", + "module": "RiskLabAI.data.weights", + "module_kind": "package", + "sha256": "7275D7101AE1F1DC977EF7B6C53819139AC9F46F629742178AA0AA453390A523", + "size": 621, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/weights/sample_weights.py", + "module": "RiskLabAI.data.weights.sample_weights", + "module_kind": "module", + "sha256": "4DA73877840EF49BDFF35B087CE1FC52354D14E9136217D16B1948C5577A9BB4", + "size": 6991, + "defined_public_names": [ + "calculate_average_uniqueness", + "calculate_time_decay", + "expand_label_for_meta_labeling", + "sample_weight_absolute_return_meta_labeling" + ], + "definitions": [ + { + "name": "expand_label_for_meta_labeling", + "kind": "function", + "signature": "(close_index: pd.Index, timestamp: pd.Series, molecule: pd.Index)" + }, + { + "name": "calculate_average_uniqueness", + "kind": "function", + "signature": "(index_matrix: pd.DataFrame)" + }, + { + "name": "sample_weight_absolute_return_meta_labeling", + "kind": "function", + "signature": "(timestamp: pd.Series, price: pd.Series, molecule: pd.Index)" + }, + { + "name": "calculate_time_decay", + "kind": "function", + "signature": "(weight: pd.Series, clf_last_weight: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/ensemble/__init__.py", + "module": "RiskLabAI.ensemble", + "module_kind": "package", + "sha256": "2301547AD658B8DD96941FD1098FBDA9AC3464E487F2A9F1811C6BD656F8F46A", + "size": 707, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "module": "RiskLabAI.ensemble.bagging_classifier_accuracy", + "module_kind": "module", + "sha256": "9B2150DA7DB358161927137E5E2ED9C3FA0249FF71E86401F29C9F96258B839C", + "size": 1255, + "defined_public_names": [ + "bagging_classifier_accuracy" + ], + "definitions": [ + { + "name": "bagging_classifier_accuracy", + "kind": "function", + "signature": "(N: int, p: float)" + } + ] + }, + { + "path": "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "module": "RiskLabAI.ensemble.empirical_bagging_accuracy", + "module_kind": "module", + "sha256": "E679640BE29A4986F0E67AF7F5A060894A72EDCD581C3F2B059D9C4177B7415F", + "size": 11622, + "defined_public_names": [ + "BaggingClassifierAccuracy", + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution" + ], + "definitions": [ + { + "name": "BaggingClassifierAccuracy", + "kind": "class", + "signature": "(self, n_estimators: int=1000, max_samples: int=100, max_features: float=1.0, random_state: Optional[int]=None)" + }, + { + "name": "calculate_bootstrap_accuracy", + "kind": "function", + "signature": "(clf: BaggingClassifier, X: pd.DataFrame, y: pd.Series, n_bootstraps: int=1000)" + }, + { + "name": "plot_bootstrap_accuracy_distribution", + "kind": "function", + "signature": "(a_n_values: np.ndarray, a_n_mean: float, a_n_std: float, ax: Optional['plt.Axes']=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/__init__.py", + "module": "RiskLabAI.features", + "module_kind": "package", + "sha256": "CE30ECC88CBFF140893C4FADFBE38DC90EE954024126EF66F1A5200AE0146C2B", + "size": 117, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/entropy_features/__init__.py", + "module": "RiskLabAI.features.entropy_features", + "module_kind": "package", + "sha256": "2DB4F80E1037C4562E8034FF3120C8DF41DE54D85C7F8E1CF04628BBA9A52374", + "size": 883, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "module": "RiskLabAI.features.entropy_features.bias_corrected", + "module_kind": "module", + "sha256": "B6E66D401B569B26452EC1129A8FB5F0B150A990595BEEBFA92437457B1DAC1D", + "size": 12824, + "defined_public_names": [ + "grassberger_entropy", + "miller_madow_entropy", + "nsb_entropy" + ], + "definitions": [ + { + "name": "miller_madow_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + }, + { + "name": "grassberger_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + }, + { + "name": "nsb_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1, alphabet_size: Optional[int]=None, n_points: int=80)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "module": "RiskLabAI.features.entropy_features.kontoyiannis", + "module_kind": "module", + "sha256": "A69F1FEB3DAC762724F5ED9F17BCAD0E9D9EC88B5946348FA80EA4059AD04119", + "size": 2793, + "defined_public_names": [ + "kontoyiannis_entropy", + "longest_match_length" + ], + "definitions": [ + { + "name": "longest_match_length", + "kind": "function", + "signature": "(message: str, i: int, n: int)" + }, + { + "name": "kontoyiannis_entropy", + "kind": "function", + "signature": "(message: str, window: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "module": "RiskLabAI.features.entropy_features.lempel_ziv", + "module_kind": "module", + "sha256": "1F015F187028C1957CC91BC91D944B10202D7D3EF37F3CAAF29D4CFB238C8FF8", + "size": 976, + "defined_public_names": [ + "lempel_ziv_entropy" + ], + "definitions": [ + { + "name": "lempel_ziv_entropy", + "kind": "function", + "signature": "(message: str)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/plug_in.py", + "module": "RiskLabAI.features.entropy_features.plug_in", + "module_kind": "module", + "sha256": "799C9ABB1AD8A864BF15E5F1E3267C5FCB482B83C38A4F0FCF9E4C5F2C81B763", + "size": 942, + "defined_public_names": [ + "plug_in_entropy_estimator" + ], + "definitions": [ + { + "name": "plug_in_entropy_estimator", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/pmf.py", + "module": "RiskLabAI.features.entropy_features.pmf", + "module_kind": "module", + "sha256": "55A1BA2FAC7BDC384C15E36B7874E01C0DD0B40D5D8FF6E6113972B81FED8D24", + "size": 1798, + "defined_public_names": [ + "ngram_counts", + "probability_mass_function" + ], + "definitions": [ + { + "name": "ngram_counts", + "kind": "function", + "signature": "(message: str, approximate_word_length: int)" + }, + { + "name": "probability_mass_function", + "kind": "function", + "signature": "(message: str, approximate_word_length: int)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/shannon.py", + "module": "RiskLabAI.features.entropy_features.shannon", + "module_kind": "module", + "sha256": "FBA55BE6A0B6E529ABD77BE038F5F885945B034A04DA042CFFD99A278735B37E", + "size": 689, + "defined_public_names": [ + "shannon_entropy" + ], + "definitions": [ + { + "name": "shannon_entropy", + "kind": "function", + "signature": "(message: str)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/__init__.py", + "module": "RiskLabAI.features.feature_importance", + "module_kind": "package", + "sha256": "24C80AA3C46B3237D6FBC513C4E4A7EED88A54304C94792E5F24F295A00AFB73", + "size": 2076, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mda", + "module_kind": "module", + "sha256": "7BF977CB4D65189BB478CAA8378D3094E7D0A8FD672C498AC3BD881FF0B818A6", + "size": 5361, + "defined_public_names": [ + "ClusteredFeatureImportanceMDA" + ], + "definitions": [ + { + "name": "ClusteredFeatureImportanceMDA", + "kind": "class", + "signature": "(self, classifier: object, clusters: dict[str, list[str]], n_splits: int=10, random_state: int=42)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mdi", + "module_kind": "module", + "sha256": "4B2B7A7F98BA4030A8B0B885C2D9077F22445F3D5B07FF470D1E247BF4346516", + "size": 3403, + "defined_public_names": [ + "ClusteredFeatureImportanceMDI" + ], + "definitions": [ + { + "name": "ClusteredFeatureImportanceMDI", + "kind": "class", + "signature": "(self, classifier: BaseEnsemble, clusters: dict[str, list[str]])" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "module": "RiskLabAI.features.feature_importance.debiased_importance", + "module_kind": "module", + "sha256": "49B8D749041A18283724E5563E1FBB70F4143BECE89C109EA70B2D1BC7BB5750", + "size": 11360, + "defined_public_names": [ + "conditional_predictive_impact", + "mdi_plus_importance" + ], + "definitions": [ + { + "name": "mdi_plus_importance", + "kind": "function", + "signature": "(x: pd.DataFrame, y: pd.Series, random_state: int=0, n_trees: int=_DEFAULT_TREES_DECOMP, classifier: RandomForestClassifier | None=None)" + }, + { + "name": "conditional_predictive_impact", + "kind": "function", + "signature": "(x: pd.DataFrame, y: pd.Series, random_state: int=0, n_splits: int=4, n_estimators: int=80, classifier: RandomForestClassifier | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_controller", + "module_kind": "module", + "sha256": "52907FFC70EE8654B29E2552FE03AD22D494DFFC9E4777021BE0E2A199E8966A", + "size": 1399, + "defined_public_names": [ + "FeatureImportanceController" + ], + "definitions": [ + { + "name": "FeatureImportanceController", + "kind": "class", + "signature": "(self, strategy_type: str, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_factory", + "module_kind": "module", + "sha256": "5D44788D9C58D531D4913E7C9D8E4C9BCEB2A634ED3CF05EF60D882EE1084A26", + "size": 1620, + "defined_public_names": [ + "FeatureImportanceFactory" + ], + "definitions": [ + { + "name": "FeatureImportanceFactory", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_mda", + "module_kind": "module", + "sha256": "872D141D6D96B4CED9660710ACE3EFBFE4FCCB74065BEEA6FEF5344DF1A61861", + "size": 4515, + "defined_public_names": [ + "FeatureImportanceMDA" + ], + "definitions": [ + { + "name": "FeatureImportanceMDA", + "kind": "class", + "signature": "(self, classifier: object, n_splits: int=10, random_state: int=42)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_mdi", + "module_kind": "module", + "sha256": "C201C6FA3986268AE49D7D2AFE177DBB468AA4E5D32CD9B9CA805DAEAA9EE5AE", + "size": 2651, + "defined_public_names": [ + "FeatureImportanceMDI" + ], + "definitions": [ + { + "name": "FeatureImportanceMDI", + "kind": "class", + "signature": "(self, classifier: BaseEnsemble)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_sfi", + "module_kind": "module", + "sha256": "69ED5D5AEA7F977E72459492D278ED5A1A3B0B5A7099CED6C96D1B5EE26C4833", + "size": 3865, + "defined_public_names": [ + "FeatureImportanceSFI" + ], + "definitions": [ + { + "name": "FeatureImportanceSFI", + "kind": "class", + "signature": "(self, classifier: object, n_splits: int=10, scoring: str='log_loss')" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_strategy", + "module_kind": "module", + "sha256": "0B5BF2F9EA40AEF208FAF010308BE4E0AF32B9378B395A8B555CF2CB1D97CCAA", + "size": 857, + "defined_public_names": [ + "FeatureImportanceStrategy" + ], + "definitions": [ + { + "name": "FeatureImportanceStrategy", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "module": "RiskLabAI.features.feature_importance.generate_synthetic_data", + "module_kind": "module", + "sha256": "67342708D90B9AC31BF31FF851E341D122600D7A75818F20CEFF31CB39E41747", + "size": 2535, + "defined_public_names": [ + "get_test_dataset" + ], + "definitions": [ + { + "name": "get_test_dataset", + "kind": "function", + "signature": "(n_features: int=100, n_informative: int=25, n_redundant: int=25, n_samples: int=10000, random_state: int=0, sigma_std: float=0.0)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "module": "RiskLabAI.features.feature_importance.orthogonal_features", + "module_kind": "module", + "sha256": "0250FFD2E9B7E2D1B5BFBF960D63EA32CBB0CD9335249222E37A66724B6B0FF9", + "size": 3140, + "defined_public_names": [ + "orthogonal_features" + ], + "definitions": [ + { + "name": "orthogonal_features", + "kind": "function", + "signature": "(features: pd.DataFrame, variance_threshold: float=0.95)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "module": "RiskLabAI.features.feature_importance.weighted_tau", + "module_kind": "module", + "sha256": "706F53A5531A3B5089A9A252883558C7561C9B1B0DB6B251904EE80A8961C098", + "size": 974, + "defined_public_names": [ + "calculate_weighted_tau" + ], + "definitions": [ + { + "name": "calculate_weighted_tau", + "kind": "function", + "signature": "(feature_importances: np.ndarray, principal_component_ranks: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/__init__.py", + "module": "RiskLabAI.features.microstructural_features", + "module_kind": "package", + "sha256": "5A47422A191048EB40286C2BA1169BA3FF04396BFA08F9582439A26521A06353", + "size": 708, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "module": "RiskLabAI.features.microstructural_features.bekker_parkinson_volatility_estimator", + "module_kind": "module", + "sha256": "BC606B1E368BC738846DCA08DE31C99A0A42D91F0CF5F8BB686EDE14021821FF", + "size": 2058, + "defined_public_names": [ + "bekker_parkinson_volatility_estimates", + "sigma_estimates" + ], + "definitions": [ + { + "name": "sigma_estimates", + "kind": "function", + "signature": "(beta: pd.Series, gamma: pd.Series)" + }, + { + "name": "bekker_parkinson_volatility_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "module": "RiskLabAI.features.microstructural_features.corwin_schultz", + "module_kind": "module", + "sha256": "9120D5D7F9763E017A896E5F7A3288F02112E1A72AA92C42717E6B2506070532", + "size": 3471, + "defined_public_names": [ + "alpha_estimates", + "beta_estimates", + "corwin_schultz_estimator", + "gamma_estimates" + ], + "definitions": [ + { + "name": "beta_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int)" + }, + { + "name": "gamma_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series)" + }, + { + "name": "alpha_estimates", + "kind": "function", + "signature": "(beta: pd.Series, gamma: pd.Series)" + }, + { + "name": "corwin_schultz_estimator", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/edge.py", + "module": "RiskLabAI.features.microstructural_features.edge", + "module_kind": "module", + "sha256": "D9C99CB172B1AA9B6F33AF39723D9BE4880214466F863F6177D7B028D7F73B61", + "size": 6251, + "defined_public_names": [ + "edge_estimator" + ], + "definitions": [ + { + "name": "edge_estimator", + "kind": "function", + "signature": "(open: _ArrayLike, high: _ArrayLike, low: _ArrayLike, close: _ArrayLike, sign: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/__init__.py", + "module": "RiskLabAI.features.structural_breaks", + "module_kind": "package", + "sha256": "46AA8C0C7DA4955E2A6397B7FCCC51B6D48C3A7016E9413A4D356EDF22D69C2F", + "size": 1122, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "module": "RiskLabAI.features.structural_breaks.pelt_change_points", + "module_kind": "module", + "sha256": "1F913C5EE9BEF2144D687068F0526C8A0F0AE5A9B0A51991E5EC686701307070", + "size": 4218, + "defined_public_names": [ + "pelt_change_points" + ], + "definitions": [ + { + "name": "pelt_change_points", + "kind": "function", + "signature": "(series, model: str='normal', penalty_multiplier: float=3.0, min_size: int=10, jump: int=5)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "module": "RiskLabAI.features.structural_breaks.structural_breaks", + "module_kind": "module", + "sha256": "0C562C044C063DFA0606EB88D74A33B54B08A69FA19A126D1B8DB7FCD3276242", + "size": 25396, + "defined_public_names": [ + "compute_beta", + "get_bsadf_sequence", + "get_bsadf_statistic", + "get_bubble_episodes", + "get_expanding_window_adf", + "get_gsadf_statistic", + "get_sadf_sequence", + "lag_dataframe", + "prepare_data", + "psy_minimum_window", + "simulate_psy_critical_values" + ], + "definitions": [ + { + "name": "lag_dataframe", + "kind": "function", + "signature": "(market_data: pd.DataFrame, lags: Union[int, list[int]])" + }, + { + "name": "prepare_data", + "kind": "function", + "signature": "(log_price_series: pd.Series, constant: str, lags: int)" + }, + { + "name": "compute_beta", + "kind": "function", + "signature": "(y_window: np.ndarray, x_window: np.ndarray)" + }, + { + "name": "get_expanding_window_adf", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str, lags: int)" + }, + { + "name": "get_bsadf_statistic", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str, lags: int)" + }, + { + "name": "psy_minimum_window", + "kind": "function", + "signature": "(sample_length: int)" + }, + { + "name": "get_sadf_sequence", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_bsadf_sequence", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_gsadf_statistic", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_bubble_episodes", + "kind": "function", + "signature": "(statistic_sequence: pd.Series, critical_value_sequence: Union[pd.Series, np.ndarray, float], min_duration: int=1)" + }, + { + "name": "simulate_psy_critical_values", + "kind": "function", + "signature": "(sample_length: int, min_sample_length: Optional[int]=None, constant: str='c', lags: int=0, n_simulations: int=2000, level: float=0.95, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "module": "RiskLabAI.features.structural_breaks.volatility_robust_sadf", + "module_kind": "module", + "sha256": "69B7F026E05E66DE4755E1A591602A59AD3B5CFE49B29DE03724F37919BDD27E", + "size": 5904, + "defined_public_names": [ + "volatility_robust_sadf" + ], + "definitions": [ + { + "name": "volatility_robust_sadf", + "kind": "function", + "signature": "(log_price: np.ndarray | pd.Series, min_length: int | None=None, n_bootstrap: int=99, random_state: int | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/hpc/__init__.py", + "module": "RiskLabAI.hpc", + "module_kind": "package", + "sha256": "C08896EA7791033B286FAB38AFDBD8EC5A79F562316AA470DD071F90C6F51384", + "size": 511, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/hpc/hpc.py", + "module": "RiskLabAI.hpc.hpc", + "module_kind": "module", + "sha256": "CC6377F6E22F2BB25D87723DE0D6C36DEB555F4D08C19BDD4B55C64D5FABEED1", + "size": 11306, + "defined_public_names": [ + "expand_call", + "linear_partitions", + "mp_pandas_obj", + "nested_partitions", + "parallel_run", + "process_jobs", + "process_jobs_sequential", + "report_progress" + ], + "definitions": [ + { + "name": "parallel_run", + "kind": "function", + "signature": "(func: Callable[..., Any], iterable: Iterable[Any], num_cpus: int=-1, lin_partition: bool=False, **kwargs)" + }, + { + "name": "report_progress", + "kind": "function", + "signature": "(job_number: int, total_jobs: int, start_time: float, task: str)" + }, + { + "name": "expand_call", + "kind": "function", + "signature": "(kargs: dict[str, Any])" + }, + { + "name": "process_jobs", + "kind": "function", + "signature": "(jobs: list[dict[str, Any]], task: Optional[str]=None, num_threads: int=-1)" + }, + { + "name": "process_jobs_sequential", + "kind": "function", + "signature": "(jobs: list[dict[str, Any]])" + }, + { + "name": "linear_partitions", + "kind": "function", + "signature": "(num_atoms: int, num_threads: int)" + }, + { + "name": "nested_partitions", + "kind": "function", + "signature": "(num_atoms: int, num_threads: int, upper_triangle: bool=False)" + }, + { + "name": "mp_pandas_obj", + "kind": "function", + "signature": "(func: Callable[..., pd.Series], pandas_object: tuple[str, pd.Index], num_threads: int=-1, mp_batches: int=1, linear_partition: bool=True, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/__init__.py", + "module": "RiskLabAI.optimization", + "module_kind": "package", + "sha256": "8B178E855B01B7AAA4A6988060A26B794A9E4A867CF5E5F3125D1CBC0AE6900C", + "size": 1211, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/optimization/hedging.py", + "module": "RiskLabAI.optimization.hedging", + "module_kind": "module", + "sha256": "4A87276BD9FBAF8726F8646E57C6B27E6274C3481692532A04F67CC0B206E149", + "size": 1962, + "defined_public_names": [ + "pca_weights" + ], + "definitions": [ + { + "name": "pca_weights", + "kind": "function", + "signature": "(cov: np.ndarray, risk_distribution: Optional[np.ndarray]=None, risk_target: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/hrp.py", + "module": "RiskLabAI.optimization.hrp", + "module_kind": "module", + "sha256": "5DD1818F65CDCE5C07870D96353627600C325ADE1C4297AAB933AB5954538047", + "size": 6108, + "defined_public_names": [ + "cluster_variance", + "distance_corr", + "hrp", + "inverse_variance_weights", + "quasi_diagonal", + "recursive_bisection" + ], + "definitions": [ + { + "name": "inverse_variance_weights", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame)" + }, + { + "name": "cluster_variance", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame, clustered_items: list[str])" + }, + { + "name": "quasi_diagonal", + "kind": "function", + "signature": "(linkage_matrix: np.ndarray)" + }, + { + "name": "recursive_bisection", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame, sorted_items: list[str])" + }, + { + "name": "distance_corr", + "kind": "function", + "signature": "(corr_matrix: np.ndarray)" + }, + { + "name": "hrp", + "kind": "function", + "signature": "(cov: pd.DataFrame, corr: pd.DataFrame)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "module": "RiskLabAI.optimization.hyper_parameter_tuning", + "module_kind": "module", + "sha256": "DCAFAFB9A5541BA40FE48FC8DBDD57E804BBD9DC5C22430F7979B0080AD81D29", + "size": 6152, + "defined_public_names": [ + "MyPipeline", + "SampleWeightedPipeline", + "clf_hyper_fit" + ], + "definitions": [ + { + "name": "SampleWeightedPipeline", + "kind": "class", + "signature": null + }, + { + "name": "clf_hyper_fit", + "kind": "function", + "signature": "(feature_data: pd.DataFrame, label: pd.Series, times: pd.Series, pipe_clf: Pipeline, param_grid: dict[str, Any], validator_type: str='purgedkfold', validator_params: Optional[dict[str, Any]]=None, bagging: Optional[list[Union[int, float]]]=None, rnd_search_iter: int=0, n_jobs: int=-1, **fit_params)" + }, + { + "name": "MyPipeline", + "kind": "class", + "signature": "(self, steps, *, memory=None, verbose=False)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/nco.py", + "module": "RiskLabAI.optimization.nco", + "module_kind": "module", + "sha256": "4FB1EE548E62CEF2BDD71E289F8F3DAD403F92AEE95820DA2BF1DA9409CAAA53", + "size": 3992, + "defined_public_names": [ + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco" + ], + "definitions": [ + { + "name": "get_optimal_portfolio_weights", + "kind": "function", + "signature": "(covariance: np.ndarray, mu: Optional[np.ndarray]=None)" + }, + { + "name": "get_optimal_portfolio_weights_nco", + "kind": "function", + "signature": "(covariance: np.ndarray, mu: Optional[np.ndarray]=None, number_clusters: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/__init__.py", + "module": "RiskLabAI.pde", + "module_kind": "package", + "sha256": "C7B95E7E7DF3A7BDD666D940EE776A5D8349ADE4CEE33D8FC453C7C56189C4CA", + "size": 1833, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/pde/equation.py", + "module": "RiskLabAI.pde.equation", + "module_kind": "module", + "sha256": "4A9A073721FAAC4FFAC3513A61B08586027204248EE8A47F184E5F174DD86E06", + "size": 10999, + "defined_public_names": [ + "BlackScholesBarenblatt", + "Equation", + "HJBLQ", + "PricingDefaultRisk", + "PricingDiffRate" + ], + "definitions": [ + { + "name": "Equation", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "PricingDefaultRisk", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "HJBLQ", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "BlackScholesBarenblatt", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "PricingDiffRate", + "kind": "class", + "signature": "(self, eqn_config: dict)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/model.py", + "module": "RiskLabAI.pde.model", + "module_kind": "module", + "sha256": "98F8C874C57C04E28883BB9B4CFD526C0D1622E27C84F723E5883D30CBFF7001", + "size": 12264, + "defined_public_names": [ + "DeepBSDE", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "ISAB", + "MAB", + "Net1", + "PMA", + "SAB", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + "TimeNet", + "TimeNetForSet" + ], + "definitions": [ + { + "name": "MAB", + "kind": "class", + "signature": "(self, dim_q: int, dim_k: int, dim_v: int, num_heads: int, ln: bool=False)" + }, + { + "name": "SAB", + "kind": "class", + "signature": "(self, dim_in: int, dim_out: int, num_heads: int, ln: bool=False)" + }, + { + "name": "ISAB", + "kind": "class", + "signature": "(self, dim_in: int, dim_out: int, num_heads: int, num_inds: int, ln: bool=False)" + }, + { + "name": "PMA", + "kind": "class", + "signature": "(self, dim: int, num_heads: int, num_seeds: int, ln: bool=False)" + }, + { + "name": "TimeNetForSet", + "kind": "class", + "signature": "(self, in_features: int=1, out_features: int=64)" + }, + { + "name": "DeepTimeSetTransformer", + "kind": "class", + "signature": "(self, input_dim: int)" + }, + { + "name": "TimeNet", + "kind": "class", + "signature": "(self, output_dim: int)" + }, + { + "name": "Net1", + "kind": "class", + "signature": "(self, input_dim: int, output_dim: int)" + }, + { + "name": "FBSNNNetwork", + "kind": "class", + "signature": "(self, layer_sizes: list[int])" + }, + { + "name": "DeepBSDE", + "kind": "class", + "signature": "(self, layer_sizes: list[int])" + }, + { + "name": "TimeDependentNetwork", + "kind": "class", + "signature": "(self, indim: int, layersize: list[int], outdim: int)" + }, + { + "name": "TimeDependentNetworkMonteCarlo", + "kind": "class", + "signature": "(self, indim: int, layersize: list[int], outdim: int, sigma: float)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/solver.py", + "module": "RiskLabAI.pde.solver", + "module_kind": "module", + "sha256": "A7F2A65833339E5E9258F165B0AC0BA9F37130061F456C12465C81179B740756", + "size": 13859, + "defined_public_names": [ + "FBSDESolver", + "FBSNNSolver", + "initialize_weights" + ], + "definitions": [ + { + "name": "initialize_weights", + "kind": "function", + "signature": "(m: nn.Module)" + }, + { + "name": "FBSDESolver", + "kind": "class", + "signature": "(self, pde: Equation, layer_sizes: list[int], learning_rate: float, solving_method: str, device: torch.device)" + }, + { + "name": "FBSNNSolver", + "kind": "class", + "signature": "(self, pde: Equation, layer_sizes: list[int], learning_rate: float, device: torch.device)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/__init__.py", + "module": "RiskLabAI.utils", + "module_kind": "package", + "sha256": "DE1F4F4832425C9B8FE275BF48A9621ABA3525BB219F43C9D9E38DBBCEAE0582", + "size": 3409, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/utils/constants.py", + "module": "RiskLabAI.utils.constants", + "module_kind": "module", + "sha256": "751B87152BF9EABFE4AE5BB9CF35F38FBD56E4FE2691BF52B749CC3DAFAACE5A", + "size": 3003, + "defined_public_names": [ + "BUY_TICKS_NUMBER", + "CLOSE_PRICE", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_DOLLAR", + "CUMULATIVE_SELL_THETA", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_TICKS", + "CUMULATIVE_VOLUME", + "DATE_TIME", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "EXPECTED_IMBALANCE", + "EXPECTED_IMBALANCE_WINDOW", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "HIGH_PRICE", + "LOW_PRICE", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + "N_TICKS_ON_BAR_FORMATION", + "OPEN_PRICE", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_TICK_RULE", + "THRESHOLD", + "TICK_NUMBER", + "TIMESTAMP" + ], + "definitions": [] + }, + { + "path": "src/RiskLabAI/utils/ewma.py", + "module": "RiskLabAI.utils.ewma", + "module_kind": "module", + "sha256": "48306B7F06272DFBABFB0CFE2B147F232038884BFCE77FEE072583A8F8F6355B", + "size": 2118, + "defined_public_names": [ + "ewma" + ], + "definitions": [ + { + "name": "ewma", + "kind": "function", + "signature": "(array: np.ndarray, window: int)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "module": "RiskLabAI.utils.momentum_mean_reverting_strategy_sides", + "module_kind": "module", + "sha256": "F19E1691AD40DF94640AEFBFA06431B39538444139277568B8A6AE247FB48509", + "size": 2156, + "defined_public_names": [ + "determine_strategy_side" + ], + "definitions": [ + { + "name": "determine_strategy_side", + "kind": "function", + "signature": "(prices: pd.Series, fast_window: int=20, slow_window: int=50, exponential: bool=False, mean_reversion: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/progress.py", + "module": "RiskLabAI.utils.progress", + "module_kind": "module", + "sha256": "74B4D2EA57526348055B657EB254D92606C00C7A49DBC2B122D5DA34A44F75D2", + "size": 1945, + "defined_public_names": [ + "progress_bar" + ], + "definitions": [ + { + "name": "progress_bar", + "kind": "function", + "signature": "(current_progress: int, total_progress: int, start_time: float, bar_length: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/publication_plots.py", + "module": "RiskLabAI.utils.publication_plots", + "module_kind": "module", + "sha256": "B00FD12382E692BB3F74F2B37EFFAB1D95BC3641679F588BBB60FB0C203764D8", + "size": 5850, + "defined_public_names": [ + "THEMES", + "apply_plot_style", + "finalize_plot", + "setup_publication_style" + ], + "definitions": [ + { + "name": "setup_publication_style", + "kind": "function", + "signature": "(theme: str='light', quality: int=300, save_plots: bool=False, save_dir: str='figs')" + }, + { + "name": "apply_plot_style", + "kind": "function", + "signature": "(ax: plt.Axes, title: str, xlabel: str, ylabel: str, legend_title: Optional[str]=None)" + }, + { + "name": "finalize_plot", + "kind": "function", + "signature": "(fig: fig.Figure, filename: str)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/update_figure_layout.py", + "module": "RiskLabAI.utils.update_figure_layout", + "module_kind": "module", + "sha256": "8C5C6DAEFE392D1AD59AF598B1D8D76732FD4B0A7A437AB3C4AB160CB1CDFFB2", + "size": 1324, + "defined_public_names": [ + "update_figure_layout" + ], + "definitions": [ + { + "name": "update_figure_layout", + "kind": "function", + "signature": "(fig: go.Figure, title: str, xaxis_title: str, yaxis_title: str, legend_x: float=1.0, legend_y: float=1.0)" + } + ] + } + ], + "package_export_surfaces": [ + { + "module": "RiskLabAI", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils" + ] + }, + { + "module": "RiskLabAI.backtest", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "validation", + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction", + "conditional_expected_drawdown", + "sharpe_difference_test", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "fit_ornstein_uhlenbeck", + "bet_timing", + "calculate_holding_period", + "calculate_hhi_concentration", + "calculate_hhi", + "compute_drawdowns_time_under_water", + "synthetic_back_testing", + "probability_bet_size", + "average_bet_sizes", + "strategy_bet_sizing", + "avg_active_signals", + "mp_avg_active_signals", + "discrete_signal", + "generate_signal", + "bet_size_sigmoid", + "target_position", + "inverse_price", + "limit_price", + "compute_sigmoid_width", + "avgActiveSignals", + "mpAvgActiveSignals", + "discreteSignal", + "Signal", + "betSize", + "TPos", + "inversePrice", + "limitPrice", + "getW", + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic", + "implied_precision", + "bin_frequency", + "binomial_sharpe_ratio", + "mix_gaussians", + "failure_probability", + "calculate_strategy_risk", + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "estimated_sharpe_ratio_z_statistics", + "strategy_type1_error_probability", + "theta_for_type2_error", + "strategy_type2_error_probability", + "pbo_sharpe_ratio", + "performance_evaluation", + "probability_of_backtest_overfitting", + "probabilistic_sharpe_ratio", + "benchmark_sharpe_ratio", + "sharpe_ratio_p_values", + "holm_adjusted_p_values", + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + "sharpe_ratio_influence_function", + "newey_west_long_run_variance", + "newey_west_automatic_lag", + "lplz_sharpe_inference", + "overall_backtest_overfitting_simulation", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation", + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "noised_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "measure_all_cv_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "get_cpu_info", + "format_cpu_info" + ] + }, + { + "module": "RiskLabAI.backtest.validation", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "CrossValidator", + "KFold", + "PurgedKFold", + "WalkForward", + "CombinatorialPurged", + "BaggedCombinatorialPurged", + "AdaptiveCombinatorialPurged", + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices", + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes", + "leakage_aware_hpo", + "deflated_sharpe_gate", + "CrossValidatorFactory", + "CrossValidatorController" + ] + }, + { + "module": "RiskLabAI.causal_factor_analysis", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol" + ] + }, + { + "module": "RiskLabAI.cluster", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "covariance_to_correlation", + "cluster_k_means_base", + "make_new_outputs", + "cluster_k_means_top", + "random_covariance_sub", + "random_block_covariance", + "random_block_correlation" + ] + }, + { + "module": "RiskLabAI.controller", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [] + }, + { + "module": "RiskLabAI.core", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "Registry", + "REGISTRIES", + "get_registry", + "list_components", + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + "Estimator", + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer" + ] + }, + { + "module": "RiskLabAI.data", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "denoise", + "differentiation", + "distance", + "labeling", + "structures", + "synthetic_data", + "weights" + ] + }, + { + "module": "RiskLabAI.data.denoise", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "marcenko_pastur_pdf", + "fit_kde", + "find_max_eval", + "pca", + "denoised_corr", + "denoised_corr2", + "cov_to_corr", + "corr_to_cov", + "denoise_cov", + "optimal_portfolio", + "nercome_denoised_covariance" + ] + }, + { + "module": "RiskLabAI.data.differentiation", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "adaptive_differencing_order", + "adaptive_fractional_difference", + "rescaled_range_hurst", + "wavelet_variance_hurst" + ] + }, + { + "module": "RiskLabAI.data.distance", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "distance_correlation", + "ksg_mutual_information" + ] + }, + { + "module": "RiskLabAI.data.labeling", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "cusum_filter_events_dynamic_threshold", + "symmetric_cusum_filter", + "daily_volatility_with_log_returns", + "vertical_barrier", + "triple_barrier", + "meta_events", + "meta_labeling", + "lin_parts", + "process_jobs", + "expand_call", + "report_progress", + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning" + ] + }, + { + "module": "RiskLabAI.data.structures", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "StandardBars", + "TimeBars", + "ExpectedImbalanceBars", + "FixedImbalanceBars", + "ExpectedRunBars", + "FixedRunBars", + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars" + ] + }, + { + "module": "RiskLabAI.data.synthetic_data", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "drift_volatility_burst", + "compute_log_returns", + "heston_merton_log_returns", + "align_params_length", + "generate_prices_from_regimes", + "parallel_generate_prices", + "random_cov", + "form_block_matrix", + "form_true_matrix", + "simulates_cov_mu" + ] + }, + { + "module": "RiskLabAI.data.weights", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "expand_label_for_meta_labeling", + "calculate_average_uniqueness", + "sample_weight_absolute_return_meta_labeling", + "calculate_time_decay" + ] + }, + { + "module": "RiskLabAI.ensemble", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "bagging_classifier_accuracy", + "BaggingClassifierAccuracy", + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution" + ] + }, + { + "module": "RiskLabAI.features", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "entropy_features", + "feature_importance", + "microstructural_features", + "structural_breaks" + ] + }, + { + "module": "RiskLabAI.features.entropy_features", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "shannon_entropy", + "lempel_ziv_entropy", + "probability_mass_function", + "ngram_counts", + "plug_in_entropy_estimator", + "kontoyiannis_entropy", + "longest_match_length", + "miller_madow_entropy", + "grassberger_entropy", + "nsb_entropy" + ] + }, + { + "module": "RiskLabAI.features.feature_importance", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "FeatureImportanceStrategy", + "FeatureImportanceFactory", + "FeatureImportanceController", + "FeatureImportanceMDI", + "ClusteredFeatureImportanceMDI", + "FeatureImportanceMDA", + "ClusteredFeatureImportanceMDA", + "FeatureImportanceSFI", + "mdi_plus_importance", + "conditional_predictive_impact", + "cluster_k_means_top", + "random_block_correlation", + "form_true_matrix", + "simulates_cov_mu", + "get_test_dataset", + "orthogonal_features", + "calculate_weighted_tau" + ] + }, + { + "module": "RiskLabAI.features.microstructural_features", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "beta_estimates", + "gamma_estimates", + "alpha_estimates", + "corwin_schultz_estimator", + "sigma_estimates", + "bekker_parkinson_volatility_estimates", + "edge_estimator" + ] + }, + { + "module": "RiskLabAI.features.structural_breaks", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "lag_dataframe", + "prepare_data", + "compute_beta", + "get_expanding_window_adf", + "get_bsadf_statistic", + "psy_minimum_window", + "get_sadf_sequence", + "get_bsadf_sequence", + "get_gsadf_statistic", + "get_bubble_episodes", + "simulate_psy_critical_values", + "volatility_robust_sadf", + "pelt_change_points" + ] + }, + { + "module": "RiskLabAI.hpc", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "report_progress", + "expand_call", + "process_jobs", + "process_jobs_sequential", + "linear_partitions", + "nested_partitions", + "mp_pandas_obj", + "parallel_run" + ] + }, + { + "module": "RiskLabAI.optimization", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "cluster_variance", + "quasi_diagonal", + "recursive_bisection", + "hrp", + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco", + "pca_weights", + "SampleWeightedPipeline", + "MyPipeline", + "clf_hyper_fit" + ] + }, + { + "module": "RiskLabAI.pde", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "Equation", + "PricingDefaultRisk", + "HJBLQ", + "BlackScholesBarenblatt", + "PricingDiffRate", + "TimeNet", + "Net1", + "MAB", + "SAB", + "ISAB", + "PMA", + "TimeNetForSet", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "DeepBSDE", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + "initialize_weights", + "FBSDESolver", + "FBSNNSolver", + "FBSNNolver" + ] + }, + { + "module": "RiskLabAI.utils", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + "ewma", + "compute_exponential_weighted_moving_average", + "progress_bar", + "determine_strategy_side", + "update_figure_layout", + "setup_publication_style", + "apply_plot_style", + "finalize_plot" + ] + } + ], + "causal_factor_analysis": { + "export_count": 57, + "ordered_exports": [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol" + ], + "ordered_export_sha256": "6EF5A71729FA52EF02B2275A9E10868F3284ADDDA3C21585AA9D3074BDB751F1", + "entries": [ + { + "name": "BackdoorAdjustmentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.BackdoorAdjustmentEvidence", + "kind": "record_type", + "signature": "(treatment: ForwardRef('str'), outcome: ForwardRef('str'), adjustment_set: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), endpoints_in_set: ForwardRef('NodeTuple'), descendants_in_set: ForwardRef('NodeTuple'), open_backdoor_paths: ForwardRef('tuple[PathEvidence, ...]'), admissible: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "treatment", + "outcome", + "adjustment_set", + "all_observed", + "endpoints_in_set", + "descendants_in_set", + "open_backdoor_paths", + "admissible" + ] + }, + { + "name": "BacktestStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.BacktestStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', validation: 'Optional[ValidationEvidence]' = None, monte_carlo_dgp: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalDAG", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalDAG", + "kind": "record_type", + "signature": "(nodes, directed_edges, observed_nodes)", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "nodes", + "directed_edges", + "observed_nodes" + ] + }, + { + "name": "CausalAdjustmentSetStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalAdjustmentSetStage", + "kind": "class", + "signature": "(treatment: 'str', outcome: 'str', method_label: 'str', identified: 'bool', admissible_adjustment_sets: 'tuple[tuple[str, ...], ...]', selected_adjustment_set: 'tuple[str, ...]', confounders: 'tuple[str, ...]', descendants: 'tuple[str, ...]', mediators: 'tuple[str, ...]', colliders: 'tuple[str, ...]', instruments: 'tuple[str, ...]', control_justifications: 'tuple[tuple[str, str], ...]', open_backdoor_paths: 'tuple[str, ...]', frontdoor_criteria_satisfied: 'Optional[bool]' = None, instrument_relevance_satisfied: 'Optional[bool]' = None, instrument_exclusion_satisfied: 'Optional[bool]' = None, instrument_exogeneity_satisfied: 'Optional[bool]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "ColliderCoefficients", + "qualified_name": "RiskLabAI.causal_factor_analysis.ColliderCoefficients", + "kind": "class", + "signature": "(beta_hat: 'float', theta_hat: 'float') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "ColliderDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.ColliderDiagnostics", + "kind": "class", + "signature": "(residual_variance: 'float', outcome_variance: 'float', correct_r_squared: 'float', overcontrolled_r_squared: 'float', correct_adjusted_r_squared: 'float', overcontrolled_adjusted_r_squared: 'float', correct_beta_variance: 'float', overcontrolled_beta_variance: 'float', collider_coefficient_variance: 'float', correct_beta_t_statistic: 'float', overcontrolled_beta_t_statistic: 'float', collider_t_statistic: 'float', adjusted_r_squared_prefers_overcontrolled: 'bool', absolute_beta_t_prefers_overcontrolled: 'bool') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "CausalDiscoveryStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalDiscoveryStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', graph_id: 'str', graph_nodes: 'tuple[str, ...]', directed_edges: 'tuple[tuple[str, str], ...]', graph_kind: 'str', ambiguous_edges: 'tuple[tuple[str, str], ...]', assumptions: 'tuple[str, ...]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalExplanatoryAndPredictivePowerStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalExplanatoryAndPredictivePowerStage", + "kind": "class", + "signature": "(task_types: 'tuple[str, ...]', estimator_label: 'str', explanatory_metric_labels: 'tuple[str, ...]', predictive_metric_labels: 'tuple[str, ...]', explanatory_evidence_id: 'Optional[str]', predictive_evidence_id: 'Optional[str]', naive_benchmark_id: 'str', validation: 'ValidationEvidence', multiclass_encoding: 'Optional[str]' = None, averaging_method: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalFactorProtocolReport", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalFactorProtocolReport", + "kind": "class", + "signature": "(trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', variable_selection: 'VariableSelectionStage', causal_discovery: 'CausalDiscoveryStage', causal_adjustment_set: 'CausalAdjustmentSetStage', causal_explanatory_and_predictive_power: 'CausalExplanatoryAndPredictivePowerStage', causal_portfolio_construction: 'CausalPortfolioConstructionStage', backtest: 'BacktestStage', multiple_testing_adjustments: 'MultipleTestingAdjustmentsStage') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalPortfolioConstructionStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalPortfolioConstructionStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', causal_exposures: 'tuple[str, ...]', controlled_unintended_exposures: 'tuple[str, ...]', cost_model_id: 'str', constraint_set_id: 'str', economic_rationale: 'str', fragility_scenarios: 'tuple[str, ...]', transfer_coefficient: 'Real') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "DSeparationEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.DSeparationEvidence", + "kind": "record_type", + "signature": "(left: ForwardRef('NodeTuple'), right: ForwardRef('NodeTuple'), conditioned: ForwardRef('NodeTuple'), separated: ForwardRef('bool'), paths: ForwardRef('tuple[PathEvidence, ...]'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "left", + "right", + "conditioned", + "separated", + "paths" + ] + }, + { + "name": "DifferenceInDifferencesEstimate", + "qualified_name": "RiskLabAI.causal_factor_analysis.DifferenceInDifferencesEstimate", + "kind": "record_type", + "signature": "(treated_change: ForwardRef('float'), control_change: ForwardRef('float'), estimate: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "record_fields": [ + "treated_change", + "control_change", + "estimate" + ] + }, + { + "name": "EventHorizon", + "qualified_name": "RiskLabAI.causal_factor_analysis.EventHorizon", + "kind": "class", + "signature": "(observation_id: 'str', start: 'Real', end: 'Real') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "FoldEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.FoldEvidence", + "kind": "class", + "signature": "(fold_id: 'str', path_id: 'str', train_ids: 'tuple[str, ...]', test_ids: 'tuple[str, ...]', fit_ids: 'tuple[str, ...]', prediction_ids: 'tuple[str, ...]', refit_id: 'str', refit_components: 'tuple[str, ...]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "FrontdoorAdjustmentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.FrontdoorAdjustmentEvidence", + "kind": "record_type", + "signature": "(treatment: ForwardRef('str'), outcome: ForwardRef('str'), mediator_set: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), endpoints_in_set: ForwardRef('NodeTuple'), directed_path_exists: ForwardRef('bool'), unintercepted_directed_paths: ForwardRef('tuple[PathEvidence, ...]'), open_treatment_mediator_backdoor_paths: ForwardRef('tuple[PathEvidence, ...]'), open_mediator_outcome_backdoor_paths_given_treatment: ForwardRef('tuple[PathEvidence, ...]'), graphically_admissible: ForwardRef('bool'), positivity_required: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "treatment", + "outcome", + "mediator_set", + "all_observed", + "endpoints_in_set", + "directed_path_exists", + "unintercepted_directed_paths", + "open_treatment_mediator_backdoor_paths", + "open_mediator_outcome_backdoor_paths_given_treatment", + "graphically_admissible", + "positivity_required" + ] + }, + { + "name": "InstrumentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.InstrumentEvidence", + "kind": "record_type", + "signature": "(instrument: ForwardRef('str'), treatment: ForwardRef('str'), outcome: ForwardRef('str'), conditioned: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), conditioning_disjoint: ForwardRef('bool'), conditioning_unaffected_by_treatment: ForwardRef('bool'), pearl_relevance: ForwardRef('bool'), pearl_exclusion_exogeneity: ForwardRef('bool'), pearl_graphical: ForwardRef('bool'), cfi_direct_relevance: ForwardRef('bool'), cfi_full_mediation: ForwardRef('bool'), cfi_exogeneity: ForwardRef('bool'), cfi_simple: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "instrument", + "treatment", + "outcome", + "conditioned", + "all_observed", + "conditioning_disjoint", + "conditioning_unaffected_by_treatment", + "pearl_relevance", + "pearl_exclusion_exogeneity", + "pearl_graphical", + "cfi_direct_relevance", + "cfi_full_mediation", + "cfi_exogeneity", + "cfi_simple" + ] + }, + { + "name": "MultipleTestingAdjustmentsStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.MultipleTestingAdjustmentsStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', family_partitions: 'tuple[tuple[str, tuple[str, ...]], ...]', alpha: 'Real', backtests_independent: 'bool', p_value_estimator: 'Optional[str]' = None, time_dependence_model: 'Optional[str]' = None, p_value_inputs_assume_independence: 'Optional[bool]' = None, sharpe_variance: 'Optional[Real]' = None, effective_trials: 'Optional[Real]' = None, sample_length: 'Optional[Integral]' = None, skewness: 'Optional[Real]' = None, kurtosis: 'Optional[Real]' = None, selection_bias_evidence_id: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "NodeRoleEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.NodeRoleEvidence", + "kind": "record_type", + "signature": "(node: ForwardRef('str'), treatment: ForwardRef('str'), outcome: ForwardRef('str'), collider_on_paths: ForwardRef('tuple[PathTuple, ...]'), noncollider_on_paths: ForwardRef('tuple[PathTuple, ...]'), mediator_on_directed_paths: ForwardRef('tuple[PathTuple, ...]'), backdoor_noncollider_on_paths: ForwardRef('tuple[PathTuple, ...]'), common_cause_paths: ForwardRef('tuple[PathTuple, ...]'), descendant_of_treatment: ForwardRef('bool'), ancestor_of_outcome: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "node", + "treatment", + "outcome", + "collider_on_paths", + "noncollider_on_paths", + "mediator_on_directed_paths", + "backdoor_noncollider_on_paths", + "common_cause_paths", + "descendant_of_treatment", + "ancestor_of_outcome" + ] + }, + { + "name": "PathEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.PathEvidence", + "kind": "record_type", + "signature": "(nodes: ForwardRef('PathTuple'), colliders: ForwardRef('NodeTuple'), noncolliders: ForwardRef('NodeTuple'), conditioned_noncolliders: ForwardRef('NodeTuple'), activated_colliders: ForwardRef('NodeTuple'), blocking_colliders: ForwardRef('NodeTuple'), is_open: ForwardRef('bool'), is_directed_from_left: ForwardRef('bool'), is_backdoor_from_left: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "nodes", + "colliders", + "noncolliders", + "conditioned_noncolliders", + "activated_colliders", + "blocking_colliders", + "is_open", + "is_directed_from_left", + "is_backdoor_from_left" + ] + }, + { + "name": "PopulationRegressionDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.PopulationRegressionDiagnostics", + "kind": "record_type", + "signature": "(coefficient_names: ForwardRef('Tuple[str, ...]'), coefficients: ForwardRef('Tuple[float, ...]'), r_squared: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "coefficient_names", + "coefficients", + "r_squared" + ] + }, + { + "name": "RegressionDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.RegressionDiagnostics", + "kind": "record_type", + "signature": "(coefficient_names: ForwardRef('Tuple[str, ...]'), coefficients: ForwardRef('Tuple[float, ...]'), standard_errors: ForwardRef('Tuple[float, ...]'), t_statistics: ForwardRef('Tuple[float, ...]'), r_squared: ForwardRef('float'), adjusted_r_squared: ForwardRef('float'), residual_variance: ForwardRef('float'), n_observations: ForwardRef('int'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "coefficient_names", + "coefficients", + "standard_errors", + "t_statistics", + "r_squared", + "adjusted_r_squared", + "residual_variance", + "n_observations" + ] + }, + { + "name": "SpecificationExperimentResult", + "qualified_name": "RiskLabAI.causal_factor_analysis.SpecificationExperimentResult", + "kind": "record_type", + "signature": "(structure: ForwardRef('str'), conditioning_variable_role: ForwardRef('str'), unconditioned: ForwardRef('RegressionDiagnostics'), conditioned: ForwardRef('RegressionDiagnostics'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "structure", + "conditioning_variable_role", + "unconditioned", + "conditioned" + ] + }, + { + "name": "SpecificationPopulationResult", + "qualified_name": "RiskLabAI.causal_factor_analysis.SpecificationPopulationResult", + "kind": "record_type", + "signature": "(structure: ForwardRef('str'), conditioning_variable_role: ForwardRef('str'), unconditioned: ForwardRef('PopulationRegressionDiagnostics'), conditioned: ForwardRef('PopulationRegressionDiagnostics'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "structure", + "conditioning_variable_role", + "unconditioned", + "conditioned" + ] + }, + { + "name": "StrategyPerformance", + "qualified_name": "RiskLabAI.causal_factor_analysis.StrategyPerformance", + "kind": "class", + "signature": "(correct: 'float', misspecified: 'float') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "TreatmentEffectDecomposition", + "qualified_name": "RiskLabAI.causal_factor_analysis.TreatmentEffectDecomposition", + "kind": "record_type", + "signature": "(observed_difference: ForwardRef('float'), average_treatment_effect_on_treated: ForwardRef('float'), sample_selection_bias: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "record_fields": [ + "observed_difference", + "average_treatment_effect_on_treated", + "sample_selection_bias" + ] + }, + { + "name": "ValidationEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.ValidationEvidence", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', event_horizons: 'tuple[EventHorizon, ...]', folds: 'tuple[FoldEvidence, ...]', embargo: 'Real' = 0.0) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "VariableSelectionStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.VariableSelectionStage", + "kind": "class", + "signature": "(purpose: 'str', selected_variables: 'tuple[str, ...]', method_labels: 'tuple[str, ...]', overlapping_returns: 'bool', strong_time_dependence: 'bool', validation: 'Optional[ValidationEvidence]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "average_treatment_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.average_treatment_effect", + "kind": "function", + "signature": "(treated_interventional_mean: 'Real', control_interventional_mean: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "backdoor_adjusted_average_treatment_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.backdoor_adjusted_average_treatment_effect", + "kind": "function", + "signature": "(treated_conditional_means: 'Sequence[Real]', control_conditional_means: 'Sequence[Real]', adjustment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "backdoor_adjusted_expectation", + "qualified_name": "RiskLabAI.causal_factor_analysis.backdoor_adjusted_expectation", + "kind": "function", + "signature": "(conditional_outcome_means: 'Sequence[Real]', adjustment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "causal_role_evidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.causal_role_evidence", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', node: 'str', *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'NodeRoleEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_backdoor_adjustment_set", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_backdoor_adjustment_set", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', adjustment_set: 'Iterable[str]' = (), *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'BackdoorAdjustmentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_frontdoor_adjustment_set", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_frontdoor_adjustment_set", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', mediator_set: 'Iterable[str]', *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'FrontdoorAdjustmentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_instrument", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_instrument", + "kind": "function", + "signature": "(dag: 'CausalDAG', instrument: 'str', treatment: 'str', outcome: 'str', conditioned: 'Iterable[str]' = ()) -> 'InstrumentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "collider_factor_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_factor_return", + "kind": "function", + "signature": "(x: 'Real', collider_proxy: 'Real', beta: 'Real', gamma: 'Real', delta: 'Real') -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_forecast_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_forecast_return", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_model_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_model_diagnostics", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real', n_observations: 'Integral') -> 'ColliderDiagnostics'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_overcontrolled_coefficients", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_overcontrolled_coefficients", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'ColliderCoefficients'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "collider_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounded_mediator_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounded_mediator_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounded_mediator_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounded_mediator_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounder_factor_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_factor_return", + "kind": "function", + "signature": "(x: 'Real', z: 'Real', beta: 'Real', gamma: 'Real', delta_estimated: 'Real', *, delta_realized: 'Optional[Real]' = None) -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "confounder_forecast_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_forecast_return", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta_estimated: 'Real', *, delta_realized: 'Optional[Real]' = None) -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "confounder_undercontrolled_coefficient", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_undercontrolled_coefficient", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "d_separation", + "qualified_name": "RiskLabAI.causal_factor_analysis.d_separation", + "kind": "function", + "signature": "(dag: 'CausalDAG', left: 'Iterable[str]', right: 'Iterable[str]', conditioned: 'Iterable[str]' = (), *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'DSeparationEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "difference_in_differences", + "qualified_name": "RiskLabAI.causal_factor_analysis.difference_in_differences", + "kind": "function", + "signature": "(treated_before: 'Real', treated_after: 'Real', control_before: 'Real', control_after: 'Real') -> 'DifferenceInDifferencesEstimate'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "fork_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.fork_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "fork_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.fork_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "frontdoor_adjusted_probability", + "qualified_name": "RiskLabAI.causal_factor_analysis.frontdoor_adjusted_probability", + "kind": "function", + "signature": "(mediator_probabilities_given_treatment: 'Sequence[Real]', outcome_probabilities_given_mediator_and_treatment: 'Sequence[Sequence[Real]]', treatment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "linear_instrumental_variable_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.linear_instrumental_variable_effect", + "kind": "function", + "signature": "(outcome_instrument_covariance: 'Real', treatment_instrument_covariance: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "minimal_backdoor_adjustment_sets", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimal_backdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', *, max_candidates: 'Optional[int]' = 65536) -> 'tuple[NodeTuple, ...]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "minimal_frontdoor_adjustment_sets", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimal_frontdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', *, max_candidates: 'Optional[int]' = 65536) -> 'tuple[NodeTuple, ...]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "minimum_variance_factor_weights", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimum_variance_factor_weights", + "kind": "function", + "signature": "(covariance: 'ArrayLike', factor_exposures: 'ArrayLike', target_exposures: 'ArrayLike') -> 'NDArray[np.float64]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.optimizer" + }, + { + "name": "randomized_mean_difference", + "qualified_name": "RiskLabAI.causal_factor_analysis.randomized_mean_difference", + "kind": "function", + "signature": "(treated_outcomes: 'Sequence[Real]', control_outcomes: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "treatment_effect_decomposition", + "qualified_name": "RiskLabAI.causal_factor_analysis.treatment_effect_decomposition", + "kind": "function", + "signature": "(observed_treated_mean: 'Real', observed_control_mean: 'Real', counterfactual_control_mean_for_treated: 'Real') -> 'TreatmentEffectDecomposition'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "validate_causal_factor_protocol", + "qualified_name": "RiskLabAI.causal_factor_analysis.validate_causal_factor_protocol", + "kind": "function", + "signature": "(report: 'CausalFactorProtocolReport') -> 'CausalFactorProtocolReport'", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + } + ] + } +} diff --git a/README.md b/README.md index cbaa6a2..4d74882 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,90 @@ -# RiskLabAI.py - -[![PyPI version](https://badge.fury.io/py/RiskLabAI.svg)](https://badge.fury.io/py/RiskLabAI) -[![CI](https://github.com/RiskLabAI/RiskLabAI.py/actions/workflows/ci.yml/badge.svg)](https://github.com/RiskLabAI/RiskLabAI.py/actions/workflows/ci.yml) - -A Python library for quantitative finance and financial machine learning, -implementing core methods from Marcos López de Prado's *Advances in Financial -Machine Learning* and *Machine Learning for Asset Managers*. - -The library provides implementations for: - -- **Financial data structures** — tick, volume, dollar, imbalance, and run bars -- **Labeling** — the triple-barrier method, meta-labeling, trend-scanning -- **Fractional differentiation** — standard and fixed-width window (FFD) -- **Sample weights**, **denoising** (Marčenko–Pastur), **distance metrics** -- **Cross-validation** — Purged K-Fold, Combinatorial Purged CV (+ adaptive/bagged), walk-forward -- **Feature importance** — MDI, MDA, SFI, and clustered variants -- **Portfolio optimization** — HRP, NCO, hedging -- **Backtest statistics** — PSR/DSR, PBO, strategy risk -- **Microstructure & entropy features**, **structural breaks**, and a Deep-BSDE PDE solver - -There is a companion Julia package, -[RiskLabAI.jl](https://github.com/RiskLabAI/RiskLabAI.jl), which mirrors this -API. - -## Installation - -```bash -pip install RiskLabAI -``` - -The base install is lightweight. Heavier, optional capabilities are available as -extras: - -| Extra | Installs | Enables | -|---|---|---| -| `RiskLabAI[pde]` | `torch` | the Deep-BSDE PDE solver (`RiskLabAI.pde`) | -| `RiskLabAI[plot]` | `matplotlib`, `seaborn`, `plotly` | plotting helpers | -| `RiskLabAI[synth]` | `quantecon` | synthetic-data utilities | -| `RiskLabAI[all]` | all of the above | everything | - -```bash -pip install "RiskLabAI[all]" -``` - -For development (editable install + tests), see -[`INSTALLATION.md`](INSTALLATION.md). - -## Quickstart - -Sample dollar/volume/tick bars from raw ticks: - -```python -from RiskLabAI.data.structures.standard_bars import StandardBars -from RiskLabAI.utils.constants import CUMULATIVE_DOLLAR - -# ticks: an iterable of (datetime, price, volume) -ticks = [ - ("2020-01-01 10:00:00", 100.0, 10), - ("2020-01-01 10:00:01", 101.0, 5), - ("2020-01-01 10:00:02", 100.0, 20), -] - -bars = StandardBars(bar_type=CUMULATIVE_DOLLAR, threshold=3000) -bar_list = bars.construct_bars_from_data(ticks) -# each bar: [date_time, idx, open, high, low, close, volume, -# buy_volume, sell_volume, ticks, dollar, threshold] -``` - -Discover and construct components by name through the extension registry: - -```python -import RiskLabAI.core as core - -core.list_components() # {family: [available keys]} -cv = core.CROSS_VALIDATORS.create("purgedkfold", n_splits=5, times=event_times) -``` - -## Logging - -RiskLabAI logs under the `"RiskLabAI"` logger and is silent by default. To see -progress and diagnostics, configure logging in your application: - -```python -import logging -logging.basicConfig(level=logging.INFO) -``` - -## Extending the library - -RiskLabAI is built to be extended with new models. The `RiskLabAI.core` layer -provides a component registry and base interfaces so a new bar type, labeler, -cross-validator, etc. can be registered and discovered without editing central -code. See [`EXTENDING.md`](EXTENDING.md) for a step-by-step guide with worked -examples. - -## Contributing - -Contributions are welcome. The project uses `pytest` for tests and -`black` + `ruff` for formatting/linting (run before opening a PR): - -```bash -pip install -e ".[all]" pytest black ruff -pytest -q --ignore=test/pde -black RiskLabAI test -ruff check RiskLabAI test -``` - -Please branch from `main`, keep changes focused, and update `CHANGELOG.md`. - -## License - -See [`LICENSE.txt`](LICENSE.txt). +# RiskLabAI 3.0.0 pre-package source + +Status: **local-only and blocked**. + +This directory is a temporary clean release nucleus for the next major +RiskLabAI release across Python and Julia. It is not a separate product. The +Python distribution and import namespace remain `RiskLabAI`; the Julia package +remains `RiskLabAI` with UUID `a72881da-fdaa-49c1-8962-99caf4ccfee8`. + +The distributable metadata contract is complete, but no wheel or source +distribution exists for this corrected source state, and nothing has been +published, registered, or released. The approved target version is `3.0.0`. +The source remains blocked until a separately authorized temporary artifact is +inspected and the owner separately authorizes version-control and release +actions. + +## Final Python source scope + +The `src` tree preserves all 117 runtime-source paths captured from the +current public Python `main` baseline and adds seven reviewed +`causal_factor_analysis` source files. The prior modules remain organized under +`backtest`, `cluster`, `controller`, `core`, `data`, `ensemble`, `features`, +`hpc`, `optimization`, `pde`, and `utils`. Their captured bytes are retained as +a continuity baseline; this does not clear later local changes or make the full +surface release-ready. + +The completed book-derived causal-factor namespace exposes 57 public concepts: + +- minimum-variance target-exposure allocation; +- published confounder and collider analytical diagnostics; +- immutable graphical-identification evidence and checks; and +- immutable evidence records for the seven-stage causal-factor protocol; +- randomized, stratified, difference-in-differences, and instrumental-variable + treatment-effect identities; and +- exact fork, collider, and confounded-mediator population diagnostics with + deterministic specification experiments. + +The complete preserved source library has passed its approved compatibility +matrix on CPython 3.12.13, 3.13.9, and 3.14.3. The base policy is NumPy +`>=2.2,<3`, tested at the lowest genuinely compatible and current NumPy 2.x +release for each interpreter. Feature-specific dependencies remain optional; +Numba does not constrain the non-Numba base, and the unavailable Python 3.14 +changepoint backend disables only that feature. + +## Scope and parity hold + +No path from the current public repository baseline has been deleted. The +completed 57-concept causal namespace is integrated here, and the Julia +candidate independently verifies the same 57 concepts. The full-library +runtime and dependency matrix, exact public API inventory, 78-file +package-scoped test inventory, distributable metadata, documentation, and +intended package file list are complete. The exact Python `3.0.0` and Julia +`1.0.0` targets are assigned. Artifact inspection and human-controlled +version-control, publication, and release decisions remain blocked. + +The graph routines report implications of a caller-supplied DAG; they do not +discover or certify that graph, prove positivity or instrument strength, or +estimate an effect. The protocol validator checks structural evidence records; +it does not perform the empirical stages or prove that caller declarations are +true. The analytical functions implement only the source-consistent public +results recorded by the publication audit; source-conflicted results remain +excluded. + +## Confirmed identity and stewardship + +- Product and package identity: `RiskLabAI` +- Intended license: BSD-3-Clause +- Rights holder and public maintainer: Hamid Arian +- Copyright: 2022-2026 +- Contact: arian@risklab.ai +- Python repository: https://github.com/RiskLabAI/RiskLabAI.py +- Python issues: https://github.com/RiskLabAI/RiskLabAI.py/issues +- Julia repository: https://github.com/RiskLabAI/RiskLabAI.jl +- Documentation: https://github.com/RiskLabAI/RiskLabAI.py#readme + +The rights holder has confirmed permission to publish and license every file +admitted to the clean public package. This confirmation does not remove the +artifact-inspection or human-authorization blocks. + +See `PACKAGE_IDENTITY.json` for the fail-closed machine-readable state. + +The complete Python contract is recorded in `pyproject.toml`, +`PUBLIC_API.json`, `TEST_INVENTORY.json`, and `PACKAGE_FILES.json`. The future +artifact checks are specified in `PACKAGE_INSPECTION.md`; that document is an +inspection plan, not build or release authorization. + +The 57-concept contract is documented in `docs/causal_factor_analysis.md`, and +runtime details are in `docs/compatibility.md`. A small deterministic example +is in `examples/causal_factor_analysis_quickstart.py`, and the blocked +next-major summary is in `RELEASE_NOTES_NEXT_MAJOR.md`. diff --git a/RELEASE_NOTES_NEXT_MAJOR.md b/RELEASE_NOTES_NEXT_MAJOR.md new file mode 100644 index 0000000..cfcdc1d --- /dev/null +++ b/RELEASE_NOTES_NEXT_MAJOR.md @@ -0,0 +1,44 @@ +# RiskLabAI Python next-major release notes — blocked draft + +Target version: **3.0.0**. The owner approved this exact version, but assigning +it does not authorize a build, artifact, publication, upload, or release. + +The next major Python release preserves the prior public repository source +modules and adds the verified 57-concept `RiskLabAI.causal_factor_analysis` +namespace. It intentionally drops support for Python 3.9-3.11 and NumPy 1.x; +the supported policy is CPython 3.12-3.14 with NumPy `>=2.2,<3`. + +Compatibility repairs include NumPy 2 trapezoidal integration, public +scikit-learn bootstrap logic, optional Numba acceleration, lazy optional +dependencies, a Python-3.14-specific Joblib floor, and self-contained standard +technical indicators. Python 3.14 remains supported when the optional +changepoint backend is unavailable. + +The preserved targeted-shrinkage API `RiskLabAI.data.denoise.denoised_corr2` +is restored after a refactor left its public export without the corresponding +implementation. Its historical parameter contract and published analytical +formula are covered by exact matrix, boundary, validation, and export-binding +tests. + +Compatibility aliases that remain present in 3.0.0 now consistently warn of +removal in 4.0.0. No alias is silently deleted, and the warning contract is +locked by package-scoped tests. + +The release metadata uses the PEP 639 `BSD-3-Clause` expression and license +file without the superseded license classifier. The governed CI workflow now +installs the distribution, rejects imports shadowed by the preserved repository +root in both parent and child interpreters, tests the six approved base +runtime/NumPy combinations, exercises feature-specific optional lanes, and +isolates the Python 3.14 changepoint limitation. Pinned static checks cover the +clean causal source and tests. + +No source-conflicted causal result is admitted. The metadata contract is +complete; artifacts, publication, upload, and release remain blocked. + +The static distributable metadata, exact 124-module runtime inventory, +78-file package-scoped test inventory, public API inventory, documentation, +examples, dependency groups, governed CI workflow, and 215-file intended +source-distribution list are now frozen. No retained package artifact has been +created for this corrected source state. The remaining gates are future +artifact inspection and separate human authorization for version-control, +publication, and release actions. diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json new file mode 100644 index 0000000..5f68554 --- /dev/null +++ b/TEST_INVENTORY.json @@ -0,0 +1,579 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_TEST_INVENTORY", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "runner": "pytest / six base lanes / three optional-feature lanes", + "file_count": 78, + "file_groups": { + "CAUSAL_FACTOR_ANALYSIS": 7, + "PRESERVED_LIBRARY": 70, + "TECHNICAL_INDICATOR_ORACLES": 1 + }, + "collected_cases_in_reference_full_feature_environment": { + "preserved_library": 354, + "causal_factor_analysis": 342, + "technical_indicator_oracles": 8, + "total": 704 + }, + "verified_required_matrix_cases_per_lane": { + "preserved_library_applicable": 342, + "causal_factor_analysis": 342, + "technical_indicator_oracles": 8, + "total_applicable": 692 + }, + "required_lane_count": 6, + "required_lane_case_total": 4152, + "base_lane_count": 6, + "optional_feature_lane_count": 3, + "ci_execution_count": 9, + "files": [ + { + "path": "test/backtest/test_advanced_bet_sizing.py", + "size": 3755, + "sha256": "F6AF65AA18989A2C3BBB4B43698719227F2199B825366553A7D177D1642572C2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 8 + }, + { + "path": "test/backtest/test_backtest_overfitting_simulation.py", + "size": 2489, + "sha256": "195EE1482CC4C4F1EC9346D1C8C0A600EBB14898D331BC7E37F0FB29D3E1A68A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/test_backtest_statistics.py", + "size": 4198, + "sha256": "16185AE1BEC34431ECD85EE1BE4E1A49C622C9CDB8B59CA4427148B1B6C2BA0A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/test_backtest_synthetic_data.py", + "size": 2147, + "sha256": "4E81268462F0321FDE82215768BC6ACAD07D5F70F881C5D74CCE0991A1BD6435", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/backtest/test_bet_sizing.py", + "size": 5392, + "sha256": "7CA2E228BD7421C87F15BFABFD6C79E529D239F34A685ACBF5AEE1620EE1898C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_multiple_testing.py", + "size": 3034, + "sha256": "AA6E928AD7A211C4C3D38525CC6FAFF1DF4D5F513C8EA97927D3DEFBF1672802", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 7 + }, + { + "path": "test/backtest/test_ou_trading_rules.py", + "size": 4403, + "sha256": "12498715145DA0BF5CEC21CFBCFF8EA206B7F82C01CEC15F05292C94F47A1C15", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_probabilistic_sharpe_ratio.py", + "size": 3242, + "sha256": "A62D3309AE74178AA80065FF9CFF4A96CE33A701AEB2D6F54127074875F91CC7", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_probability_of_backtest_overfitting.py", + "size": 3715, + "sha256": "178F73E2D1E3175C51EDCC0D990DFA49FB21C2843E4CF15D3726E370A42DBCBA", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/test_robust_statistics.py", + "size": 4672, + "sha256": "4954FF5BB03F7E6B040CAE874D10E04EE937DB1EEA85E7F0719FCD01A3DBA0FC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_sharpe_inference.py", + "size": 3456, + "sha256": "814021459A2904B6B5C5C9ADC90EB9E503D5437C1764CFDA431291132ED692E2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_strategy_risk.py", + "size": 3708, + "sha256": "E9696D0CDA4EAAFF01F536AD31E611658B36236BCADB5F470F0949EE4CE33820", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/test_technical_indicators.py", + "size": 3947, + "sha256": "63D92133A4731E007902E5FD73344F4B2FD6D3238E3124E2B3BD2C44B967B38C", + "group": "TECHNICAL_INDICATOR_ORACLES", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_test_set_overfitting.py", + "size": 3145, + "sha256": "C9E6DF7D60DA22FFC20FD9C4386B25A83DE9C2F24BAA7C2EC6985FF0B91949FC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "size": 3326, + "sha256": "E63E3168B1C24A68A7A9B97BD6AE86D653781CABBB75B612973D6E526F74A576", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/validation/test_bagged_combinatorial_purged.py", + "size": 2500, + "sha256": "36CD1050290F37C76B94776F7EF62D883205350373B2C251963EB3D8F424A8D5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/validation/test_combinatorial_purged.py", + "size": 4996, + "sha256": "2A7176CB64BC3117914123472650C2E28675E127AF721507FEC708A977AB014E", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_cross_validator_controller.py", + "size": 1836, + "sha256": "B27F524D6BFF839D9D11A2AF3C6F69C4E8725149DA3FF4BE7D47EE4ACFF78664", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_cross_validator_factory.py", + "size": 3073, + "sha256": "059213105205BA8A4B1EADD6923C513980EB38E7D48AF2ED9261841819E7AD04", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 9 + }, + { + "path": "test/backtest/validation/test_kfold.py", + "size": 4242, + "sha256": "42FFA8774B3398D0B2D4F6ABCEEE8934D9A600AA7A2C859E56F958F3291B2409", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/validation/test_leakage_aware_hpo.py", + "size": 3941, + "sha256": "C36A724AA150622F41C083A6C4370AF51E791D5FB27A1109F2F3A7F52B4F1CB2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_path_adaptive_cpcv.py", + "size": 8956, + "sha256": "5DDCF270F911FD61C06898BFBBF32E524711F035DBE27044E3137EED3BBC57DF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 10 + }, + { + "path": "test/backtest/validation/test_path_bagged_cpcv.py", + "size": 3965, + "sha256": "A040B21C906867C5CA0B196755CDC8C3F4F79032E9E6E5A045C779AEB43197B1", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_purged_kfold.py", + "size": 5007, + "sha256": "C64CB49639B8AB7E3878AE0A541F42CE3D3853405E8763B9331F6A2A0EA3153D", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_walk_forward.py", + "size": 4290, + "sha256": "DB6846CCB5C99C47B07F2C54C405C28F2E903E4D5EAB68405DAD4F7B37C27564", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "size": 3644, + "sha256": "195A3291A6CB6EAEE929F02951BAA45BD6E933AB751CCC30469402BBDA12DE9A", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 7 + }, + { + "path": "test/causal_factor_analysis/test_factor_mirage.py", + "size": 16311, + "sha256": "B64153FE7B39D103022268E503A132F76E967A05DB103D3339A662A9DA0A890C", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 32 + }, + { + "path": "test/causal_factor_analysis/test_graph_identification.py", + "size": 35155, + "sha256": "16CA24092CF9D05D2A7718FF1BACBF050592EFDDE18C7B7CCF2B20121250F789", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 47 + }, + { + "path": "test/causal_factor_analysis/test_minimum_variance.py", + "size": 17057, + "sha256": "9B2D772C018F99D358168AF9462DC48E0B7C53C924370A8B564AC1C4C66CE1FB", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 33 + }, + { + "path": "test/causal_factor_analysis/test_protocol.py", + "size": 38965, + "sha256": "E1E456BE476A5D990F781E240868B6C344FA0345FC78DC3C027BE63271BCA240", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 43 + }, + { + "path": "test/causal_factor_analysis/test_specification_diagnostics.py", + "size": 9088, + "sha256": "6FD3A8C9F1DFDC665AE9DCD80E41798D5D1EAE48BB9B2F4EE5AAB61B1B3393D4", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 10 + }, + { + "path": "test/causal_factor_analysis/test_treatment_effects.py", + "size": 5955, + "sha256": "434750E2D2889FA64E49BAE08372D0E08C9DC145BE88AB5CD58B942B4D1F5B6B", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 12 + }, + { + "path": "test/cluster/test_clustering.py", + "size": 3269, + "sha256": "33F301B160E90B4FA6BA369524E6A3D30F3BB63887B4898D6DC6708CCD900A7A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/controller/test_bars_initializer.py", + "size": 2484, + "sha256": "13B2E36A9D10E25284C3CDCE19F04320524623790BE64F79C851D31703D3724C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/controller/test_data_structure_controller.py", + "size": 2833, + "sha256": "9510A0FBD3B714FE87742519A9200CBFD0D6CEB99C14A9ED369F51EA7E3AAA56", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/core/test_base.py", + "size": 2701, + "sha256": "0BFC387B4C7A94205708140A7485F1AF1D3BFA1FC9472A1898ACBCEABA2DFE64", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/core/test_builtin_parity.py", + "size": 5095, + "sha256": "560CB3D389459FEA8572E7C71B5F66563A1EF17C4545B782A989CD8A67FCAD64", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 8 + }, + { + "path": "test/core/test_registry.py", + "size": 5789, + "sha256": "C6F667A94DB9803219591A3E4E8F963E820E5508C996A9850386BD4E8A8FED03", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 18 + }, + { + "path": "test/data/denoise/test_denoising.py", + "size": 6035, + "sha256": "4CAE78A888AEEE68C65C7A4625208E5EFC0CAD101D7DE05C35D2C0B4F0272FFE", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 11 + }, + { + "path": "test/data/denoise/test_nercome.py", + "size": 4263, + "sha256": "9B3799A2EB458510D869EE2624B525C2746D923E09BB11C5FB4EBDFA6385FA8C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/differentiation/test_adaptive_differentiation.py", + "size": 3832, + "sha256": "63AF31E69F26F186EA26B531189FEEDFEC653055DC6FA954E714F92F0A419CA5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/differentiation/test_differentiation.py", + "size": 3712, + "sha256": "C95ECCBBC869053624EB81A990F91003C835B4CCB1ABFEA56B01ECA347C54C33", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/data/distance/test_codependence.py", + "size": 3364, + "sha256": "1E206FE53D4C899011C33D2B08E5CA5D8A0CFAD47391E4EAD5A08FD24DEA78FF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 7 + }, + { + "path": "test/data/distance/test_distance_metric.py", + "size": 2758, + "sha256": "C20662C35D3253A3C0157456B3385523DDF6042D74C0BD031CF0DAE168D57589", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/data/labeling/test_financial_labels.py", + "size": 2624, + "sha256": "5B194D19D2565708359FB005A22336DD9C82F6BEED562D8B5E91DCE08387F8FD", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/data/labeling/test_labeling.py", + "size": 4598, + "sha256": "571BEBC1BBD55A1E1F3F2163B79C1B5D2BB3F07F355A850B5E11106EA28B33E3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/structures/test_hardened_information_driven_bars.py", + "size": 4756, + "sha256": "EC65CE1DDC9C43C72569E0E2087203309EC4C557F06FBDAA6DE4A88BF74C6650", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/data/structures/test_imbalance_bars.py", + "size": 3432, + "sha256": "95B11988BA1E13F67868EDB84E361290E8206214E67A2ADD368EC50A04035678", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/data/structures/test_run_bars.py", + "size": 2298, + "sha256": "AA84352C3434FA6BE5B36E1E04506FFFB04A71B723F52D1E76E3684AB76F5FD5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/data/structures/test_standard_bars.py", + "size": 2573, + "sha256": "7FBDEE31397875FD3CD6AA0755F79376C8298C623EDCEFEC62FCBE02801FDE81", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/structures/test_time_bars.py", + "size": 1948, + "sha256": "777F07CBF7B1285660BE260CF28DAEDD8409119B580220789C06C571E83012A9", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "size": 2496, + "sha256": "D36F02ED24979C2AC250DA8F78DCF00E84AC15629A66FF0F49B864DC7F29234F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "size": 2706, + "sha256": "3EDD684CD47C177AC3B14201D00F92D58B653F80015FBBF5FE8B199681A9CFF3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/weights/test_sample_weights.py", + "size": 3920, + "sha256": "6A682B9C8273A9480CC40EDEF503F4ADB254FDE03A7C6846A6576B2844177DA5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/ensemble/test_bagging_classifier_accuracy.py", + "size": 1055, + "sha256": "B9E9FCDEC62CCA57773D775874B1A49FE9FFDBA70AE75C1A5B44D21BE5F42F70", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/entropy_features/test_entropy.py", + "size": 8265, + "sha256": "EB0639713017FA205239818CF5EB33B37488D1F5015E41BE572DF9B3DC4B92A3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 14 + }, + { + "path": "test/features/feature_importance/test_debiased_importance.py", + "size": 3135, + "sha256": "9CB7F3A10AADC4263C5A3BC18952A832C61DD1E106E7C6D97E44480D9B77E303", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/features/feature_importance/test_feature_importance.py", + "size": 4539, + "sha256": "1CC36BB1739EEDBA96BC9CFE6C784F4B10796AD8B7030D2423BFADDA83AC4BBC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/features/feature_importance/test_generate_synthetic_data.py", + "size": 1297, + "sha256": "60E8710D8FB14D2DB9C8A8561B760A7C82E78A40B58D050DE4B454FAD9D09A19", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/feature_importance/test_orthogonal_features.py", + "size": 1377, + "sha256": "2FFF242081857CAE3677849ED9615F812C02C4F8142C2BE3F165EB7F6163E43F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/feature_importance/test_weighted_tau.py", + "size": 717, + "sha256": "DE4E792D04BE710A70F0531FB3BA4CF6F4FD281CD1CCB0AF66775B2AFF9DEB5B", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/microstructural_features/test_edge.py", + "size": 2005, + "sha256": "E7F97BFC152F1ABC9CDA7ABACCAA96F424E2F5F60178AB358B282944AF7BB1CE", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/features/microstructural_features/test_microstructure.py", + "size": 2089, + "sha256": "FAF86E2B87ED6C052039270C8FDD500399B71AFFAB9B3833ABD01C2EB7065A93", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/features/structural_breaks/test_pelt_change_points.py", + "size": 2480, + "sha256": "41D6A781C71F971EDF58728FC9C7F963C751CA07B23B8ADA7F640542EBA80DEF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/features/structural_breaks/test_structural_breaks.py", + "size": 9740, + "sha256": "3AB71A45767135BA7B3104135A86E45C301F2131FCEB190695FCAA7B5FD34872", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 14 + }, + { + "path": "test/features/structural_breaks/test_volatility_robust_sadf.py", + "size": 2732, + "sha256": "F098186FFD06AC96600F32986039804A177193313D1E237F2EE97264968E7BB8", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/hpc/test_hpc.py", + "size": 2213, + "sha256": "2E3414DD74A72FD86FED54059515E2A088F99F4C23492A1992A55A5F311F09A8", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/optimization/test_hedging.py", + "size": 2026, + "sha256": "C34C92CCD66E0B895140B2D45270AC94803AB408532888E04C6FABC9C444BC22", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/optimization/test_hrp.py", + "size": 3689, + "sha256": "08A03D26DDCA53A9793C46D76AFEC5EA2D1203B8A833895A93C29E0EBCD6A4C7", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/optimization/test_hyper_parameter_tuning.py", + "size": 3405, + "sha256": "C756502F9BA1FFC0EAFCF2F0EE3E73F500CB07D956378FFADE018CC610AEC650", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/optimization/test_nco.py", + "size": 1875, + "sha256": "07393543334EED0500954A475403AFDF68BFDB5D9AEC14768E5D21D91E41A2AE", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/pde/test_pde_solver.py", + "size": 1764, + "sha256": "9606F9460D87E1268B55A2EC5FC8AD7BE92B4E9DB64F219E0C444343CDB763DD", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/test_consolidation.py", + "size": 3427, + "sha256": "653D2C917A29D41030CE835E18736ED607BCBBC9E7A75B1A16FEFC286F391F8F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/test_deprecation.py", + "size": 1746, + "sha256": "1318D33E6B9790C48F3E366568733FB14574D0D7ECCF7086D5EF5D8BE8545930", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/test_performance.py", + "size": 6986, + "sha256": "857264FBB2C1507995BB18CBD2E9081849D9C3F4EECA73F58FBC0371FAE42F73", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/utils/test_ewma.py", + "size": 920, + "sha256": "A75275626813BC8DFE6A0A792370FC1B013ED8A5398232468CE7E871137F70D6", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "size": 2321, + "sha256": "EEBBFAC83B87A3A2BF72C9D9B203DB11D8794D9CBE7CA5125DEBE141C10E3889", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/utils/test_progress.py", + "size": 1022, + "sha256": "2CC8FBD465B007948A09C0E8975288A13D46590113FB9DA6AA441C07C55F23BA", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + } + ] +} diff --git a/docs/causal_factor_analysis.md b/docs/causal_factor_analysis.md new file mode 100644 index 0000000..fa3f013 --- /dev/null +++ b/docs/causal_factor_analysis.md @@ -0,0 +1,49 @@ +# Causal factor analysis + +The public causal API implements the source-consistent analytical scope of *Causal Factor Investing* and exposes 57 concepts. The Python namespace is `RiskLabAI.causal_factor_analysis`; the matching Julia namespace is `RiskLabAI.CausalFactorAnalysis`. + +## Public surface + +| Group | Records and functions | +|---|---| +| Optimizer (1) | `minimum_variance_factor_weights` | +| Factor-mirage analytics (10) | `StrategyPerformance`, `ColliderCoefficients`, `ColliderDiagnostics`, `confounder_undercontrolled_coefficient`, `confounder_factor_return`, `confounder_forecast_return`, `collider_overcontrolled_coefficients`, `collider_factor_return`, `collider_forecast_return`, `collider_model_diagnostics` | +| Protocol (12) | `EventHorizon`, `FoldEvidence`, `ValidationEvidence`, `VariableSelectionStage`, `CausalDiscoveryStage`, `CausalAdjustmentSetStage`, `CausalExplanatoryAndPredictivePowerStage`, `CausalPortfolioConstructionStage`, `BacktestStage`, `MultipleTestingAdjustmentsStage`, `CausalFactorProtocolReport`, `validate_causal_factor_protocol` | +| Graph identification (14) | `CausalDAG`, `PathEvidence`, `DSeparationEvidence`, `BackdoorAdjustmentEvidence`, `FrontdoorAdjustmentEvidence`, `InstrumentEvidence`, `NodeRoleEvidence`, `d_separation`, `check_backdoor_adjustment_set`, `minimal_backdoor_adjustment_sets`, `check_frontdoor_adjustment_set`, `minimal_frontdoor_adjustment_sets`, `check_instrument`, `causal_role_evidence` | +| Treatment effects (10) | `DifferenceInDifferencesEstimate`, `TreatmentEffectDecomposition`, `average_treatment_effect`, `backdoor_adjusted_average_treatment_effect`, `backdoor_adjusted_expectation`, `difference_in_differences`, `frontdoor_adjusted_probability`, `linear_instrumental_variable_effect`, `randomized_mean_difference`, `treatment_effect_decomposition` | +| Specification diagnostics (10) | `PopulationRegressionDiagnostics`, `RegressionDiagnostics`, `SpecificationExperimentResult`, `SpecificationPopulationResult`, `collider_population_diagnostics`, `collider_specification_experiment`, `confounded_mediator_population_diagnostics`, `confounded_mediator_specification_experiment`, `fork_population_diagnostics`, `fork_specification_experiment` | + +## Behavioral contract + +The optimizer requires a symmetric positive-definite covariance matrix and a full-column-rank exposure matrix. It rejects singular or numerically unresolved constraints rather than adding hidden regularization. + +Graph routines operate on a caller-supplied acyclic graph with an explicit observed-node set. They provide path evidence for d-separation, back-door, front-door, and instrument checks. These are graphical criteria, not causal-discovery procedures, positivity proofs, empirical instrument-strength tests, or claims that the supplied graph is true. Inclusion-minimal adjustment sets are not necessarily variance-optimal or minimum-cost sets. + +Treatment-effect functions evaluate explicit identification formulas from caller-supplied interventional, conditional, or grouped quantities. They do not infer that the identification assumptions hold in observed data. + +Protocol records are immutable evidence containers. The validator checks the declared seven-stage structure and fail-closed evidence rules; it does not execute empirical research or certify the truth of a declaration. + +Specification diagnostics expose exact population identities and deterministic experiments for fork, collider, and confounded-mediator structures. They are mathematical diagnostics rather than automated model selection. + +## Cross-language behavior + +Python returns immutable tuple-backed evidence records and NumPy arrays; Julia returns immutable structs and Julia vectors or matrices. Python raises `ValueError` for invalid analytical inputs; Julia raises `ArgumentError`. Canonical node and set ordering, graphical criteria, result fields, resource limits, numerical tolerances, and estimands otherwise match. + +Both implementations are tested against independent analytical results and graph oracles. Agreement between the languages is additional evidence, never the sole correctness test. + +## Exclusions + +Three later-source results remain excluded because their sources conflict: `CFA-CONF-01`, `CFA-COLL-01`, and `MIRAGE-COLL-SHIFT-01`. The clean namespace does not delegate to the excluded legacy causal implementation. + +See `../examples/causal_factor_analysis_quickstart.py` for a deterministic example and `compatibility.md` for the tested runtime policy. + +The exact ordered names, signatures, immutable record fields, implementation +modules, and source hashes are frozen in `../PUBLIC_API.json`. The seven causal +test files and their hashes are part of the exact package-scoped inventory in +`../TEST_INVENTORY.json`. + +Cross-language parity is defined over the exact 57-name set and behavior, not +over incidental export-list order. Python preserves its approved `__all__` +order; the Julia inventory uses canonical lexical order. Both language-native +digests and the canonical name-set digest are recorded in the blocked +pre-package contract. diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..245f817 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,132 @@ +# Python runtime and dependency support + +Status: **matrix verified; release blocked**. + +This policy applies to the blocked RiskLabAI `3.0.0` release candidate. The +assigned version does not authorize a build, package, upload, publication, or +release. + +## Supported interpreter and NumPy policy + +RiskLabAI supports standard GIL builds of CPython 3.12, 3.13, and 3.14. The +base numerical policy is `numpy>=2.2,<3`. A matrix lane uses only versions for +which compatible distributions actually exist; it never forces an older +NumPy source release onto a newer interpreter. + +| Lane | CPython | NumPy | Required-dependency position | +|---|---:|---:|---| +| 3.12 minimum | 3.12.13 | 2.2.0 | approved floors | +| 3.12 current | 3.12.13 | 2.5.2 | current compatible releases | +| 3.13 minimum | 3.13.9 | 2.2.0 | approved floors | +| 3.13 current | 3.13.9 | 2.5.2 | current compatible releases | +| 3.14 minimum | 3.14.3 | 2.3.2 | lowest compatible binary set | +| 3.14 current | 3.14.3 | 2.5.2 | current compatible releases | + +The NumPy lower bound was reconsidered at metadata freeze on 2026-08-21. NumPy +2.2 remains inside the two-year Scientific Python SPEC 0 support window through +2026-12-08 and has passed the approved Python 3.12 and 3.13 minimum lanes, so +`numpy>=2.2,<3` is retained for RiskLabAI 3.0.0. Python 3.14 resolves NumPy +2.3.2 or newer because that is its first tested compatible binary lane. A +future RiskLabAI release must reassess the floor rather than silently carrying +it forward. See https://scientific-python.org/specs/spec-0000/. + +The required dependency policy is: + +- pandas `>=2.3,<4`; +- SciPy `>=1.15,<2`; +- scikit-learn `>=1.6,<2`; +- statsmodels `>=0.14.6,<0.15`; +- Joblib `>=1.4,<2` on Python below 3.14 and `>=1.5.2,<2` on Python 3.14; +- PyWavelets `>=1.9,<2`. + +The Python 3.14 minimum lane resolves newer compatible SciPy and +scikit-learn releases where older releases do not publish compatible wheels. +This is an interpreter-specific resolution fact, not a reason to raise the +floor for Python 3.12 or 3.13. + +## Optional capabilities + +Optional dependencies do not constrain the lean base: + +- `speed`: Numba `>=0.67,<0.68`, with NumPy `<2.6` for this group only; +- `pde`: PyTorch `>=2.10,<3`; +- `synth`: QuantEcon `>=0.11.4,<0.12`; +- `hpo`: Optuna `>=4.9,<5`; +- `plot`: matplotlib `>=3.10,<4`, seaborn `>=0.13.2,<0.14`, and Plotly `>=6,<7`; +- `symbolic`: SymPy `>=1.14,<2`; +- `profile`: memory-profiler `==0.61.0` provisionally; +- `simulation`: tqdm `>=4.70,<5`; +- `changepoints`: ruptures `==1.1.10` on Python 3.12 and 3.13 only. + +Python 3.14 remains supported when ruptures is unavailable. Only +`pelt_change_points` is unavailable in that environment, and calling it raises +a direct dependency error. The rest of RiskLabAI remains importable. + +The financial-feature simulation no longer relies on the stale `ta` package. +ADX, RSI, CCI, stochastic momentum, ROC, ATR, and Ichimoku lines are computed +internally from standard definitions and have hand-calculated oracle tests. + +## Continuous integration contract + +The governed CI workflow installs the distribution non-editably before every +runtime test. Import-origin probes run outside the repository checkout and +reject any `RiskLabAI` module resolved from the preserved repository root. +Tests that launch child interpreters also use clean temporary working +directories, so a subprocess cannot silently resolve the preserved root +package in place of the installed `src` distribution. + +The base matrix contains six valid interpreter/NumPy combinations: the lowest +supported NumPy binary and the latest supported NumPy 2.x release on each of +CPython 3.12, 3.13, and 3.14. Three additional optional-feature lanes install +the feature-specific groups on those interpreter lines. The `speed` group +keeps its narrower NumPy `<2.6` constraint, and the Python 3.14 lane verifies +that only the unavailable changepoint backend is skipped. + +Pinned Black and Ruff checks cover the independently maintained clean causal +source and tests. Preserved source and regression tests remain governed by the +complete installed-package runtime matrix and are not rewritten solely to +satisfy a new style tool. + +## Deprecation policy + +Public compatibility aliases retained in RiskLabAI 3.0.0 remain callable and +emit `DeprecationWarning`. Their removal target is 4.0.0. Package-scoped tests +bind both continued availability and the stated target; no preserved alias is +deleted in this release. + +## Matrix evidence + +Every required base lane passed all 342 frozen causal-factor tests. Across six +lanes this is 2,052 causal assertions with no failures. + +The final self-contained package test tree collects exactly 704 tests: 354 +preserved-library tests, 342 causal-factor tests, and 8 independent technical- +indicator oracles. Its complete Python 3.12 current-dependency run succeeded +with 703 passes and one declared Windows platform skip. The package inventory +records both the collected suite and the 692-test applicable base matrix used +on every supported interpreter/dependency lane. + +The complete preserved base suite passed 342 applicable tests per lane. Five +platform or optional-capability cases were skipped by their declared guards; +the two QuantEcon-dependent cases were exercised separately in the `synth` +lanes. The new indicator-oracle suite passed 8 tests in every lane. Numba's +compiled and pure-Python paths agreed in every lane. PyTorch 2.10 and 2.13 both +passed the preserved PDE tests on all three interpreter lines. Plotting, +symbolic, profiling, HPO, and synthetic-data smoke tests passed in their +feature-specific environments. + +One real conditional floor was discovered: Joblib releases before 1.5.2 are +not reliable for the preserved Windows parallel-HPO path on Python 3.14. +Joblib 1.5.2 and 1.5.3 pass; older Python lines continue to pass with Joblib +1.4.x. + +The causal public design remains exactly 57 concepts. Compatibility repairs +did not import or reintroduce the excluded legacy causal implementation. + +## Remaining release gates + +The metadata, public surface, tests, source allowlists, installed-origin probes, +and CI contract are complete. Source installs used isolated verification +environments and retained no wheel or source-distribution artifact. Formal +artifact inspection and separate human authorization for version-control, +publication, and release remain outstanding. diff --git a/examples/causal_factor_analysis_quickstart.py b/examples/causal_factor_analysis_quickstart.py new file mode 100644 index 0000000..c3800be --- /dev/null +++ b/examples/causal_factor_analysis_quickstart.py @@ -0,0 +1,37 @@ +"""Small, deterministic examples for the frozen causal-factor API.""" + +import numpy as np +import RiskLabAI + +from RiskLabAI.causal_factor_analysis import ( + CausalDAG, + average_treatment_effect, + check_backdoor_adjustment_set, + minimum_variance_factor_weights, +) + +assert RiskLabAI.__version__ == "3.0.0" + + +covariance = np.diag([1.0, 2.0, 4.0]) +factor_exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) +target_exposures = np.array([0.0, 1.0]) +weights = minimum_variance_factor_weights( + covariance, factor_exposures, target_exposures +) +np.testing.assert_allclose(weights, [-2.0 / 7.0, 5.0 / 7.0, 2.0 / 7.0]) + +effect = average_treatment_effect(3.5, 1.25) +assert effect == 2.25 + +dag = CausalDAG( + nodes=("T", "U", "Y"), + directed_edges=(("U", "T"), ("U", "Y"), ("T", "Y")), + observed_nodes=("T", "U", "Y"), +) +assert not check_backdoor_adjustment_set(dag, "T", "Y").admissible +assert check_backdoor_adjustment_set(dag, "T", "Y", ("U",)).admissible + +print("minimum-variance weights:", weights) +print("average treatment effect:", effect) +print("back-door adjustment set: ('U',)") diff --git a/pyproject.toml b/pyproject.toml index 88dacff..154c63b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,115 +1,105 @@ [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=77", "wheel>=0.45"] build-backend = "setuptools.build_meta" [project] name = "RiskLabAI" -version = "2.0.3" +version = "3.0.0" +description = "Financial machine-learning and causal-factor analysis tools for research and model validation." +readme = "README.md" +requires-python = ">=3.12,<3.15" +license = "BSD-3-Clause" +license-files = ["LICENSE"] authors = [ - { name = "RiskLab", email = "arian@risklab.ai" }, + { name = "Hamid Arian", email = "arian@risklab.ai" }, ] -description = "Financial AI using Python, based on 'Advances in Financial Machine Learning' and 'Machine Learning for Asset Managers'." -readme = "README.md" -license = { file = "LICENSE" } -requires-python = ">=3.9" -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: BSD License", - "Intended Audience :: Developers", - "Intended Audience :: Financial and Insurance Industry", - "Intended Audience :: Science/Research", - "Operating System :: OS Independent", - "Topic :: Office/Business :: Financial", - "Topic :: Scientific/Engineering :: Artificial Intelligence", +maintainers = [ + { name = "Hamid Arian", email = "arian@risklab.ai" }, ] keywords = [ - "finance", - "machine learning", - "quantitative finance", - "fintech", - "de prado", - "backtesting", + "causal inference", + "factor investing", + "financial machine learning", + "quantitative finance", + "risk management", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Financial and Insurance Industry", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", ] - -# --- CORE DEPENDENCIES --- -# These are fast to install and essential for the library's main functions. dependencies = [ - "numpy>=1.22,<3", - "pandas>=1.5", - "scipy>=1.8", - "scikit-learn>=1.1", - "statsmodels>=0.13", - "numba>=0.57", - "joblib>=1.1", - "ta>=0.10", - "tqdm>=4.60", - "sympy>=1.10", + "numpy>=2.2,<3", + "pandas>=2.3,<4", + "scipy>=1.15,<2", + "scikit-learn>=1.6,<2", + "statsmodels>=0.14.6,<0.15", + "joblib>=1.4,<2; python_version < '3.14'", + "joblib>=1.5.2,<2; python_version >= '3.14'", + "PyWavelets>=1.9,<2", ] -# --- OPTIONAL DEPENDENCIES --- -# These are installed on-demand by users, e.g., pip install RiskLabAI[pde] [project.optional-dependencies] -pde = ["torch"] -synth = ["quantecon"] -plot = ["matplotlib", "seaborn" , "plotly"] -dev = [ - "yfinance", - "memory_profiler", - "jupyterlab", - "stable-baselines3", - "gymnasium", - "torchvision", # Often needed with torch - "torchaudio", # Often needed with torch - "pytest", - # Linters pinned to match CI exactly (.github/workflows/ci.yml). - "black==26.5.1", - "ruff==0.15.17", +speed = [ + "numba>=0.67,<0.68", + "numpy>=2.2,<2.6", ] -all = [ - "RiskLabAI[pde]", - "RiskLabAI[synth]", - "RiskLabAI[plot]", +pde = ["torch>=2.10,<3"] +synth = ["quantecon>=0.11.4,<0.12"] +hpo = ["optuna>=4.9,<5"] +plot = [ + "matplotlib>=3.10,<4", + "seaborn>=0.13.2,<0.14", + "plotly>=6,<7", ] - +symbolic = ["sympy>=1.14,<2"] +profile = ["memory-profiler==0.61.0"] +simulation = ["tqdm>=4.70,<5"] +changepoints = ["ruptures==1.1.10; python_version < '3.14'"] +test = ["pytest>=8.4,<10"] [project.urls] -Homepage = "https://github.com/RiskLabAI/RiskLabAI.py" -"Bug Tracker" = "https://github.com/RiskLabAI/RiskLabAI.py/issues" -Changelog = "https://github.com/RiskLabAI/RiskLabAI.py/blob/main/CHANGELOG.md" +Repository = "https://github.com/RiskLabAI/RiskLabAI.py" +Issues = "https://github.com/RiskLabAI/RiskLabAI.py/issues" +Documentation = "https://github.com/RiskLabAI/RiskLabAI.py#readme" -# Package only the library; the repo root also holds local-only appraisal -# harnesses under bench/, which must never be discovered or shipped. -[tool.setuptools.packages.find] -include = ["RiskLabAI*"] +[tool.setuptools] +package-dir = { "" = "src" } +packages = [ + "RiskLabAI", + "RiskLabAI.backtest", + "RiskLabAI.backtest.validation", + "RiskLabAI.causal_factor_analysis", + "RiskLabAI.cluster", + "RiskLabAI.controller", + "RiskLabAI.core", + "RiskLabAI.data", + "RiskLabAI.data.denoise", + "RiskLabAI.data.differentiation", + "RiskLabAI.data.distance", + "RiskLabAI.data.labeling", + "RiskLabAI.data.structures", + "RiskLabAI.data.synthetic_data", + "RiskLabAI.data.weights", + "RiskLabAI.ensemble", + "RiskLabAI.features", + "RiskLabAI.features.entropy_features", + "RiskLabAI.features.feature_importance", + "RiskLabAI.features.microstructural_features", + "RiskLabAI.features.structural_breaks", + "RiskLabAI.hpc", + "RiskLabAI.optimization", + "RiskLabAI.pde", + "RiskLabAI.utils", +] +include-package-data = false [tool.pytest.ini_options] testpaths = ["test"] -# Resolve `import RiskLabAI` from the repo root so a bare `pytest` works without -# an editable install (and regardless of any untracked bench/ working tree). -pythonpath = ["."] - -[tool.black] -line-length = 88 -target-version = ["py39"] - -[tool.ruff] -line-length = 88 -target-version = "py39" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "UP", "B"] -# E501 deferred until the mechanical black formatting pass lands -ignore = ["E501"] - -[tool.ruff.lint.per-file-ignores] -# Package __init__ files exist to re-export their submodules' public API. -"__init__.py" = ["F401", "F403", "F405"] -# pde/solver builds the networks from `from ...model import *`; the torch-only -# model module is intentionally left star-imported. -"RiskLabAI/pde/solver.py" = ["F403", "F405"] -# base.py re-exports the canonical interfaces lazily via module __getattr__, -# so the names listed in __all__ are resolved on access, not at definition. -"RiskLabAI/core/base.py" = ["F822"] -# importorskip("torch") must execute before the torch-dependent imports below -# it, so module-level imports cannot all sit at the top of the file. -"test/pde/test_pde_solver.py" = ["E402"] +addopts = "-ra --strict-config --strict-markers" diff --git a/src/RiskLabAI/__init__.py b/src/RiskLabAI/__init__.py new file mode 100644 index 0000000..2d2afd3 --- /dev/null +++ b/src/RiskLabAI/__init__.py @@ -0,0 +1,53 @@ +"""RiskLabAI financial machine-learning and causal-factor toolkit.""" + +from importlib import import_module as _import_module + +from . import causal_factor_analysis + +__version__ = "3.0.0" + +_LEGACY_SUBMODULES = frozenset( + { + "backtest", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils", + } +) + +__all__ = [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils", +] + + +def __getattr__(name): + """Load only a declared legacy subpackage when it is first requested.""" + if name in _LEGACY_SUBMODULES: + module = _import_module(".{}".format(name), __name__) + globals()[name] = module + return module + raise AttributeError("module {!r} has no attribute {!r}".format(__name__, name)) + + +def __dir__(): + """Return the exact declared package surface.""" + return list(__all__) diff --git a/src/RiskLabAI/_deprecation.py b/src/RiskLabAI/_deprecation.py new file mode 100644 index 0000000..e3207e0 --- /dev/null +++ b/src/RiskLabAI/_deprecation.py @@ -0,0 +1,123 @@ +""" +Internal helpers for deprecating public names without breaking callers. + +Used by the 2.0.0 naming canon (see ``NAMING_CANON_2.0.0.md``): every renamed +function, class, or constant keeps working throughout the 2.0.x series and emits +a :class:`DeprecationWarning` naming its replacement. The aliases are scheduled +for removal in 4.0.0. + +Three mechanisms, one per kind of object: + +- :func:`deprecated_alias` wraps a renamed **function**. +- :func:`deprecated_class` subclasses a renamed **class**. +- :func:`warn_deprecated_name` is called from a module ``__getattr__`` (PEP 562) + for renamed **module-level names** such as constants. +""" + +from __future__ import annotations + +import functools +import warnings +from typing import Any, Callable, TypeVar + +__all__ = ["deprecated_alias", "deprecated_class", "warn_deprecated_name"] + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def deprecated_alias(new_func: _F, old_name: str, *, removed_in: str) -> _F: + """ + Return a thin wrapper that calls ``new_func`` but warns under the old name. + + Parameters + ---------- + new_func : Callable + The renamed (canonical) function. + old_name : str + The deprecated public name being kept alive. + removed_in : str + Version in which the alias will be removed (e.g. ``"4.0.0"``). + + Returns + ------- + Callable + A wrapper with ``__name__`` set to ``old_name`` that emits a + :class:`DeprecationWarning` and delegates to ``new_func``. + """ + + @functools.wraps(new_func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + warnings.warn( + f"{old_name}() is deprecated and will be removed in {removed_in}; " + f"use {new_func.__name__}() instead.", + DeprecationWarning, + stacklevel=2, + ) + return new_func(*args, **kwargs) + + wrapper.__name__ = old_name + wrapper.__qualname__ = old_name + wrapper.__doc__ = ( + f"Deprecated alias for :func:`{new_func.__name__}` " + f"(removed in {removed_in})." + ) + return wrapper # type: ignore[return-value] + + +def deprecated_class(new_cls: type, old_name: str, *, removed_in: str) -> type: + """ + Return a subclass of ``new_cls`` that warns when instantiated. + + The subclass behaves identically to ``new_cls`` but emits a + :class:`DeprecationWarning` on construction, naming the replacement. + + Parameters + ---------- + new_cls : type + The renamed (canonical) class. + old_name : str + The deprecated public class name being kept alive. + removed_in : str + Version in which the alias will be removed. + """ + + def __init__(self: Any, *args: Any, **kwargs: Any) -> None: + warnings.warn( + f"{old_name} is deprecated and will be removed in {removed_in}; " + f"use {new_cls.__name__} instead.", + DeprecationWarning, + stacklevel=2, + ) + new_cls.__init__(self, *args, **kwargs) + + shim = type(old_name, (new_cls,), {"__init__": __init__}) + shim.__doc__ = ( + f"Deprecated alias for :class:`{new_cls.__name__}` " + f"(removed in {removed_in})." + ) + shim.__module__ = new_cls.__module__ + return shim + + +def warn_deprecated_name(old_name: str, new_name: str, *, removed_in: str) -> None: + """ + Emit a deprecation warning for a renamed module-level name. + + Call this from a module's ``__getattr__`` (PEP 562) before returning the + value bound to ``new_name``. + + Parameters + ---------- + old_name : str + The deprecated name that was accessed. + new_name : str + The canonical replacement name. + removed_in : str + Version in which the old name will be removed. + """ + warnings.warn( + f"{old_name} is deprecated and will be removed in {removed_in}; " + f"use {new_name} instead.", + DeprecationWarning, + stacklevel=3, + ) diff --git a/src/RiskLabAI/backtest/__init__.py b/src/RiskLabAI/backtest/__init__.py new file mode 100644 index 0000000..614e601 --- /dev/null +++ b/src/RiskLabAI/backtest/__init__.py @@ -0,0 +1,210 @@ +""" +RiskLabAI Backtesting Module + +This package provides tools for: +- Backtest simulation +- Bet sizing and strategy risk analysis +- Overfitting detection (PBO, PSR) +- Advanced cross-validation (in the .validation submodule) +""" + +from . import validation +from .advanced_bet_sizing import ( + PlattCalibrator, + distributionally_robust_kelly_fraction, + expected_calibration_error, + kelly_bet_fraction, +) +from .backtest_overfitting_simulation import ( + backtest_overfitting_simulation_financial_metrics_rank_correlation, + backtest_overfitting_simulation_model_complexity, + format_cpu_info, + get_cpu_info, + measure_all_cv_computational_requirements, + measure_cpcv_parallelization, + measure_cpcv_scalability, + noised_backtest_overfitting_simulation, + # This file contains many functions, exporting the main ones + overall_backtest_overfitting_simulation, + overall_novel_methods_backtest_overfitting_simulation, + temporal_backtest_overfitting_simulation, + time_temporal_backtest_overfitting_simulation, + varying_embargo_backtest_overfitting_simulation, +) +from .backtest_statistics import ( + bet_timing, + calculate_hhi, + calculate_hhi_concentration, + calculate_holding_period, + compute_drawdowns_time_under_water, +) +from .backtest_statistics import ( + sharpe_ratio as pbo_sharpe_ratio, +) +from .backtest_synthetic_data import synthetic_back_testing +from .bet_sizing import ( + Signal, + TPos, + average_bet_sizes, + avg_active_signals, + avgActiveSignals, + bet_size_sigmoid, + betSize, + compute_sigmoid_width, + discrete_signal, + discreteSignal, + generate_signal, + getW, + inverse_price, + inversePrice, + limit_price, + limitPrice, + mp_avg_active_signals, + mpAvgActiveSignals, + probability_bet_size, + strategy_bet_sizing, + target_position, +) +from .multiple_testing import ( + benjamini_hochberg_yekutieli_adjusted_p_values, + haircut_sharpe_ratios, + holm_adjusted_p_values, + sharpe_ratio_p_values, +) +from .ou_trading_rules import ( + fit_ornstein_uhlenbeck, + optimal_ou_trading_rule, + ou_rule_metrics, +) +from .probabilistic_sharpe_ratio import ( + benchmark_sharpe_ratio, + probabilistic_sharpe_ratio, +) +from .probability_of_backtest_overfitting import ( + performance_evaluation, + probability_of_backtest_overfitting, +) +from .robust_statistics import ( + conditional_expected_drawdown, + sharpe_difference_test, +) +from .sharpe_inference import ( + lplz_sharpe_inference, + newey_west_automatic_lag, + newey_west_long_run_variance, + sharpe_ratio_influence_function, +) +from .strategy_risk import ( + bin_frequency, + binomial_sharpe_ratio, + calculate_strategy_risk, + failure_probability, + implied_precision, + mix_gaussians, + sharpe_ratio_trials, + target_sharpe_ratio_symbolic, +) +from .test_set_overfitting import ( + estimated_sharpe_ratio_z_statistics, + expected_max_sharpe_ratio, + generate_max_sharpe_ratios, + mean_std_error, + strategy_type1_error_probability, + strategy_type2_error_probability, + theta_for_type2_error, +) + +# Define what `from RiskLabAI.backtest import *` will import +__all__ = [ + "validation", + # from advanced_bet_sizing (Appraisal 18 admits) + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction", + # from robust_statistics + ou_trading_rules (Appraisal 22, 23 admits) + "conditional_expected_drawdown", + "sharpe_difference_test", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "fit_ornstein_uhlenbeck", + # from backtest_statistics + "bet_timing", + "calculate_holding_period", + "calculate_hhi_concentration", + "calculate_hhi", + "compute_drawdowns_time_under_water", + # from backtest_synthetic_data + "synthetic_back_testing", + # from bet_sizing (canonical 2.0.0 names) + "probability_bet_size", + "average_bet_sizes", + "strategy_bet_sizing", + "avg_active_signals", + "mp_avg_active_signals", + "discrete_signal", + "generate_signal", + "bet_size_sigmoid", + "target_position", + "inverse_price", + "limit_price", + "compute_sigmoid_width", + # from bet_sizing (deprecated aliases, removed in 4.0.0) + "avgActiveSignals", + "mpAvgActiveSignals", + "discreteSignal", + "Signal", + "betSize", + "TPos", + "inversePrice", + "limitPrice", + "getW", + # from strategy_risk + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic", + "implied_precision", + "bin_frequency", + "binomial_sharpe_ratio", + "mix_gaussians", + "failure_probability", + "calculate_strategy_risk", + # from test_set_overfitting + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "estimated_sharpe_ratio_z_statistics", + "strategy_type1_error_probability", + "theta_for_type2_error", + "strategy_type2_error_probability", + # from probability_of_backtest_overfitting + "pbo_sharpe_ratio", + "performance_evaluation", + "probability_of_backtest_overfitting", + # from probabilistic_sharpe_ratio + "probabilistic_sharpe_ratio", + "benchmark_sharpe_ratio", + # from multiple_testing + "sharpe_ratio_p_values", + "holm_adjusted_p_values", + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + # from sharpe_inference + "sharpe_ratio_influence_function", + "newey_west_long_run_variance", + "newey_west_automatic_lag", + "lplz_sharpe_inference", + # from backtest_overfitting_..._simulation + "overall_backtest_overfitting_simulation", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation", + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "noised_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "measure_all_cv_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "get_cpu_info", + "format_cpu_info", +] diff --git a/src/RiskLabAI/backtest/advanced_bet_sizing.py b/src/RiskLabAI/backtest/advanced_bet_sizing.py new file mode 100644 index 0000000..5172948 --- /dev/null +++ b/src/RiskLabAI/backtest/advanced_bet_sizing.py @@ -0,0 +1,227 @@ +r""" +Advanced bet sizing: probability calibration before sizing, and distributionally-robust Kelly. + +de Prado sizes a bet from a predicted probability via a sigmoid (AFML ch.10, in `bet_sizing.py`). Two +admitted extensions sit alongside that baseline: + +- **Platt calibration before sizing** (Meyer-Barziy-Joubert 2023). A classifier's raw probability is + usually miscalibrated, so sizing straight from it over- or under-bets. Fitting a Platt (logistic) + calibrator on a held-out block and sizing from the calibrated probability recovers near-optimal sizing + when the input is miscalibrated, and is a safe no-op when it is already calibrated. +- **Distributionally-robust Kelly** (Sun-Boyd 2018). Point-estimate Kelly is fragile to estimation + error. Sizing at the pessimistic edge of a confidence box on the win probability gives near-optimal + growth with materially lower drawdown under estimation uncertainty, and converges to full Kelly as the + box collapses with abundant data. + +Admitted in Appraisal 18 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tags, verbatim from the verdict: + + Platt calibration: calibrate the meta-label probability before sizing when it is miscalibrated + (measurable by ECE); use Platt, not isotonic, which overfits well-calibrated inputs; the benefit is + largest under strong miscalibration and converges to no-op when already calibrated. + + DR-Kelly: prefer DR-Kelly over de Prado sigmoid / full Kelly when estimation uncertainty is material + (small samples, noisy edge): near-optimal growth with lower drawdown; converges to Kelly with + abundant data. + +Held-out confirmed (appraisals/18_results, HELDOUT.md): on the sealed high-miscalibration corner Platt +closes the growth gap to the Kelly optimum distinguishably more than the de Prado sigmoid (paired +0.0032 [0.0030, 0.0034]); on the sealed tight-drawdown / small-sample corner DR-Kelly has lower +from-initial drawdown than full Kelly (0.144 vs 0.409) with near-optimal growth. Real-data net-of-cost +confirmation for both is a tracked obligation (`REAL_DATA_FOLLOWUPS.md`). Isotonic calibration and +risk-constrained Kelly were NOT admitted (shelved). Evidence and caveats: appraisals/18_verdict.md. + +The DR-Kelly box-ambiguity worst case has a closed form (Kelly at the pessimistic probability, because a +long favorable bet's log-growth is monotone in the win probability), so no convex solver is required here; +``cvxpy`` (Apache-2.0) would only be needed for a general, non-box ambiguity set. + +References +---------- +Platt, J. (1999) Probabilistic outputs for support vector machines and comparisons to regularized + likelihood methods. Advances in Large Margin Classifiers. +Meyer, B., Barziy, I. and Joubert, J. F. (2023) Meta-labeling: calibration and the bet-sizing of + meta-label probabilities. +Sun, Q. and Boyd, S. (2018) Distributional robustness in Kelly gambling. (Distributionally-robust Kelly.) +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 10 (the de Prado sigmoid baseline). +""" + +from __future__ import annotations + +import numpy as np +from sklearn.linear_model import LogisticRegression + + +def kelly_bet_fraction( + probability, + payout_ratio: float = 1.0, + max_fraction: float = 1.0, +) -> np.ndarray: + r""" + Growth-optimal Kelly fraction for a binary bet, clipped to ``[0, max_fraction]`` (long only). + + For a favorable bet that pays ``+payout_ratio`` per unit staked on a win (probability ``p``) and + ``-1`` on a loss, the Kelly fraction is :math:`f^* = (p(b+1) - 1)/b` with ``b = payout_ratio``. + + Parameters + ---------- + probability : float or np.ndarray + Win probability/probabilities. + payout_ratio : float, default 1.0 + Net odds ``b`` (even money by default). + max_fraction : float, default 1.0 + Upper clip on the fraction. + + Returns + ------- + np.ndarray + The Kelly fraction(s) in ``[0, max_fraction]``. + + Examples + -------- + >>> float(kelly_bet_fraction(0.6)) + 0.2 + """ + p = np.asarray(probability, dtype=float) + b = float(payout_ratio) + f = (p * (b + 1.0) - 1.0) / b + return np.clip(f, 0.0, max_fraction) + + +def expected_calibration_error(probabilities, outcomes, n_bins: int = 10) -> float: + r""" + Expected Calibration Error: the bin-mass-weighted gap between confidence and accuracy. + + Use this to decide whether calibration is worthwhile: a high ECE indicates the probabilities are + miscalibrated (the regime where :class:`PlattCalibrator` helps); a near-zero ECE means calibration + should be a no-op. + + Parameters + ---------- + probabilities : np.ndarray + Predicted probabilities in ``[0, 1]``. + outcomes : np.ndarray + Binary realized outcomes ``{0, 1}``. + n_bins : int, default 10 + Number of equal-width probability bins. + + Returns + ------- + float + The ECE in ``[0, 1]``. + """ + p = np.asarray(probabilities, dtype=float) + y = np.asarray(outcomes, dtype=float) + edges = np.linspace(0.0, 1.0, n_bins + 1) + n = p.size + ece = 0.0 + for i in range(n_bins): + lo, hi = edges[i], edges[i + 1] + mask = (p > lo) & (p <= hi) if i > 0 else (p >= lo) & (p <= hi) + if mask.sum() == 0: + continue + ece += (mask.sum() / n) * abs(y[mask].mean() - p[mask].mean()) + return float(ece) + + +class PlattCalibrator: + r""" + Platt scaling: a logistic recalibration of predicted probabilities, fit on a held-out block. + + Fits a logistic regression of the binary outcome on ``logit(p_hat)``; ``transform`` maps a raw + probability to the calibrated one. Prefer Platt over isotonic calibration: isotonic overfits the + calibration map and hurts already-calibrated inputs, whereas Platt's two-parameter logistic form is + a safe near-no-op when the input is already calibrated (the characterized failure mode in the + verdict). Calibrate before sizing (feed the calibrated probability into the de Prado sigmoid or a + Kelly sizer) when ECE indicates miscalibration. See the module docstring for the verbatim scope tag + and appraisals/18_verdict.md. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> y = rng.integers(0, 2, 500) + >>> p_hat = np.clip(0.5 + 0.3 * (2 * y - 1) + 0.1 * rng.standard_normal(500), 0.01, 0.99) + >>> cal = PlattCalibrator().fit(p_hat, y) + >>> p_cal = cal.transform(p_hat) + >>> p_cal.shape == p_hat.shape + True + """ + + def __init__(self) -> None: + self._model: LogisticRegression | None = None + + @staticmethod + def _logit(p_hat) -> np.ndarray: + x = np.clip(np.asarray(p_hat, dtype=float), 1e-4, 1 - 1e-4) + return np.log(x / (1.0 - x)).reshape(-1, 1) + + def fit(self, predicted_probabilities, outcomes) -> PlattCalibrator: + """Fit the Platt logistic map on (predicted probability, binary outcome). Returns ``self``.""" + y = np.asarray(outcomes, dtype=int) + if np.unique(y).size < 2: + # Degenerate block (one class only): leave as an identity calibrator. + self._model = None + return self + model = LogisticRegression(C=1e6, solver="lbfgs", max_iter=1000) + model.fit(self._logit(predicted_probabilities), y) + self._model = model + return self + + def transform(self, predicted_probabilities) -> np.ndarray: + """Map raw probabilities to calibrated probabilities (identity if fit on one class only).""" + if self._model is None: + return np.asarray(predicted_probabilities, dtype=float) + return self._model.predict_proba(self._logit(predicted_probabilities))[:, 1] + + +def distributionally_robust_kelly_fraction( + probability_estimate, + n_observations: int, + payout_ratio: float = 1.0, + max_fraction: float = 1.0, + confidence_z: float = 1.0, +) -> np.ndarray: + r""" + Distributionally-robust Kelly fraction over a confidence box on the win probability (Sun-Boyd 2018). + + Builds a one-sided confidence box ``p_lo = p_hat - z * sqrt(p_hat (1 - p_hat) / n)`` and sizes with + growth-optimal Kelly at the pessimistic ``p_lo``. For a long favorable bet the worst-case log-growth + over the box is attained at the lowest win probability, so this closed form is the box-ambiguity + distributionally-robust optimum (no convex solver needed). As ``n`` grows the box collapses and the + fraction converges to full Kelly at ``p_hat``. + + Prefer DR-Kelly over de Prado sigmoid / full Kelly when estimation uncertainty is material (small + samples, noisy edge): near-optimal growth with lower drawdown; converges to Kelly with abundant data. + See the module docstring for the verbatim scope tag and appraisals/18_verdict.md. + + Parameters + ---------- + probability_estimate : float or np.ndarray + The estimated win probability ``p_hat``. + n_observations : int + The number of observations behind the estimate (drives the box width). + payout_ratio : float, default 1.0 + Net odds ``b``. + max_fraction : float, default 1.0 + Upper clip on the fraction. + confidence_z : float, default 1.0 + Box half-width in standard errors (1.0 ~ one-sigma pessimism). + + Returns + ------- + np.ndarray + The distributionally-robust Kelly fraction(s) in ``[0, max_fraction]``. + + Examples + -------- + >>> import numpy as np + >>> small = float(distributionally_robust_kelly_fraction(0.6, n_observations=40)) + >>> large = float(distributionally_robust_kelly_fraction(0.6, n_observations=100000)) + >>> small < large <= 0.2 + 1e-9 # converges up to full Kelly (0.2) as n grows + True + """ + p_hat = np.clip(np.asarray(probability_estimate, dtype=float), 1e-4, 1 - 1e-4) + standard_error = np.sqrt(p_hat * (1.0 - p_hat) / max(int(n_observations), 1)) + p_lo = np.clip(p_hat - confidence_z * standard_error, 1e-4, 1 - 1e-4) + return kelly_bet_fraction( + p_lo, payout_ratio=payout_ratio, max_fraction=max_fraction + ) diff --git a/src/RiskLabAI/backtest/backtest_overfitting_simulation.py b/src/RiskLabAI/backtest/backtest_overfitting_simulation.py new file mode 100644 index 0000000..2315e63 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_overfitting_simulation.py @@ -0,0 +1,1560 @@ +""" +Orchestrates complex backtest overfitting and simulation scenarios, +including PBO, DSR, and hardware performance profiling. + +## TODO: +- [ ] **Refactor Feature Generation:** Move hardcoded parameters in + `financial_features_backtest_overfitting_simulation` + (e.g., volatility=100, rolling=20, indicator windows) + to a configuration dictionary or function arguments. +- [ ] **Refactor Profiling:** Move hardware/performance functions + (`get_cpu_info`, `format_cpu_info`, + `measure_computational_requirements`, etc.) + to a separate `RiskLabAI.utils.profiling` module + to reduce this module's responsibilities. +""" + +import itertools +import subprocess +import time +import warnings +from math import ceil +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from scipy import stats as ss +from scipy.stats import kendalltau +from sklearn.linear_model import LogisticRegression + +from RiskLabAI.core import CROSS_VALIDATORS +from RiskLabAI.data.differentiation import fractionally_differentiated_log_price +from RiskLabAI.data.labeling import ( + cusum_filter_events_dynamic_threshold, + daily_volatility_with_log_returns, + meta_events, + meta_labeling, + vertical_barrier, +) +from RiskLabAI.data.weights import sample_weight_absolute_return_meta_labeling +from RiskLabAI.utils import determine_strategy_side + +from .bet_sizing import strategy_bet_sizing +from .probabilistic_sharpe_ratio import ( + benchmark_sharpe_ratio, + probabilistic_sharpe_ratio, +) +from .probability_of_backtest_overfitting import probability_of_backtest_overfitting + + +def _progress(iterable): + try: + from tqdm import tqdm + except ImportError: + return iterable + return tqdm(iterable) + + +def _wilder_change_average(values: np.ndarray, window: int) -> np.ndarray: + """Return Wilder's recursive average for quantities defined on price changes.""" + if not isinstance(window, int) or isinstance(window, bool) or window < 1: + raise ValueError("window must be a positive integer") + + array = np.asarray(values, dtype=float) + result = np.full(array.shape, np.nan, dtype=float) + if array.ndim != 1: + raise ValueError("values must be one-dimensional") + if array.size <= window: + return result + + seed = array[1 : window + 1] + if not np.isfinite(seed).all(): + return result + result[window] = float(seed.mean()) + for position in range(window + 1, array.size): + value = array[position] + if not np.isfinite(value): + result[position] = np.nan + elif np.isfinite(result[position - 1]): + result[position] = ( + result[position - 1] * (window - 1) + value + ) / window + return result + + +def _relative_strength_index(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + changes = np.empty(values.shape, dtype=float) + changes[0] = 0.0 + changes[1:] = np.diff(values) + average_gain = _wilder_change_average(np.maximum(changes, 0.0), window) + average_loss = _wilder_change_average(np.maximum(-changes, 0.0), window) + + denominator = average_gain + average_loss + result = np.full(values.shape, np.nan, dtype=float) + np.divide(100.0 * average_gain, denominator, out=result, where=denominator > 0.0) + result[(denominator == 0.0) & np.isfinite(denominator)] = 50.0 + return pd.Series(result, index=prices.index, dtype=float) + + +def _average_true_range(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + true_range = np.empty(values.shape, dtype=float) + true_range[0] = 0.0 + true_range[1:] = np.abs(np.diff(values)) + return pd.Series( + _wilder_change_average(true_range, window), + index=prices.index, + dtype=float, + ) + + +def _average_directional_index(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + changes = np.empty(values.shape, dtype=float) + changes[0] = 0.0 + changes[1:] = np.diff(values) + true_range = np.abs(changes) + positive_movement = np.maximum(changes, 0.0) + negative_movement = np.maximum(-changes, 0.0) + + average_range = _wilder_change_average(true_range, window) + average_positive = _wilder_change_average(positive_movement, window) + average_negative = _wilder_change_average(negative_movement, window) + positive_index = np.zeros(values.shape, dtype=float) + negative_index = np.zeros(values.shape, dtype=float) + np.divide( + 100.0 * average_positive, + average_range, + out=positive_index, + where=average_range > 0.0, + ) + np.divide( + 100.0 * average_negative, + average_range, + out=negative_index, + where=average_range > 0.0, + ) + direction_sum = positive_index + negative_index + directional_index = np.zeros(values.shape, dtype=float) + np.divide( + 100.0 * np.abs(positive_index - negative_index), + direction_sum, + out=directional_index, + where=direction_sum > 0.0, + ) + + result = np.full(values.shape, np.nan, dtype=float) + first = 2 * window - 1 + if values.size > first: + result[first] = float(directional_index[window : first + 1].mean()) + for position in range(first + 1, values.size): + result[position] = ( + result[position - 1] * (window - 1) + + directional_index[position] + ) / window + return pd.Series(result, index=prices.index, dtype=float) + + +def _commodity_channel_index(prices: pd.Series, window: int = 20) -> pd.Series: + rolling_mean = prices.rolling(window, min_periods=window).mean() + mean_deviation = prices.rolling(window, min_periods=window).apply( + lambda values: float(np.mean(np.abs(values - values.mean()))), raw=True + ) + denominator = 0.015 * mean_deviation + result = (prices - rolling_mean) / denominator + return result.where(denominator != 0.0, 0.0).astype(float) + + +def _stochastic_oscillator( + prices: pd.Series, window: int = 14, smooth_window: int = 3 +) -> tuple[pd.Series, pd.Series]: + rolling_low = prices.rolling(window, min_periods=window).min() + rolling_high = prices.rolling(window, min_periods=window).max() + price_range = rolling_high - rolling_low + oscillator = 100.0 * (prices - rolling_low) / price_range + oscillator = oscillator.where(price_range != 0.0, 50.0).astype(float) + signal = oscillator.rolling( + smooth_window, min_periods=smooth_window + ).mean() + return oscillator, signal + + +def _ichimoku_lines( + prices: pd.Series, + conversion_window: int = 9, + base_window: int = 26, + span_window: int = 52, +) -> tuple[pd.Series, pd.Series, pd.Series, pd.Series]: + def midpoint(window: int) -> pd.Series: + rolling = prices.rolling(window, min_periods=window) + return (rolling.max() + rolling.min()) / 2.0 + + conversion_line = midpoint(conversion_window) + base_line = midpoint(base_window) + span_a = (conversion_line + base_line) / 2.0 + span_b = midpoint(span_window) + return conversion_line, base_line, span_a, span_b + + +def _create_cross_validator(validator_type, **kwargs): + """ + Build a cross-validator from the core registry (the 2.0.0 single source). + + Mirrors the old ``_create_cross_validator(...)`` behaviour, + including dropping keyword arguments a given validator does not accept. + """ + return CROSS_VALIDATORS.create(validator_type, filter_unknown_kwargs=True, **kwargs) + + +def financial_features_backtest_overfitting_simulation( + prices: pd.Series, noise_scale: float = 0.0, random_state: Optional[int] = None +) -> pd.DataFrame: + """ + Create a DataFrame of financial features from the given price series. + + Args: + prices (pd.Series): Time series of asset prices. + noise_scale (float): Scale of Gaussian noise to be added to the features. Default is 0.0. + random_state (Optional[int]): Seed for random number generator. Default is None. + + Returns: + pd.DataFrame: DataFrame containing the computed financial features. + + Notes: + RSI, ATR, and ADX use Wilder smoothing. CCI uses the standard 0.015 + scaling constant; stochastic momentum uses a 14-period range and a + 3-period signal; ROC uses 12 periods; and Ichimoku uses 9, 26, and 52 + periods without a visual displacement. These formulas are implemented + locally so this preserved API does not depend on an unmaintained + technical-analysis package. + """ + rng = np.random.default_rng(random_state) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + # Features + features = pd.DataFrame() + features["FracDiff"] = fractionally_differentiated_log_price(prices) + features["Volatility"] = daily_volatility_with_log_returns(prices, 100) + features["Z-Score"] = (prices - prices.rolling(20).mean()) / prices.rolling( + 20 + ).std() + macd_line = np.log(prices.ewm(span=12).mean() / prices.ewm(span=26).mean()) + signal_line = macd_line.ewm(span=9).mean() + features["Log MACD Histogram"] = macd_line - signal_line + features["ADX"] = _average_directional_index(prices) + features["RSI"] = _relative_strength_index(prices) + features["CCI"] = _commodity_channel_index(prices) + stochastic, stochastic_signal = _stochastic_oscillator(prices) + features["Stochastic"] = stochastic + features["ROC"] = prices.pct_change(12) * 100.0 + features["ATR"] = _average_true_range(prices) + features["Log DPO"] = np.log( + prices.rolling(11).mean() / prices.rolling(20).mean() + ) + + # 1. MACD Crossovers: + features["MACD Position"] = 0 # default to no crossover + features.loc[features["Log MACD Histogram"] >= 0, "MACD Position"] = 1 + features.loc[features["Log MACD Histogram"] < 0, "MACD Position"] = -1 + + # 2. ADX Trend Strength: + features["ADX Strength"] = 0 + features.loc[features["ADX"] > 25, "ADX Strength"] = 1 + features.loc[features["ADX"] < 25, "ADX Strength"] = -1 + + # 3. RSI Overbought/Oversold: + features["RSI Signal"] = 0 + features.loc[features["RSI"] > 70, "RSI Signal"] = 1 + features.loc[features["RSI"] < 30, "RSI Signal"] = -1 + + # 4. CCI Overbought/Oversold: + features["CCI Signal"] = 0 + features.loc[features["CCI"] > 100, "CCI Signal"] = 1 + features.loc[features["CCI"] < -100, "CCI Signal"] = -1 + + # 5. Stochastic Oscillator Overbought/Oversold: + features["Stochastic Signal"] = 0 + features.loc[stochastic_signal > 80, "Stochastic Signal"] = 1 + features.loc[stochastic_signal < 20, "Stochastic Signal"] = -1 + + # 6. ROC Momentum Shifts: + features["ROC Momentum"] = 0 + features.loc[features["ROC"] > 0, "ROC Momentum"] = 1 # Positive momentum + features.loc[features["ROC"] < 0, "ROC Momentum"] = -1 # Negative momentum + tenkan_sen, kijun_sen, senkou_span_a, senkou_span_b = _ichimoku_lines( + prices + ) + + # 1. Kumo Breakouts + features["Kumo Breakout"] = 0 + features.loc[ + (prices > senkou_span_a) & (prices > senkou_span_b), "Kumo Breakout" + ] = 1 + features.loc[ + (prices < senkou_span_a) & (prices < senkou_span_b), "Kumo Breakout" + ] = -1 + + # 2. TK Crosses + features["TK Position"] = 0 + features.loc[tenkan_sen >= kijun_sen, "TK Position"] = 1 + features.loc[tenkan_sen < kijun_sen, "TK Position"] = -1 + + # 4. Price Relative to Kumo + features["Price Kumo Position"] = 0 + features.loc[ + (prices > senkou_span_a) & (prices > senkou_span_b), "Price Kumo Position" + ] = 1 + features.loc[ + (prices < senkou_span_a) & (prices < senkou_span_b), "Price Kumo Position" + ] = -1 + + # 5. Cloud Thickness + features["Cloud Thickness"] = np.log(senkou_span_a / senkou_span_b) + + # 6. Momentum Confirmation + features["Momentum Confirmation"] = 0 + features.loc[ + (tenkan_sen > senkou_span_a) & (prices > senkou_span_a), + "Momentum Confirmation", + ] = 1 + features.loc[ + (tenkan_sen < senkou_span_a) & (prices < senkou_span_a), + "Momentum Confirmation", + ] = -1 + + if noise_scale > 0.0: + for col in features.columns: + noise = rng.normal( + loc=0, scale=noise_scale * features[col].std(), size=features[col].shape + ) + features[col] += noise + + return features + + +def backtest_overfitting_simulation_results( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + cross_validators: dict[str, Any], + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> dict[str, list[dict[str, Any]]]: + """ + Conducts a simulation to evaluate the performance of trading strategies and models. + + This function simulates a trading environment to assess various cross-validation methods in the context of financial analytics. It uses a set of market regime parameters and machine learning models to backtest trading strategies and compute metrics indicative of overfitting. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior, including window sizes and flags for mean reversion. + models (dict): A collection of machine learning models and their associated parameters. + cross_validators (dict): A dictionary of cross-validation methods. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Dict[str, List[Dict[str, Any]]]: A dictionary containing the results of the backtest for each cross-validation method tested. + """ + + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation( + prices, noise_scale=noise_scale, random_state=random_state + ) + + results = {cv: [] for cv in cross_validators.keys()} + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict(zip(strategy_parameters_keys, strategy_parameters_value)) + if ( + ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][0] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][0] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][1] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][1] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][2] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][2] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][3] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][3] + ) + ): + + strategy_sides = determine_strategy_side(prices, **strategy_params) + else: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + external_feature = volatility.loc[index] + + # Iterate over each model and hyperparameter configuration + for model_name, model_details in models.items(): + model = model_details["Model"] + param_grid = model_details["Parameters"] + + # Generate all combinations of hyperparameters + model_keys, model_values = zip(*param_grid.items()) + for model_value in itertools.product(*model_values): + params = dict(zip(model_keys, model_value)) + model.set_params(**params) + + for cross_validator_type, cross_validator in cross_validators.items(): + if "times" in cross_validator.__dict__: + cross_validator.times = times + if "external_feature" in cross_validator.__dict__: + cross_validator.external_feature = external_feature + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack( + list(map(lambda x: x[:, 1], predictions.values())) + ).mean(axis=0), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": params, + }, + "Returns": strategy_log_returns, + } + ) + + return results + + +def overall_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> tuple[dict[str, float], dict[str, float]]: + """ + Conducts an overall backtest overfitting simulation to calculate the metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, float], Dict[str, float]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, + strategy_parameters, + models, + cross_validators, + noise_scale, + random_state, + n_jobs=n_jobs, + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate + ) + cv_pbo[cv] = pbo + + return cv_pbo, cv_deflated_sr + + +def temporal_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + overfitting_partitions_length: int, + n_jobs: int = 1, +) -> tuple[dict[str, list[float]], dict[str, list[float]]]: + """ + Conducts a temporal backtest overfitting simulation to calculate the metrics in chunks. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + overfitting_partitions_length (int): The number of partitions to divide the dataset into for temporal overfitting analysis. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, List[float]], Dict[str, List[float]]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + cv_deflated_sr = {cv: [] for cv in results.keys()} + cv_pbo = {cv: [] for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Calculate the number of chunks using ceil + n_chunks = ceil(performances.shape[0] / overfitting_partitions_length) + for chunk in np.array_split(performances, n_chunks): + sharpe_ratios = chunk.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + + deflated_sr = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(chunk), + ss.skew(chunk[best_strategy_index]), + ss.kurtosis(chunk[best_strategy_index]), + return_test_statistic=True, + ) + cv_deflated_sr[cv].append(deflated_sr) + + pbo, logit_values = probability_of_backtest_overfitting( + chunk.values, risk_free_return=step_risk_free_rate, n_jobs=n_jobs + ) + cv_pbo[cv].append(pbo) + + return cv_pbo, cv_deflated_sr + + +def time_temporal_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + overfitting_partitions_duration: str = "A", # Annual grouping by default + n_jobs: int = 1, +) -> tuple[dict[str, pd.Series], dict[str, pd.Series]]: + """ + Conducts a time-temporal backtest overfitting simulation to calculate the metrics in time-indexed chunks. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + overfitting_partitions_duration (str): The frequency for time-based grouping to divide the dataset into for temporal overfitting analysis. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, pd.Series], Dict[str, pd.Series]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested, indexed by time. + """ + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + cv_deflated_sr = {cv: pd.Series(dtype=float) for cv in results.keys()} + cv_pbo = {cv: pd.Series(dtype=float) for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Group by the specified duration and calculate metrics + grouped = performances.groupby(pd.Grouper(freq=overfitting_partitions_duration)) + for timestamp, chunk in grouped: + sharpe_ratios = chunk.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + + deflated_sr = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(chunk), + ss.skew(chunk[best_strategy_index]), + ss.kurtosis(chunk[best_strategy_index]), + return_test_statistic=True, + ) + cv_deflated_sr[cv].at[timestamp] = deflated_sr + + pbo, logit_values = probability_of_backtest_overfitting( + chunk.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv].at[timestamp] = pbo + + return cv_pbo, cv_deflated_sr + + +def varying_embargo_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + embargo_values: list[float], + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a backtest overfitting simulation with varying embargo values to calculate the metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + embargo_values (List[float]): List of embargo values to test. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each embargo value and cross-validation method. + """ + + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation(prices) + + cv_pbo_embargo = pd.DataFrame( + index=embargo_values, columns=["Purged K-Fold", "Combinatorial Purged"] + ) + cv_deflated_sr_embargo = pd.DataFrame( + index=embargo_values, columns=["Purged K-Fold", "Combinatorial Purged"] + ) + + for embargo in embargo_values: + results = { + "Purged K-Fold": [], + "Combinatorial Purged": [], + } + + cross_validators = { + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=embargo + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=embargo, + ), + } + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict( + zip(strategy_parameters_keys, strategy_parameters_value) + ) + if ( + ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][0] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][0] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][1] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][1] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][2] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][2] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][3] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][3] + ) + ): + + strategy_sides = determine_strategy_side(prices, **strategy_params) + else: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + + # Iterate over each model and hyperparameter configuration + for model_name, model_details in models.items(): + model = model_details["Model"] + param_grid = model_details["Parameters"] + + # Generate all combinations of hyperparameters + model_keys, model_values = zip(*param_grid.items()) + for model_value in itertools.product(*model_values): + params = dict(zip(model_keys, model_value)) + model.set_params(**params) + + for ( + cross_validator_type, + cross_validator, + ) in cross_validators.items(): + if "times" in cross_validator.__dict__: + cross_validator.times = times + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack( + list(map(lambda x: x[:, 1], predictions.values())) + ).mean(axis=0), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": params, + }, + "Returns": strategy_log_returns, + } + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv] = pbo + + for cv in results.keys(): + cv_pbo_embargo.loc[embargo, cv] = cv_pbo[cv] + cv_deflated_sr_embargo.loc[embargo, cv] = cv_deflated_sr[cv] + + return cv_pbo_embargo, cv_deflated_sr_embargo + + +def sharpe_ratio(returns, risk_free_rate=0): + """Calculate the Sharpe ratio of the given returns.""" + std = returns.std() + if std == 0: + return 0.0 + return (returns.mean() - risk_free_rate) / std + + +def sortino_ratio(returns, risk_free_rate=0): + """Calculate the Sortino ratio of the given returns.""" + downside_returns = returns[returns < risk_free_rate] + expected_return = returns.mean() - risk_free_rate + + if downside_returns.empty: + return np.inf if expected_return > 0 else 0.0 + + downside_risk = downside_returns.std() + + if downside_risk == 0: + return np.inf if expected_return > 0 else 0.0 + + return expected_return / downside_risk + + +def expected_shortfall(returns, step_risk_free_rate, confidence_level=0.05): + """Calculate the expected shortfall (conditional VaR) of the given returns.""" + var = np.percentile(returns, 100 * confidence_level) + es = returns[returns <= var].mean() + return es + + +def backtest_overfitting_simulation_financial_metrics_rank_correlation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + n_jobs: int = 1, +) -> pd.DataFrame: + """ + Conducts a backtest overfitting simulation and calculates the rank correlation of financial metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + pd.DataFrame: DataFrame containing the rank correlations for each cross-validation method and each metric. + """ + # Run the backtest overfitting simulation to get results + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + metrics = { + "Sharpe Ratio": sharpe_ratio, + "Sortino Ratio": sortino_ratio, + "Expected Shortfall": expected_shortfall, + } + + rank_correlations = {cv: {} for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Split the data into two halves + midpoint = len(performances) // 2 + first_half = performances.iloc[:midpoint] + second_half = performances.iloc[midpoint:] + + # Calculate the metrics for each half + first_half_metrics = { + metric_name: first_half.apply( + lambda x, fn=metric_func: fn(x, step_risk_free_rate), axis=0 + ) + for metric_name, metric_func in metrics.items() + } + second_half_metrics = { + metric_name: second_half.apply( + lambda x, fn=metric_func: fn(x, step_risk_free_rate), axis=0 + ) + for metric_name, metric_func in metrics.items() + } + + # Rank the trials in each half + first_half_ranks = { + metric_name: metrics_values.rank() + for metric_name, metrics_values in first_half_metrics.items() + } + second_half_ranks = { + metric_name: metrics_values.rank() + for metric_name, metrics_values in second_half_metrics.items() + } + + # Calculate the rank correlation for each metric + for metric_name in metrics.keys(): + rank_corr, _ = kendalltau( + first_half_ranks[metric_name], second_half_ranks[metric_name] + ) + rank_correlations[cv][metric_name] = rank_corr + + # Create the final DataFrame + rank_corr_df = pd.DataFrame(rank_correlations) + + return rank_corr_df + + +def backtest_overfitting_simulation_model_complexity( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, Any], + step_risk_free_rate: float, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a backtest overfitting simulation to compare the PBO and DSR values of each CV method for simple and complex models. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each model and each CV method. + """ + # Initialize DataFrames to store results + pbo_df = pd.DataFrame(columns=["Combinatorial Purged"], index=models.keys()) + dsr_df = pd.DataFrame(columns=["Combinatorial Purged"], index=models.keys()) + + # Create features + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation(prices) + + for model_name, model in models.items(): + results = { + "Combinatorial Purged": [], + } + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict( + zip(strategy_parameters_keys, strategy_parameters_value) + ) + try: + strategy_sides = determine_strategy_side(prices, **strategy_params) + + except ValueError: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + + cross_validators = { + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=times, + embargo=0.02, + ), + } + + for cross_validator_type, cross_validator in cross_validators.items(): + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack(list(map(lambda x: x[:, 1], predictions.values()))).mean( + axis=0 + ), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": {}, + }, + "Returns": strategy_log_returns, + } + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv] = pbo + + for cv in results.keys(): + pbo_df.loc[model_name, cv] = cv_pbo[cv] + dsr_df.loc[model_name, cv] = cv_deflated_sr[cv] + + return pbo_df, dsr_df + + +def noised_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scales: list[float], + random_state: int = None, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a noised backtest overfitting simulation to compare the new PBO/DSR values for different noise scales. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scales (List[float]): List of noise scale values to test. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each noise scale and each CV method. + """ + # Initialize DataFrames to store results + pbo_df = pd.DataFrame( + columns=["Walk-Forward", "K-Fold", "Purged K-Fold", "Combinatorial Purged"], + index=noise_scales, + ) + dsr_df = pd.DataFrame( + columns=["Walk-Forward", "K-Fold", "Purged K-Fold", "Combinatorial Purged"], + index=noise_scales, + ) + + for noise_scale in noise_scales: + # Perform the overall backtest overfitting simulation with the given noise scale + cv_pbo, cv_deflated_sr = overall_backtest_overfitting_simulation( + prices=prices, + strategy_parameters=strategy_parameters, + models=models, + step_risk_free_rate=step_risk_free_rate, + noise_scale=noise_scale, + random_state=random_state, + n_jobs=n_jobs, + ) + + # Store the results in the DataFrames + for cv in cv_pbo.keys(): + pbo_df.loc[noise_scale, cv] = cv_pbo[cv] + dsr_df.loc[noise_scale, cv] = cv_deflated_sr[cv] + + return pbo_df, dsr_df + + +def overall_novel_methods_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> tuple[dict[str, float], dict[str, float]]: + """ + Conducts an overall backtest overfitting simulation to calculate the metrics for the novel CPCV methods. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, float], Dict[str, float]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + + cross_validators = { + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + "Bagged Combinatorial Purged": _create_cross_validator( + "baggedcombinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=0.02, + random_state=random_state, + ), + "Adaptive Combinatorial Purged": _create_cross_validator( + "adaptivecombinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=0.02, + ), + } + + results = backtest_overfitting_simulation_results( + prices, + strategy_parameters, + models, + cross_validators, + noise_scale, + random_state, + n_jobs=n_jobs, + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate + ) + cv_pbo[cv] = pbo + + return cv_pbo, cv_deflated_sr + + +def get_cpu_info(): + # Run the lscpu command + result = subprocess.run(["lscpu"], stdout=subprocess.PIPE) + # Decode the output from bytes to string + lscpu_output = result.stdout.decode("utf-8") + + # Parse the lscpu output + cpu_info = {} + for line in lscpu_output.split("\n"): + if line.strip(): + parts = line.split(":", 1) + if len(parts) == 2: + key, value = parts + cpu_info[key.strip()] = value.strip() + + # Extract useful information + useful_info = { + "Architecture": cpu_info.get("Architecture"), + "CPU op-mode(s)": cpu_info.get("CPU op-mode(s)"), + "CPU(s)": cpu_info.get("CPU(s)"), + "Thread(s) per core": cpu_info.get("Thread(s) per core"), + "Core(s) per socket": cpu_info.get("Core(s) per socket"), + "Socket(s)": cpu_info.get("Socket(s)"), + "Vendor ID": cpu_info.get("Vendor ID"), + "Model name": cpu_info.get("Model name"), + "CPU MHz": cpu_info.get("CPU MHz"), + "L1d cache": cpu_info.get("L1d cache"), + "L1i cache": cpu_info.get("L1i cache"), + "L2 cache": cpu_info.get("L2 cache"), + "L3 cache": cpu_info.get("L3 cache"), + } + + return useful_info + + +def format_cpu_info(cpu_info): + report = ( + f"Architecture: {cpu_info['Architecture']}\n" + f"CPU Operational Modes: {cpu_info['CPU op-mode(s)']}\n" + f"Total CPUs: {cpu_info['CPU(s)']}\n" + f"Threads per Core: {cpu_info['Thread(s) per core']}\n" + f"Cores per Socket: {cpu_info['Core(s) per socket']}\n" + f"Sockets: {cpu_info['Socket(s)']}\n" + f"Vendor ID: {cpu_info['Vendor ID']}\n" + f"Model Name: {cpu_info['Model name']}\n" + f"CPU MHz: {cpu_info['CPU MHz']}\n" + f"L1d Cache: {cpu_info['L1d cache']}\n" + f"L1i Cache: {cpu_info['L1i cache']}\n" + f"L2 Cache: {cpu_info['L2 cache']}\n" + f"L3 Cache: {cpu_info['L3 cache']}\n" + ) + return report + + +# Function to generate random data, target, weights, and times +def generate_random_data( + n_samples: int, n_features: int +) -> tuple[pd.DataFrame, pd.Series, np.ndarray, pd.Series]: + date_range = pd.date_range(start="1980-01-01", periods=n_samples, freq="1h") + data = pd.DataFrame( + np.random.randn(n_samples, n_features), + columns=[f"feature_{i}" for i in range(n_features)], + index=date_range, + ) + target = pd.Series(np.random.randint(0, 2, n_samples), index=date_range) + weights = pd.Series(np.random.rand(n_samples), index=date_range) + weights = weights / weights.sum() + times = pd.Series(date_range + pd.DateOffset(hours=3), index=date_range) + return data, target, weights, times + + +# Function to measure computational requirements +def measure_computational_requirements( + cross_validator, model, data, target, weights, n_jobs: int = 1 +) -> dict[str, Any]: + from memory_profiler import memory_usage # optional dependency: RiskLabAI[dev] + + start_time = time.time() + mem_usage = memory_usage( + ( + cross_validator.backtest_predictions, + (model, data, target, weights), + {"predict_probability": True, "n_jobs": n_jobs}, + ), + interval=0.1, + ) + end_time = time.time() + return {"execution_time": end_time - start_time, "memory_usage": max(mem_usage)} + + +# Main function to measure computational requirements for all CV methods +def measure_all_cv_computational_requirements( + cross_validators: dict[str, Any], + n_samples: int = 40 * 252, + n_features: int = 22, + n_jobs: int = 1, + n_repeats: int = 30, +) -> pd.DataFrame: + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + # Measure computational requirements for each CV method + results = { + cv_name: {"execution_time": [], "memory_usage": []} + for cv_name in cross_validators.keys() + } + + for _ in _progress(range(n_repeats)): + for cv_name, cross_validator in cross_validators.items(): + # print(f"Measuring computational requirements for {cv_name} (repeat {_ + 1}/{n_repeats})...") + # Update cross-validator with times if required + if "times" in cross_validator.__dict__: + cross_validator.times = times + result = measure_computational_requirements( + cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + results[cv_name]["execution_time"].append(result["execution_time"]) + results[cv_name]["memory_usage"].append(result["memory_usage"]) + + # Calculate mean and standard deviation for each CV method + results_summary = {} + for cv_name, metrics in results.items(): + results_summary[cv_name] = { + "execution_time_mean": np.mean(metrics["execution_time"]), + "execution_time_std": np.std(metrics["execution_time"]), + "memory_usage_mean": np.mean(metrics["memory_usage"]), + "memory_usage_std": np.std(metrics["memory_usage"]), + } + + # Convert results to DataFrame + results_df = pd.DataFrame(results_summary).T + + return results_df + + +_DEFAULT_N_JOBS_LIST = list(range(1, 9)) + + +def measure_cpcv_parallelization( + n_samples: int = 40 * 252, + n_features: int = 22, + n_repeats: int = 30, + n_jobs_list: list[int] = _DEFAULT_N_JOBS_LIST, +) -> pd.DataFrame: + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + # Define the CPCV cross-validator + cpcv_cross_validator = _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=times, embargo=0.02 + ) + + # Measure computational requirements with and without parallelization + results = {} + + for n_jobs in _progress(n_jobs_list): + key = f"n_jobs_{n_jobs}" + results[key] = {"execution_time": [], "memory_usage": []} + for _ in range(n_repeats): + result = measure_computational_requirements( + cpcv_cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + results[key]["execution_time"].append(result["execution_time"]) + results[key]["memory_usage"].append(result["memory_usage"]) + + # Calculate mean and standard deviation for each setting + results_summary = {} + for key, metrics in results.items(): + results_summary[key] = { + "execution_time_mean": np.mean(metrics["execution_time"]), + "execution_time_std": np.std(metrics["execution_time"]), + "memory_usage_mean": np.mean(metrics["memory_usage"]), + "memory_usage_std": np.std(metrics["memory_usage"]), + } + + # Convert results to DataFrame + results_df = pd.DataFrame(results_summary).T + + return results_df + + +def measure_cpcv_scalability( + sample_sizes: list[int], + feature_sizes: list[int], + n_repeats: int = 1, + n_jobs: int = 1, +) -> pd.DataFrame: + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + execution_times = pd.DataFrame(index=sample_sizes, columns=feature_sizes) + memory_usages = pd.DataFrame(index=sample_sizes, columns=feature_sizes) + + for n_samples, n_features in itertools.product(sample_sizes, feature_sizes): + execution_time_list = [] + memory_usage_list = [] + + for _ in range(n_repeats): + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the CPCV cross-validator + cpcv_cross_validator = _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=times, + embargo=0.02, + ) + + # Measure computational requirements + result = measure_computational_requirements( + cpcv_cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + execution_time_list.append(result["execution_time"]) + memory_usage_list.append(result["memory_usage"]) + + # Calculate mean execution time and memory usage + execution_times.loc[n_samples, n_features] = np.mean(execution_time_list) + memory_usages.loc[n_samples, n_features] = np.mean(memory_usage_list) + + return execution_times, memory_usages diff --git a/src/RiskLabAI/backtest/backtest_statistics.py b/src/RiskLabAI/backtest/backtest_statistics.py new file mode 100644 index 0000000..e7ce374 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_statistics.py @@ -0,0 +1,300 @@ +""" +Calculates various backtest statistics like holding period, +concentration, and drawdowns. + +## TODO: +- [ ] Optimize `calculate_holding_period` with Numba (@jit) + to improve performance, similar to the `sharpe_ratio` function. +- [ ] Add an `annualized_sharpe_ratio` helper function that + wraps the Numba `sharpe_ratio` and scales it by sqrt(freq). +""" + +import numpy as np +import pandas as pd + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + + +@jit(nopython=True) +def sharpe_ratio(returns: np.ndarray, risk_free_rate: float = 0.0) -> float: + """ + Calculate the Sharpe Ratio (Numba-optimized). + + Parameters + ---------- + returns : np.ndarray + An array of returns. + risk_free_rate : float, default=0.0 + The risk-free rate. + + Returns + ------- + float + The calculated Sharpe Ratio. + """ + excess_returns = returns - risk_free_rate + std_dev = np.std(excess_returns) + + if std_dev == 0.0: + return 0.0 + + return np.mean(excess_returns) / std_dev + + +def bet_timing(target_positions: pd.Series) -> pd.Index: + """ + Determine the timestamps of bets, defined as when positions + are closed (return to zero) or flipped (sign change). + + Parameters + ---------- + target_positions : pd.Series + Time series of target positions. + + Returns + ------- + pd.Index + A DatetimeIndex of timestamps when bets are timed. + + Example + ------- + >>> dates = pd.to_datetime(['2020-01-01', '2020-01-02', '2020-01-03', \ + '2020-01-04', '2020-01-05', '2020-01-06']) + >>> positions = pd.Series([0, 1, 1, 0, -1, 0], index=dates) + >>> bet_timing(positions) + DatetimeIndex(['2020-01-04', '2020-01-05', '2020-01-06'], \ +dtype='datetime64[ns]', freq=None) + """ + # 1. Bets at points where position returns to 0 + zero_positions = target_positions[target_positions == 0].index + + # 2. Bets at points where position flips sign + # Find timestamps where t=0 and t-1 != 0 + lagged_non_zero = target_positions.shift(1).fillna(0) + lagged_non_zero = lagged_non_zero[lagged_non_zero != 0].index + bets = zero_positions.intersection(lagged_non_zero) + + # Find timestamps where sign flips, e.g., 1 to -1 + sign_flips = target_positions.iloc[1:] * target_positions.iloc[:-1].values + bets = bets.union(sign_flips[sign_flips < 0].index).sort_values() + + # 3. Add last timestamp if not already included + if target_positions.index[-1] not in bets: + bets = bets.append(pd.Index([target_positions.index[-1]])) + + return bets + + +def calculate_holding_period( + target_positions: pd.Series, +) -> tuple[pd.DataFrame, float]: + """ + Derive the average holding period in days. + + Uses the average entry time pairing algorithm. + + Parameters + ---------- + target_positions : pd.Series + Time series of target positions. + + Returns + ------- + Tuple[pd.DataFrame, float] + - hold_period_df: DataFrame with holding time ('dT') and + weight ('w') for each bet. + - mean_holding_period: The weighted-average holding period in days. + """ + hold_period_data = [] + time_entry = 0.0 # Average entry time + position_diff = target_positions.diff() + # Time difference in fractional days + time_diff = (target_positions.index - target_positions.index[0]) / np.timedelta64( + 1, "D" + ) + + for i in range(1, target_positions.shape[0]): + current_pos = target_positions.iloc[i] + prev_pos = target_positions.iloc[i - 1] + diff = position_diff.iloc[i] + + if diff * prev_pos >= 0: # Position increase or flat + if current_pos != 0: + # Update average entry time + time_entry = (time_entry * prev_pos + time_diff[i] * diff) / current_pos + else: # Position decrease or flip + if current_pos * prev_pos < 0: # Position flip + # Close old position + hold_period_data.append( + { + "index": target_positions.index[i], + "dT": time_diff[i] - time_entry, + "w": abs(prev_pos), + } + ) + time_entry = time_diff[i] # Reset entry time + else: # Position decrease (but not flip) + hold_period_data.append( + { + "index": target_positions.index[i], + "dT": time_diff[i] - time_entry, + "w": abs(diff), + } + ) + + if not hold_period_data: + return pd.DataFrame(columns=["dT", "w"]), np.nan + + hold_period_df = pd.DataFrame(hold_period_data).set_index("index") + + if hold_period_df["w"].sum() > 0: + mean_holding_period = ( + hold_period_df["dT"] * hold_period_df["w"] + ).sum() / hold_period_df["w"].sum() + else: + mean_holding_period = np.nan + + return hold_period_df, mean_holding_period + + +def calculate_hhi(bet_returns: pd.Series) -> float: + """ + Calculate the Herfindahl-Hirschman Index (HHI) for concentration. + + HHI is normalized to be between 0 and 1. + - 0 indicates a perfectly diversified portfolio. + - 1 indicates a fully concentrated portfolio. + + Parameters + ---------- + bet_returns : pd.Series + Series of returns for each bet. + + Returns + ------- + float + The normalized HHI value, or np.nan if undefined. + """ + if bet_returns.shape[0] <= 2: + return np.nan + + total_return = bet_returns.sum() + if total_return == 0: + return np.nan + + weights = bet_returns / total_return + hhi = (weights**2).sum() + + # Normalize HHI + n = bet_returns.shape[0] + hhi_normalized = (hhi - 1.0 / n) / (1.0 - 1.0 / n) + + return hhi_normalized + + +def calculate_hhi_concentration(returns: pd.Series) -> tuple[float, float, float]: + """ + Calculate HHI concentration for positive, negative, and monthly returns. + + Parameters + ---------- + returns : pd.Series + Time series of returns. + + Returns + ------- + Tuple[float, float, float] + - hhi_positive: HHI for positive returns. + - hhi_negative: HHI for negative returns. + - hhi_time: HHI for time-based (monthly) concentration. + """ + hhi_positive = calculate_hhi(returns[returns >= 0]) + hhi_negative = calculate_hhi(returns[returns < 0]) + + # Calculate time concentration (by month) + time_concentration = returns.groupby(pd.Grouper(freq="M")).count() + hhi_time = calculate_hhi(time_concentration) + + return hhi_positive, hhi_negative, hhi_time + + +def compute_drawdowns_time_under_water( + pnl_series: pd.Series, dollars: bool = False +) -> tuple[pd.Series, pd.Series]: + """ + Compute series of drawdowns and the time under water. + + Parameters + ---------- + pnl_series : pd.Series + Time series of returns or dollar PnL. + dollars : bool, default=False + If True, treat `pnl_series` as dollar PnL. + If False, treat as returns (e.g., 1.01, 0.99) and + calculate drawdowns as percentages. + + Returns + ------- + Tuple[pd.Series, pd.Series] + - drawdown: pd.Series indexed by start date, showing max drawdown. + - time_under_water: pd.Series indexed by start date, showing + duration of drawdown in fractional years. + """ + series_df = pnl_series.to_frame("PnL") + series_df["HWM"] = series_df["PnL"].expanding().max() + + # Group by High Water Mark (HWM) + groups = series_df.groupby("HWM") + drawdown_analysis_data = [] + + for hwm, group in groups: + if len(group) <= 1 or hwm == group["PnL"].min(): + continue # Skip groups with no drawdown + + drawdown_analysis_data.append( + { + "Start": group.index[0], + "Stop": group.index[-1], + "HWM": hwm, + "Min": group["PnL"].min(), + } + ) + + if not drawdown_analysis_data: + return pd.Series(dtype=float), pd.Series(dtype=float) + + drawdown_analysis_df = pd.DataFrame(drawdown_analysis_data) + drawdown_analysis_df = drawdown_analysis_df.set_index("Start") + + if dollars: + drawdown = drawdown_analysis_df["HWM"] - drawdown_analysis_df["Min"] + else: + # Percentage drawdown + drawdown = 1.0 - (drawdown_analysis_df["Min"] / drawdown_analysis_df["HWM"]) + + # Time under water in fractional years + time_under_water = ( + (drawdown_analysis_df["Stop"] - drawdown_analysis_df.index) + / np.timedelta64(1, "D") + / 365.25 + ) + + drawdown.index.name = "Datetime" + time_under_water.index.name = "Datetime" + + return drawdown, time_under_water diff --git a/src/RiskLabAI/backtest/backtest_synthetic_data.py b/src/RiskLabAI/backtest/backtest_synthetic_data.py new file mode 100644 index 0000000..c60abb1 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_synthetic_data.py @@ -0,0 +1,99 @@ +""" +Performs backtesting on synthetic price data generated using the +Ornstein-Uhlenbeck (OU) process. +""" + +from itertools import product +from random import gauss + +import numpy as np + +_DEFAULT_PROFIT_TAKING_RANGE = np.linspace(0.5, 10, 20) +_DEFAULT_STOP_LOSS_RANGE = np.linspace(0.5, 10, 20) + + +def synthetic_back_testing( + forecast: float, + half_life: float, + sigma: float, + n_iteration: int = 100000, + maximum_holding_period: int = 100, + profit_taking_range: np.ndarray = _DEFAULT_PROFIT_TAKING_RANGE, + stop_loss_range: np.ndarray = _DEFAULT_STOP_LOSS_RANGE, + seed: int = 0, +) -> list[tuple[float, float, float, float, float]]: + r""" + Perform backtesting on synthetic price data from an OU process. + + This function simulates multiple price paths based on the + Ornstein-Uhlenbeck (OU) process and tests a grid of + profit-taking (PT) and stop-loss (SL) levels. + + The OU process is given by: + .. math:: + P_t = (1 - \rho) F + \rho P_{t-1} + \sigma Z_t + + Where: + - \(P_t\) is the price at time t + - \(F\) is the forecast price (long-term mean) + - \(\rho\) is the autoregression coefficient + - \(\sigma\) is the standard deviation of noise + - \(Z_t \sim N(0, 1)\) is standard normal noise + + Parameters + ---------- + forecast : float + The forecasted price (long-term mean F). + half_life : float + The half-life of the mean-reversion process. + sigma : float + The standard deviation of the noise (\(\sigma\)). + n_iteration : int, default=100000 + Number of price paths to simulate for each PT/SL combination. + maximum_holding_period : int, default=100 + Maximum number of steps to hold the position. + profit_taking_range : np.ndarray, default=np.linspace(0.5, 10, 20) + An array of profit-taking levels to test. + stop_loss_range : np.ndarray, default=np.linspace(0.5, 10, 20) + An array of stop-loss levels to test. + seed : int, default=0 + The initial price for all simulations. + + Returns + ------- + List[Tuple[float, float, float, float, float]] + A list of tuples. Each tuple contains: + (profit_taking, stop_loss, mean_return, std_return, sharpe_ratio) + """ + # compute ρ coefficient from half-life + rho = 2 ** (-1.0 / half_life) + back_test_results = [] + + for profit_taking, stop_loss in product(profit_taking_range, stop_loss_range): + stop_returns = [] + for _ in range(n_iteration): + price, holding_period = float(seed), 0 # initial price + while True: + # Update price using O-U process + price = (1 - rho) * forecast + rho * price + sigma * gauss(0, 1) + gain = price - seed # compute gain from initial seed price + holding_period += 1 + + # Check stop conditions + if ( + gain > profit_taking + or gain < -stop_loss + or holding_period > maximum_holding_period + ): + stop_returns.append(gain) + break + + mean_return = np.mean(stop_returns) + std_return = np.std(stop_returns) + sharpe_ratio = mean_return / std_return if std_return > 0 else 0.0 + + back_test_results.append( + (profit_taking, stop_loss, mean_return, std_return, sharpe_ratio) + ) + + return back_test_results diff --git a/src/RiskLabAI/backtest/bet_sizing.py b/src/RiskLabAI/backtest/bet_sizing.py new file mode 100644 index 0000000..2badeea --- /dev/null +++ b/src/RiskLabAI/backtest/bet_sizing.py @@ -0,0 +1,509 @@ +""" +Functions for calculating bet size based on model probabilities and +other strategy parameters. + +Includes implementations from de Prado (2018). +""" + +import numpy as np +import pandas as pd +from scipy.stats import norm + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + +from RiskLabAI._deprecation import deprecated_alias +from RiskLabAI.hpc import mp_pandas_obj + + +def probability_bet_size(probabilities: np.ndarray, sides: np.ndarray) -> np.ndarray: + r""" + Calculate the bet size based on probabilities and side. + + The bet size is derived from the probability of a correct bet, + transformed from [0, 1] to [-1, 1]. + + .. math:: + \text{bet_size} = \text{side} \times (2 \times \Phi(p) - 1) + + Where \(\Phi\) is the CDF of a standard normal distribution and + \(p\) is the probability from the meta-model. + + Parameters + ---------- + probabilities : np.ndarray + Array of probabilities (e.g., from a meta-model). + sides : np.ndarray + Array indicating the side of the bet (e.g., 1 for long, -1 for short). + + Returns + ------- + np.ndarray + Array of bet sizes, ranging from -1 to 1. + """ + return sides * (2 * norm.cdf(probabilities) - 1) + + +@jit(nopython=True) +def average_bet_sizes( + price_dates: np.ndarray, + start_dates: np.ndarray, + end_dates: np.ndarray, + bet_sizes: np.ndarray, +) -> np.ndarray: + """ + Compute average bet sizes for each date (Numba-optimized). + + This function calculates the concurrent average bet size for each + timestamp in `price_dates`. A bet is active at `price_dates[i]` if + `start_dates[j] <= price_dates[i] <= end_dates[j]`. + + Parameters + ---------- + price_dates : np.ndarray + Array of timestamps for which to calculate the average bet size. + Must be numerical (e.g., int64 from .to_numpy()). + start_dates : np.ndarray + Array of bet start timestamps (numerical). + end_dates : np.ndarray + Array of bet end timestamps (numerical). + bet_sizes : np.ndarray + Array of bet sizes corresponding to each (start, end) pair. + + Returns + ------- + np.ndarray + Array of average bet sizes for each date in `price_dates`. + """ + num_dates = len(price_dates) + avg_bet_sizes = np.zeros(num_dates) + + for i in range(num_dates): + total = 0.0 + count = 0 + for j in range(len(start_dates)): + if start_dates[j] <= price_dates[i] <= end_dates[j]: + total += bet_sizes[j] + count += 1 + if count > 0: + avg_bet_sizes[i] = total / count + + return avg_bet_sizes + + +def strategy_bet_sizing( + price_timestamps: pd.Index, + times: pd.Series, + sides: pd.Series, + probabilities: pd.Series, +) -> pd.Series: + """ + Calculate the average bet size for a trading strategy. + + This function orchestrates the calculation of bet sizes from + probabilities and then computes the concurrent average bet size + over the strategy's history. + + Parameters + ---------- + price_timestamps : pd.Index + The Index of the master price series, defining all timestamps + for the output. + times : pd.Series + Series with bet start times as indices and end times as values. + sides : pd.Series + Series indicating the side of the position (1 or -1). + probabilities : pd.Series + Series of probabilities (from meta-model) for each bet. + + Returns + ------- + pd.Series + Series of average bet sizes, indexed by `price_timestamps`. + """ + # Align inputs + common_index = times.index.intersection(sides.index).intersection( + probabilities.index + ) + if len(common_index) == 0: + return pd.Series(0.0, index=price_timestamps) + + _times = times.loc[common_index] + _sides = sides.loc[common_index] + _probabilities = probabilities.loc[common_index] + + # 1. Calculate individual bet sizes + bet_sizes_arr = probability_bet_size(_probabilities.to_numpy(), _sides.to_numpy()) + + # 2. Calculate concurrent average + avg_bet_sizes_arr = average_bet_sizes( + price_timestamps.to_numpy(), + _times.index.to_numpy(), + _times.values, + bet_sizes_arr, + ) + + return pd.Series(avg_bet_sizes_arr, index=price_timestamps) + + +# --- Bet-sizing functions from de Prado (2018), AFML Chapter 10. --- + + +def avg_active_signals(signals: pd.DataFrame, n_threads: int) -> pd.DataFrame: + """ + Calculate the average signal among active signals using parallel processing. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.2 + + Parameters + ---------- + signals : pd.DataFrame + DataFrame with signal start times as index, and columns 't1' (end time) + and 'signal' (signal value). + n_threads : int + Number of threads to use for parallel execution via `mp_pandas_obj`. + + Returns + ------- + pd.DataFrame + DataFrame containing the average active signal at each time point. + """ + # 1) time points where signals change (either one starts or one ends) + time_points = set(signals["t1"].dropna().values) + time_points = time_points.union(signals.index.values) + time_points = list(time_points) + time_points.sort() + + # 2) call parallel function + out = mp_pandas_obj( + mp_avg_active_signals, + ("molecule", time_points), + n_threads, + signals=signals, + ) + return out + + +def mp_avg_active_signals(signals: pd.DataFrame, molecule: list) -> pd.Series: + """ + Worker function for `avg_active_signals`. + + At time `loc`, average signal among those still active. + Signal is active if: + a) issued before or at `loc` AND + b) `loc` before signal's endtime, or endtime is still unknown (NaT). + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.2 + + Parameters + ---------- + signals : pd.DataFrame + DataFrame of signals (see `avgActiveSignals`). + molecule : list + The list of timestamps (timePoints) assigned to this worker. + + Returns + ------- + pd.Series + Series indexed by `loc` (timestamp) with the average signal value. + """ + # A signal i is active at `loc` iff start_i <= loc < t1_i. The average of + # active signals therefore equals (sum of `signal` over signals started by + # `loc`) minus (sum over signals ended by `loc`), divided by the analogous + # active count. Prefix sums + searchsorted compute this for every `loc` in + # O((n + m) log n) instead of the O(n * m) double scan, with identical + # values (verified in test/test_performance.py). + molecule_list = list(molecule) + if len(signals) == 0 or len(molecule_list) == 0: + return pd.Series(0.0, index=molecule_list) + + molecule_array = np.asarray(molecule_list) + signal_values = signals["signal"].to_numpy(dtype=float) + + starts = signals.index.values + start_order = np.argsort(starts, kind="mergesort") + sorted_starts = starts[start_order] + cum_signal_start = np.concatenate(([0.0], np.cumsum(signal_values[start_order]))) + cum_count_start = np.arange(len(sorted_starts) + 1) + + finite = ~pd.isnull(signals["t1"]).to_numpy() + end_times = signals["t1"].to_numpy()[finite] + end_order = np.argsort(end_times, kind="mergesort") + sorted_ends = end_times[end_order] + cum_signal_end = np.concatenate( + ([0.0], np.cumsum(signal_values[finite][end_order])) + ) + cum_count_end = np.arange(len(sorted_ends) + 1) + + started = np.searchsorted(sorted_starts, molecule_array, side="right") + ended = np.searchsorted(sorted_ends, molecule_array, side="right") + active_signal = cum_signal_start[started] - cum_signal_end[ended] + active_count = cum_count_start[started] - cum_count_end[ended] + + with np.errstate(invalid="ignore", divide="ignore"): + averages = np.where(active_count > 0, active_signal / active_count, 0.0) + + return pd.Series(averages, index=molecule_list) + + +def discrete_signal(signal: pd.Series, step_size: float) -> pd.Series: + """ + Discretize a signal to a specific step size, capping at +/- 1. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.3 + + Parameters + ---------- + signal : pd.Series + The continuous signal values (e.g., from -1 to 1). + step_size : float + The step size for discretization (e.g., 0.1). + + Returns + ------- + pd.Series + The discretized signal. + """ + discretized = (signal / step_size).round() * step_size + discretized[discretized > 1] = 1.0 + discretized[discretized < -1] = -1.0 + return discretized + + +def generate_signal( + events: pd.DataFrame, + step_size: float, + probability: pd.Series, + prediction: pd.Series, + n_classes: int, + n_threads: int, +) -> pd.Series: + """ + Generate a discretized, averaged signal from model predictions. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.1 + + Parameters + ---------- + events : pd.DataFrame + DataFrame of events, must include 't1' (end times) and + optionally 'side' (for meta-labeling). + step_size : float + The step size for final signal discretization. + probability : pd.Series + Probability of class 1 (e.g., from `predict_proba`). + prediction : pd.Series + The predicted class (e.g., 1 or -1). + n_classes : int + Number of classes in the prediction. + n_threads : int + Number of threads for `avg_active_signals`. + + Returns + ------- + pd.Series + The final, averaged, and discretized signal. + """ + if probability.shape[0] == 0: + return pd.Series() + + # 1) generate signals from multinomial classification (one-vs-rest, OvR) + # t-value of OvR + t_value = (probability - 1.0 / n_classes) / ( + (probability * (1.0 - probability)) ** 0.5 + ) + signal = prediction * (2 * norm.cdf(t_value) - 1) # signal = side * size + + if "side" in events: + signal *= events.loc[signal.index, "side"] # meta-labeling + + # 2) compute average signal among those concurrently open + signal_df = signal.to_frame("signal").join(events[["t1"]], how="left") + avg_signal = avg_active_signals(signal_df, n_threads) + + # 3) discretize signal + discretized_signal = discrete_signal(signal=avg_signal, step_size=step_size) + return discretized_signal + + +def bet_size_sigmoid(w: float, x: float) -> float: + """ + Calculate bet size using a sigmoid function. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + w : float + Coefficient regulating the sigmoid width. + x : float + The divergence (e.g., forecast_price - market_price). + + Returns + ------- + float + The bet size, bounded between -1 and 1. + """ + return x / np.sqrt(w + x**2) + + +def target_position( + w: float, f: float, actual_price: float, maximum_position_size: int +) -> int: + """ + Calculate the target position size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + w : float + Coefficient regulating the sigmoid width. + f : float + Forecasted price. + actual_price : float + Actual (current) market price. + maximum_position_size : int + Maximum absolute position size. + + Returns + ------- + int + The target position size (integer). + """ + return int(bet_size_sigmoid(w, f - actual_price) * maximum_position_size) + + +def inverse_price(f: float, w: float, m: float) -> float: + """ + Calculates the inverse price given a bet size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + f : float + Forecasted price. + w : float + Coefficient regulating the sigmoid width. + m : float + Bet size (as a fraction of max size, -1 to 1). + + Returns + ------- + float + The implied price for bet size `m`. + """ + if m == 1.0 or m == -1.0: + return f # Avoid division by zero + return f - m * np.sqrt(w / (1 - m**2)) + + +def limit_price( + target_position_size: int, + current_position: int, + f: float, + w: float, + maximum_position_size: int, +) -> float: + """ + Calculate the limit price for adjusting position. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + target_position_size : int + The target position size. + current_position : int + The current position size. + f : float + Forecasted price. + w : float + Coefficient regulating the sigmoid width. + maximum_position_size : int + Maximum absolute position size. + + Returns + ------- + float + The average limit price. + """ + if target_position_size == current_position: + return f # No change + + sgn = np.sign(target_position_size - current_position) + limit = 0.0 + + # Average price from current to target position + for i in range(abs(current_position + sgn), abs(target_position_size + sgn)): + limit += inverse_price(f, w, i / float(maximum_position_size)) + + limit /= abs(target_position_size - current_position) + return limit + + +def compute_sigmoid_width(x: float, m: float) -> float: + """ + Get the 'w' coefficient implied by a divergence and bet size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + x : float + Divergence (forecast - market). + m : float + Bet size (fraction, -1 to 1, not 0, 1, or -1). + + Returns + ------- + float + The implied 'w' coefficient. + """ + if m == 0.0 or m == 1.0 or m == -1.0: + return np.inf # w is undefined + return x**2 * ((1 / m**2) - 1) + + +# --------------------------------------------------------------------------- # +# Deprecated camelCase aliases (the historical AFML-style names). Each keeps +# working and emits a DeprecationWarning; scheduled for removal in 4.0.0. +# See NAMING_CANON_2.0.0.md. +# --------------------------------------------------------------------------- # +avgActiveSignals = deprecated_alias( + avg_active_signals, "avgActiveSignals", removed_in="4.0.0" +) +mpAvgActiveSignals = deprecated_alias( + mp_avg_active_signals, "mpAvgActiveSignals", removed_in="4.0.0" +) +discreteSignal = deprecated_alias(discrete_signal, "discreteSignal", removed_in="4.0.0") +Signal = deprecated_alias(generate_signal, "Signal", removed_in="4.0.0") +betSize = deprecated_alias(bet_size_sigmoid, "betSize", removed_in="4.0.0") +TPos = deprecated_alias(target_position, "TPos", removed_in="4.0.0") +inversePrice = deprecated_alias(inverse_price, "inversePrice", removed_in="4.0.0") +limitPrice = deprecated_alias(limit_price, "limitPrice", removed_in="4.0.0") +getW = deprecated_alias(compute_sigmoid_width, "getW", removed_in="4.0.0") diff --git a/src/RiskLabAI/backtest/multiple_testing.py b/src/RiskLabAI/backtest/multiple_testing.py new file mode 100644 index 0000000..9968f8d --- /dev/null +++ b/src/RiskLabAI/backtest/multiple_testing.py @@ -0,0 +1,193 @@ +r""" +Multiple-testing haircuts for the Sharpe ratio (Holm FWER, Benjamini-Hochberg-Yekutieli FDR). + +When a researcher screens many strategies or factors and reports the significant ones, the selection +inflates the apparent significance. These corrections discount it by controlling, across the whole +family of trials, either the family-wise error rate (the chance of *any* false discovery) or the +false-discovery rate (the expected *fraction* of false discoveries). They complement the Deflated +Sharpe Ratio (`probabilistic_sharpe_ratio` / `expected_max_sharpe_ratio`): the DSR asks whether the +single best strategy survives its trial count, while these screen a whole family with a stated +error-control target. + +Admitted in Appraisal 07 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + To judge a family of screened strategies or factors (not just the single best), prefer Holm when + you must control the chance of any false positive across the family (FWER), and BHY when you want + to bound the expected fraction of false discoveries (FDR) and can accept lower power. The Deflated + Sharpe remains the tool when the question is whether one specific best strategy survives its trial + count. Bonferroni is dominated by Holm; the t>3 hurdle over-rejects; the double-bootstrap does not + control its stated error as implemented and is not admitted. + +Evidence and caveats: appraisals/07_verdict.md. + +References +---------- +Holm, S. (1979) A simple sequentially rejective multiple test procedure. + Scandinavian Journal of Statistics, 6(2), 65-70. +Benjamini, Y. and Yekutieli, D. (2001) The control of the false discovery rate in multiple testing + under dependency. Annals of Statistics, 29(4), 1165-1188. +Harvey, C. R. and Liu, Y. (2015) Backtesting. Journal of Portfolio Management, 42(1), 13-28. +""" + +from __future__ import annotations + +import numpy as np +from scipy import stats as ss + + +def sharpe_ratio_p_values( + sharpe_ratios: np.ndarray, number_of_returns: int +) -> np.ndarray: + r""" + One-sided p-values for a positive edge (H1: SR > 0) from per-period Sharpe ratios. + + Under the i.i.d. null the Sharpe t-statistic is :math:`t = \hat{SR}\sqrt{T}`, so the one-sided + p-value is :math:`\Phi(-t)`. + + Parameters + ---------- + sharpe_ratios : np.ndarray + Per-period Sharpe ratios of the screened strategies. + number_of_returns : int + The number of return observations T each Sharpe is estimated from. + + Returns + ------- + np.ndarray + One-sided p-values, one per strategy. + """ + t_stats = np.asarray(sharpe_ratios, dtype=float) * np.sqrt(number_of_returns) + return ss.norm.sf(t_stats) + + +def holm_adjusted_p_values(p_values: np.ndarray) -> np.ndarray: + r""" + Holm (1979) step-down family-wise-error-rate adjusted p-values. + + Sorting the p-values ascending, the adjusted value at rank i is + :math:`\max_{j \le i} (M - j + 1) p_{(j)}`, clipped to 1, then mapped back to the input order. A + strategy is a discovery at level alpha when its adjusted p-value is below alpha. Holm controls the + FWER under any dependence and uniformly dominates Bonferroni (same size, at least as powerful). + + Parameters + ---------- + p_values : np.ndarray + Raw per-strategy p-values. + + Returns + ------- + np.ndarray + Holm-adjusted p-values, in the original order. + """ + p = np.asarray(p_values, dtype=float) + m = p.size + if m == 0: + return p.copy() + order = np.argsort(p) + sorted_p = p[order] + adjusted_sorted = np.maximum.accumulate((m - np.arange(m)) * sorted_p) + adjusted_sorted = np.clip(adjusted_sorted, 0.0, 1.0) + adjusted = np.empty(m) + adjusted[order] = adjusted_sorted + return adjusted + + +def benjamini_hochberg_yekutieli_adjusted_p_values(p_values: np.ndarray) -> np.ndarray: + r""" + Benjamini-Hochberg-Yekutieli (BHY, 2001) step-up false-discovery-rate adjusted p-values. + + Uses the Yekutieli dependence constant :math:`c(M) = \sum_{i=1}^{M} 1/i`, valid under arbitrary + dependence. Sorting the p-values ascending, the adjusted value at rank i is the running minimum + (from the largest rank down) of :math:`c(M)\, M\, p_{(i)} / i`, clipped to 1. A strategy is a + discovery at level alpha when its adjusted p-value is below alpha. BHY controls the FDR (the + expected fraction of false discoveries) under any dependence. + + Parameters + ---------- + p_values : np.ndarray + Raw per-strategy p-values. + + Returns + ------- + np.ndarray + BHY-adjusted p-values, in the original order. + """ + p = np.asarray(p_values, dtype=float) + m = p.size + if m == 0: + return p.copy() + order = np.argsort(p) + sorted_p = p[order] + c_m = np.sum(1.0 / np.arange(1, m + 1)) + ranks = np.arange(1, m + 1) + factor = c_m * m / ranks + # Step-up: enforce monotonicity from the largest p-value downward. + adjusted_sorted = np.minimum.accumulate((factor * sorted_p)[::-1])[::-1] + adjusted_sorted = np.clip(adjusted_sorted, 0.0, 1.0) + adjusted = np.empty(m) + adjusted[order] = adjusted_sorted + return adjusted + + +def haircut_sharpe_ratios( + sharpe_ratios: np.ndarray, + number_of_returns: int, + method: str = "holm", + significance_level: float = 0.05, +) -> dict[str, np.ndarray]: + r""" + Apply a multiple-testing haircut to a family of screened Sharpe ratios. + + Converts each Sharpe to its one-sided p-value, adjusts the p-values for multiple testing (Holm for + FWER or BHY for FDR), flags the surviving discoveries at ``significance_level``, and reports the + haircut Sharpe - the Sharpe implied by the adjusted p-value, :math:`\hat{SR}_{adj} = + \Phi^{-1}(1 - p_{adj}) / \sqrt{T}` - so the discount from selection is explicit. + + Prefer Holm when you must control the chance of any false positive across the family (FWER), and + BHY when you want to bound the expected fraction of false discoveries (FDR) and can accept lower + power. The Deflated Sharpe remains the tool when the question is whether one specific best strategy + survives its trial count. See the module docstring for the full regime tag and + appraisals/07_verdict.md. + + Parameters + ---------- + sharpe_ratios : np.ndarray + Per-period Sharpe ratios of the screened strategies (the trial set). + number_of_returns : int + The number of return observations T. + method : str, default "holm" + ``"holm"`` (FWER) or ``"bhy"`` (FDR). + significance_level : float, default 0.05 + The error-control level alpha. + + Returns + ------- + dict + ``p_values`` (raw), ``adjusted_p_values``, ``significant`` (boolean discoveries at alpha), and + ``haircut_sharpe_ratios`` (the Sharpe implied by each adjusted p-value). + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest import haircut_sharpe_ratios + >>> sharpes = np.array([0.25, 0.18, 0.05, 0.02]) # per-period Sharpes from a 4-strategy search + >>> result = haircut_sharpe_ratios(sharpes, number_of_returns=120, method="holm") + >>> bool(result["significant"][0]) + True + """ + p_values = sharpe_ratio_p_values(sharpe_ratios, number_of_returns) + if method == "holm": + adjusted = holm_adjusted_p_values(p_values) + elif method == "bhy": + adjusted = benjamini_hochberg_yekutieli_adjusted_p_values(p_values) + else: + raise ValueError(f"unknown method {method!r}; use 'holm' or 'bhy'.") + significant = adjusted < significance_level + haircut_t = ss.norm.isf(np.clip(adjusted, 1e-300, 1.0)) + haircut_sr = haircut_t / np.sqrt(number_of_returns) + return { + "p_values": p_values, + "adjusted_p_values": adjusted, + "significant": significant, + "haircut_sharpe_ratios": haircut_sr, + } diff --git a/src/RiskLabAI/backtest/ou_trading_rules.py b/src/RiskLabAI/backtest/ou_trading_rules.py new file mode 100644 index 0000000..7e25287 --- /dev/null +++ b/src/RiskLabAI/backtest/ou_trading_rules.py @@ -0,0 +1,311 @@ +r""" +Closed-form optimal Ornstein-Uhlenbeck trading rules (Lipton-Lopez de Prado 2020). + +de Prado finds optimal profit-take / stop-loss thresholds for a mean-reverting (OU) process by a +Monte-Carlo grid search (AFML ch.13): simulate many OU paths and, for each (profit-take, stop-loss) pair +on a grid, score the realized trade outcomes and pick the grid argmax. That grid carries simulation +noise, costs compute, and is limited by the grid resolution. The Lipton-Lopez de Prado closed form gives +the exact optimum for the OU model from first-passage theory, with no simulation noise and negligible +compute. + +For an OU deviation ``Y`` with ``dY = -theta Y dt + sigma dW`` entered at deviation ``y0 = -entry_gap``, +with an upper barrier ``b = y0 + profit_take`` (gain ``+profit_take``) and a lower barrier +``a = y0 - stop_loss`` (gain ``-stop_loss``), the probability of hitting the upper barrier first is +``u(y0) = [S(y0) - S(a)] / [S(b) - S(a)]`` with scale function ``S(x) = int_0^x exp(theta s^2 / sigma^2) ds`` +(proportional to the imaginary error function), and the mean first-exit time follows the Karlin-Taylor +Green's-function form. The per-trade gain is two-valued, so ``E[gain] = profit_take * u - stop_loss * (1-u) +- cost``, and the objective is the expected net return per unit time ``E[gain] / E[tau]`` (the well-posed +optimal-trading-rule objective; the raw per-trade Sharpe is degenerate, maximized by profit_take -> 0). + +Admitted in Appraisal 23 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tag, verbatim from the verdict: + + prefer the closed-form OU rule over the Monte-Carlo PT/SL grid whenever the OU model is used - it + gives the exact optimum with no simulation noise at a fraction of the compute; both are only as good + as the OU assumption, so check the OU fit at decision time. + +Held-out confirmed (appraisals/23_results, HELDOUT.md): on the sealed OU corner the closed form agrees +with the Monte-Carlo grid optimum within grid resolution (the grid pick's regret on the exact surface +stays ~0.0002-0.0008 and tightens as the grid refines) at a compute speed-up that grows with resolution +(3.4x -> 16x), and it degrades in step with the grid off-model (no false out-of-model win). The advantage +is exactness and speed under the OU model, not robustness to misspecification, so check the OU fit at +decision time. The de Prado Monte-Carlo PT/SL grid is left unchanged. Evidence: appraisals/23_verdict.md. + +References +---------- +Lipton, A. and Lopez de Prado, M. (2020) A closed-form solution for optimal mean-reverting trading + strategies. SSRN / Risk. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 13 (the Monte-Carlo PT/SL grid). +""" + +from __future__ import annotations + +import numpy as np +from scipy.optimize import minimize +from scipy.special import erfi + + +def theta_from_half_life(half_life: float) -> float: + """Continuous mean-reversion speed ``theta`` from a half-life in time units.""" + return float(np.log(2.0) / float(half_life)) + + +def stationary_std(theta: float, sigma: float) -> float: + """Stationary standard deviation of the OU deviation: ``sigma / sqrt(2 theta)``.""" + return float(sigma) / np.sqrt(2.0 * float(theta)) + + +def _scale_relative(x: float, a: float, b: float, c: float) -> float: + """``exp(-cK)(S(x) - S(a))`` with ``K = max(a^2, b^2)`` (overflow-safe), S the OU scale function.""" + big = max(a * a, b * b) + rc = np.sqrt(c) + val = ( + (np.sqrt(np.pi) / (2.0 * rc)) * (erfi(rc * x) - erfi(rc * a)) * np.exp(-c * big) + ) + if np.isfinite(val): + return float(val) + s = np.linspace(a, x, 2000) + return float(np.trapezoid(np.exp(c * (s * s - big)), s)) + + +def hit_upper_probability( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + r""" + Probability the OU deviation hits the upper barrier (profit-take) before the lower (stop-loss). + + Entered at ``y0 = -entry_gap``, barriers ``a = y0 - stop_loss`` and ``b = y0 + profit_take``; + ``u = [S(y0) - S(a)] / [S(b) - S(a)]`` in an overflow-safe max-subtracted form. + """ + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + if b <= a: + return float("nan") + num = _scale_relative(y0, a, b, c) + den = _scale_relative(b, a, b, c) + if den <= 0: + return float("nan") + return float(np.clip(num / den, 0.0, 1.0)) + + +def mean_exit_time( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + r""" + Closed-form OU mean first-exit time ``E[tau]`` from the two-barrier interval, via the Karlin-Taylor + Green's function ``v(x) = int_a^b G(x, y) m(y) dy`` (speed density ``m``), evaluated by quadrature. + """ + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + if b <= a: + return float("nan") + rc = np.sqrt(c) + pref = np.sqrt(np.pi) / (2.0 * rc) + ys = np.linspace(a, b, 400) + erfi_y, erfi_a, erfi_b, erfi_y0 = ( + erfi(rc * ys), + float(erfi(rc * a)), + float(erfi(rc * b)), + float(erfi(rc * y0)), + ) + if not (np.all(np.isfinite(erfi_y)) and np.isfinite(erfi_b)): + return _mean_exit_time_quadrature( + entry_gap, profit_take, stop_loss, theta, sigma + ) + s_ya = pref * (erfi_y - erfi_a) + s_by = pref * (erfi_b - erfi_y) + s_ba = pref * (erfi_b - erfi_a) + s_y0a = pref * (erfi_y0 - erfi_a) + s_by0 = pref * (erfi_b - erfi_y0) + green = np.where(ys <= y0, s_ya * s_by0, s_y0a * s_by) / s_ba + speed = (2.0 / (sigma * sigma)) * np.exp(-c * ys * ys) + return float(np.trapezoid(green * speed, ys)) + + +def _mean_exit_time_quadrature( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + """Overflow-safe ``E[tau]`` for very wide barriers (max-subtracted scale-function quadrature).""" + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + big = max(a * a, b * b) + + def s_hat(x): + s = np.linspace(a, x, 3000) + return np.trapezoid(np.exp(c * (s * s - big)), s) + + s_hat_b, s_hat_y0 = s_hat(b), s_hat(y0) + ys = np.linspace(a, b, 3000) + s_ya = np.array([s_hat(y) for y in ys]) + s_by = s_hat_b - s_ya + green = np.where(ys <= y0, s_ya * (s_hat_b - s_hat_y0), s_hat_y0 * s_by) / s_hat_b + speed = (2.0 / (sigma * sigma)) * np.exp(c * (big - ys * ys)) + return float(np.trapezoid(green * speed, ys)) + + +def ou_rule_metrics( + profit_take: float, + stop_loss: float, + theta: float, + sigma: float, + entry_gap: float, + cost: float = 0.0, +) -> dict: + r""" + Exact OU per-trade metrics for a rule entered ``entry_gap`` from the mean (all thresholds in the same + units as the OU deviation; time in model time units). + + Returns ``hit_probability`` (``u``), ``expected_gain`` (net mean per-trade gain), ``std_gain``, + ``expected_holding_time`` (``E[tau]``), ``return_rate`` (``expected_gain / E[tau]``, the optimization + objective), and ``time_scaled_sharpe`` (``expected_gain / (std_gain sqrt(E[tau]))``, a secondary + metric that is degenerate as ``u -> 1``). + """ + bad = dict( + hit_probability=np.nan, + expected_gain=-np.inf, + std_gain=np.nan, + expected_holding_time=np.nan, + return_rate=-np.inf, + time_scaled_sharpe=-np.inf, + ) + if profit_take <= 0 or stop_loss <= 0: + return bad + u = hit_upper_probability(entry_gap, profit_take, stop_loss, theta, sigma) + if not np.isfinite(u): + return bad + q = 1.0 - u + expected_gain = profit_take * u - stop_loss * q - cost + std_gain = (profit_take + stop_loss) * np.sqrt(max(u * q, 0.0)) + e_tau = mean_exit_time(entry_gap, profit_take, stop_loss, theta, sigma) + if not np.isfinite(e_tau) or e_tau <= 0: + return bad + return dict( + hit_probability=float(u), + expected_gain=float(expected_gain), + std_gain=float(std_gain), + expected_holding_time=float(e_tau), + return_rate=float(expected_gain / e_tau), + time_scaled_sharpe=( + float(expected_gain / (std_gain * np.sqrt(e_tau))) + if std_gain > 0 + else -np.inf + ), + ) + + +def optimal_ou_trading_rule( + theta: float, + sigma: float, + entry_gap: float, + cost: float = 0.0, + bounds: tuple = (0.25, 4.0), +) -> dict: + r""" + Exact optimal OU profit-take / stop-loss by maximizing the closed-form expected net return per unit + time (no simulation). + + Multi-start L-BFGS-B on the negative ``return_rate`` objective over the ``(profit_take, stop_loss)`` + box. Prefer this over the Monte-Carlo PT/SL grid whenever the OU model is used: it gives the exact + optimum with no simulation noise at a fraction of the compute. Both are only as good as the OU + assumption, so check the OU fit at decision time (see :func:`fit_ornstein_uhlenbeck` and the module + docstring for the verbatim scope tag and appraisals/23_verdict.md). + + Parameters + ---------- + theta : float + OU mean-reversion speed. + sigma : float + OU instantaneous volatility. + entry_gap : float + Deviation from the mean at entry (the trade is opened ``entry_gap`` below the mean). + cost : float, default 0.0 + Per-trade transaction cost in the same units as the thresholds. + bounds : tuple, default (0.25, 4.0) + ``(low, high)`` box for both the profit-take and the stop-loss. + + Returns + ------- + dict + ``profit_take``, ``stop_loss``, ``return_rate`` (the maximized objective), and the full + :func:`ou_rule_metrics` of the optimum. + + Examples + -------- + >>> rule = optimal_ou_trading_rule(theta=0.1, sigma=0.1, entry_gap=1.0) + >>> rule["profit_take"] > 0 and rule["stop_loss"] > 0 + True + """ + lo, hi = bounds + + def negative_objective(x): + v = ou_rule_metrics(x[0], x[1], theta, sigma, entry_gap, cost)["return_rate"] + return -v if np.isfinite(v) else 1e9 + + span = hi - lo + starts = [ + (lo + 0.15 * span, lo + 0.2 * span), + (lo + 0.1 * span, lo + 0.6 * span), + (lo + 0.3 * span, lo + 0.4 * span), + ] + best = None + for start in starts: + res = minimize( + negative_objective, x0=start, method="L-BFGS-B", bounds=[(lo, hi), (lo, hi)] + ) + if best is None or res.fun < best.fun: + best = res + profit_take, stop_loss = float(best.x[0]), float(best.x[1]) + metrics = ou_rule_metrics(profit_take, stop_loss, theta, sigma, entry_gap, cost) + return { + "profit_take": profit_take, + "stop_loss": stop_loss, + "return_rate": -float(best.fun), + **metrics, + } + + +def fit_ornstein_uhlenbeck(series: np.ndarray, dt: float = 1.0) -> dict: + r""" + Fit an OU process by AR(1) regression ``x_t = a + b x_{t-1} + e`` (the OU goodness-of-fit check). + + Returns ``theta`` (mean-reversion speed), ``sigma`` (per-unit-time noise), ``mu`` (long-run mean), + ``rho`` (AR(1) coefficient), ``half_life``, ``stationary_std``, and ``r2`` (the AR(1) fit quality, a + simple OU goodness-of-fit proxy). Check ``r2`` / the half-life before trusting the closed-form rule: + both the closed form and the Monte-Carlo grid are only as good as the OU assumption. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = np.zeros(2000) + >>> for t in range(1, 2000): + ... x[t] = 0.9 * x[t - 1] + rng.standard_normal() * 0.1 + >>> fit = fit_ornstein_uhlenbeck(x) + >>> fit["theta"] > 0 and 0.0 <= fit["r2"] <= 1.0 + True + """ + x = np.asarray(series, dtype=float) + x = x[np.isfinite(x)] + x0, x1 = x[:-1], x[1:] + design = np.vstack([np.ones_like(x0), x0]).T + coef, *_ = np.linalg.lstsq(design, x1, rcond=None) + a, b = float(coef[0]), float(coef[1]) + residual = x1 - (a + b * x0) + b = min(max(b, 1e-6), 0.999999) + theta = -np.log(b) / dt + mu = a / (1.0 - b) + var_eps = float(np.var(residual, ddof=2)) + sigma = np.sqrt(var_eps * 2.0 * theta / (1.0 - b * b)) + ss_tot = float(np.var(x1, ddof=1)) * (len(x1) - 1) + r2 = 1.0 - float(np.sum(residual**2)) / ss_tot if ss_tot > 0 else 0.0 + return dict( + theta=float(theta), + sigma=float(sigma), + mu=float(mu), + rho=float(b), + half_life=float(np.log(2.0) / theta), + stationary_std=stationary_std(theta, sigma), + r2=float(r2), + ) diff --git a/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py b/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py new file mode 100644 index 0000000..6f644a1 --- /dev/null +++ b/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py @@ -0,0 +1,144 @@ +""" +Implements the Probabilistic Sharpe Ratio (PSR) and related metrics +as described by Marcos Lopez de Prado. + +## TODO: +- [ ] Add a `compute_psr_curve` helper function (as seen in the + original notebook) that iterates `probabilistic_sharpe_ratio` + over a range of `observed_sharpe_ratio` values to + easily plot the PSR curve. +""" + +import numpy as np +from scipy import stats as ss + + +def probabilistic_sharpe_ratio( + observed_sharpe_ratio: float, + benchmark_sharpe_ratio: float, + number_of_returns: int, + skewness_of_returns: float = 0.0, + kurtosis_of_returns: float = 3.0, + return_test_statistic: bool = False, +) -> float: + r""" + Calculate the Probabilistic Sharpe Ratio (PSR). + + The PSR estimates the probability that the observed Sharpe ratio (SR) + is greater than a benchmark SR (e.g., the expected maximum SR from + multiple trials) given the track record length and return moments. + + The test statistic is: + .. math:: + Z = \frac{(\hat{SR} - SR^*) \sqrt{T-1}} + {\sqrt{1 - S \hat{SR} + \frac{K-1}{4} \hat{SR}^2}} + + PSR = N(Z) + + Where: + - \(\hat{SR}\) is the observed Sharpe ratio + - \(SR^*\) is the benchmark Sharpe ratio + - \(T\) is the number of returns + - \(S\) is the skewness of returns + - \(K\) is the kurtosis of returns + + Parameters + ---------- + observed_sharpe_ratio : float + The observed Sharpe ratio of the strategy. + benchmark_sharpe_ratio : float + The benchmark Sharpe ratio (e.g., E[max(SR)]). + number_of_returns : int + The number of return observations (T). + skewness_of_returns : float, default=0.0 + The skewness of the returns (S). + kurtosis_of_returns : float, default=3.0 + The kurtosis of the returns (K). Assumes 3 for normal. + return_test_statistic : bool, default=False + If True, return the Z-statistic instead of the CDF value (PSR). + + Returns + ------- + float + The Probabilistic Sharpe Ratio (if `return_test_statistic` is False) + or the Z test statistic (if True). + + Example + ------- + >>> psr = probabilistic_sharpe_ratio( + ... observed_sharpe_ratio=2.5, + ... benchmark_sharpe_ratio=1.0, + ... number_of_returns=252, + ... skewness_of_returns=-0.5, + ... kurtosis_of_returns=4.0 + ... ) + >>> print(f"PSR: {psr:.2f}") + PSR: 1.00 + """ + denominator = ( + 1 + - skewness_of_returns * observed_sharpe_ratio + + (kurtosis_of_returns - 1) / 4 * observed_sharpe_ratio**2 + ) + + # Handle cases where denominator is non-positive due to extreme inputs + if denominator <= 0: + return 0.0 if not return_test_statistic else -np.inf + + test_statistic = ( + (observed_sharpe_ratio - benchmark_sharpe_ratio) + * np.sqrt(number_of_returns - 1) + ) / np.sqrt(denominator) + + if return_test_statistic: + return test_statistic + + return ss.norm.cdf(test_statistic) + + +def benchmark_sharpe_ratio(sharpe_ratio_estimates: list[float]) -> float: + r""" + Calculate the expected maximum Sharpe Ratio (Benchmark SR). + + This is used as the benchmark SR* in the PSR calculation. It represents + the expected maximum SR one would observe from N independent trials. + + .. math:: + SR^* = \sigma_{SR} \left[ (1 - \gamma) \Phi^{-1}(1 - \frac{1}{N}) + + \gamma \Phi^{-1}(1 - \frac{1}{N} e^{-1}) \right] + + Where: + - \(\sigma_{SR}\) is the standard deviation of SR estimates + - \(\gamma\) is the Euler-Mascheroni constant + - \(\Phi^{-1}\) is the inverse CDF of a standard normal distribution + - \(N\) is the number of SR estimates (trials) + + Parameters + ---------- + sharpe_ratio_estimates : List[float] + A list or array of Sharpe ratio estimates from N different trials. + + Returns + ------- + float + The Benchmark Sharpe Ratio (E[max(SR)]). + + Example + ------- + >>> sr_list = [0.5, 1.2, -0.3, 0.8, 1.5, 0.9] + >>> bsr = benchmark_sharpe_ratio(sr_list) + >>> print(f"Benchmark SR: {bsr:.2f}") + Benchmark SR: 1.03 + """ + n_estimates = len(sharpe_ratio_estimates) + if n_estimates <= 1: + return np.mean(sharpe_ratio_estimates) if n_estimates == 1 else 0.0 + + standard_deviation = np.array(sharpe_ratio_estimates).std() + + term1 = (1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / n_estimates) + term2 = np.euler_gamma * ss.norm.ppf(1 - 1 / (n_estimates * np.e)) + + benchmark_value = standard_deviation * (term1 + term2) + + return benchmark_value diff --git a/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py b/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py new file mode 100644 index 0000000..9ccb6d4 --- /dev/null +++ b/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py @@ -0,0 +1,156 @@ +""" +Implements the Probability of Backtest Overfitting (PBO) calculation. + +## TODO: +- [ ] Add a `get_pbo` wrapper function (as seen in the + original notebook) that simplifies the call to + `probability_of_backtest_overfitting` and returns + only the PBO value. +- [ ] Add a `pbo_overfitting_plot` helper function to + visualize the logit distribution (as seen in the + original notebook). +""" + +from itertools import combinations +from typing import Callable, Optional + +import numpy as np +from joblib import Parallel, delayed + +from .backtest_statistics import sharpe_ratio + + +def performance_evaluation( + train_partition: np.ndarray, + test_partition: np.ndarray, + n_strategies: int, + metric: Callable[[np.ndarray, float], float], + risk_free_return: float, +) -> tuple[bool, float]: + """ + Evaluate strategy performance on train/test splits. + + This function finds the best strategy on the training partition and + then ranks its performance on the test partition. + + Parameters + ---------- + train_partition : np.ndarray + Training data partition (T_train x N_strategies). + test_partition : np.ndarray + Testing data partition (T_test x N_strategies). + n_strategies : int + Number of strategies (N). + metric : Callable + Metric function (e.g., `sharpe_ratio`). + risk_free_return : float + Risk-free return for the metric. + + Returns + ------- + Tuple[bool, float] + - is_overfit: True if the best in-sample strategy is not + in the top half of out-of-sample strategies. + - logit_value: The logit-transformed relative rank. + """ + # 1. Find best strategy on training data + evaluate_train = [ + metric(train_partition[:, i], risk_free_return) for i in range(n_strategies) + ] + best_strategy_idx = np.argmax(evaluate_train) + + # 2. Evaluate all strategies on test data + evaluate_test = [ + metric(test_partition[:, i], risk_free_return) for i in range(n_strategies) + ] + + # 3. Find rank of the best_strategy in the test set + # Sort test evaluations and find where best_strategy_idx lands + ranks = np.argsort(np.argsort(evaluate_test)) + rank_of_best_is_strategy = ranks[best_strategy_idx] + 1 # 1-based rank + + # 4. Calculate relative rank (omega_bar) and logit + w_bar = rank_of_best_is_strategy / (n_strategies + 1) + logit_value = np.log(w_bar / (1 - w_bar)) + + # Overfit if logit is <= 0 (i.e., rank is in bottom half) + return logit_value <= 0.0, logit_value + + +def probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + risk_free_return: float = 0.0, + metric: Optional[Callable[[np.ndarray, float], float]] = None, + n_jobs: int = 1, +) -> tuple[float, np.ndarray]: + r""" + Compute the Probability of Backtest Overfitting (PBO). + + PBO is the frequency with which the best in-sample (IS) strategy + underperforms relative to the median out-of-sample (OOS) performance. + + It uses combinatorial splits of the data into train/test partitions. + Total combinations = C(S, S/2), where S = n_partitions. + + .. math:: + \left(\begin{array}{c} + S \\ + S / 2 + \end{array}\right) = \prod_{i=0}^{S / 2^{-1}} \frac{S-i}{S / 2-i} + + Parameters + ---------- + performances : np.ndarray + Matrix of T observations x N strategies. + n_partitions : int, default=16 + Number of partitions (S). Must be even. + risk_free_return : float, default=0.0 + Risk-free return for calculating the metric (e.g., Sharpe ratio). + metric : Callable, optional + Metric function (e.g., `sharpe_ratio`). Defaults to the + Numba-jitted `sharpe_ratio` in this module. + n_jobs : int, default=1 + Number of parallel jobs for `joblib`. + + Returns + ------- + Tuple[float, np.ndarray] + - pbo: The Probability of Backtest Overfitting (0 to 1). + - logit_values: Array of logit values from all combinations. + """ + if n_partitions % 2 != 0: + raise ValueError("Number of partitions must be even.") + + if metric is None: + metric = sharpe_ratio + + _, n_strategies = performances.shape + partitions = np.array_split(performances, n_partitions) + partition_indices = list(range(n_partitions)) + + # Get all combinations of training partition indices + partition_combinations_indices = list( + combinations(partition_indices, n_partitions // 2) + ) + + results = Parallel(n_jobs=n_jobs)( + delayed(performance_evaluation)( + np.concatenate([partitions[i] for i in train_indices], axis=0), + np.concatenate( + [partitions[i] for i in partition_indices if i not in train_indices], + axis=0, + ), + n_strategies, + metric, + risk_free_return, + ) + for train_indices in partition_combinations_indices + ) + + results_arr = np.array(results) + + pbo = results_arr[:, 0].mean() + logit_values = results_arr[:, 1] + + return pbo, logit_values diff --git a/src/RiskLabAI/backtest/robust_statistics.py b/src/RiskLabAI/backtest/robust_statistics.py new file mode 100644 index 0000000..89c224b --- /dev/null +++ b/src/RiskLabAI/backtest/robust_statistics.py @@ -0,0 +1,289 @@ +r""" +Robust backtest statistics: Conditional Expected Drawdown (CED) and the Ledoit-Wolf Sharpe-difference test. + +de Prado reports maximum drawdown and time-under-water and compares Sharpe ratios directly +(`backtest_statistics.py`). Maximum drawdown is a single extreme order statistic - high variance, not +coherent, not attributable - and the naive Sharpe-difference comparison assumes i.i.d. returns, so it +over-rejects under the serial dependence real returns carry. Two admitted upgrades sit alongside those +baselines: + +- **Conditional Expected Drawdown (CED, Goldberg-Mahmoud 2017)**: the tail mean (CVaR) of the + maximum-drawdown distribution at a fixed horizon, a lower-variance, coherent, factor-attributable + drawdown-risk measure; and +- **the Ledoit-Wolf (2008) bootstrap Sharpe-difference test**: a HAC-studentized statistic calibrated by + a studentized circular block bootstrap, which holds nominal size under serial dependence where the + naive test inflates. + +Admitted in Appraisal 22 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tags, verbatim from the verdict: + + CED: prefer CED over max-drawdown as a drawdown-risk statistic: lower estimator variance and better + ranking of true drawdown risk, most of all on short/heavy-tailed tracks; converges to max-DD on + benign returns; factor-attributable. + + Ledoit-Wolf Sharpe-difference test: prefer it over the naive test when comparing two Sharpes under + serial dependence / heavy tails (it holds nominal size where the naive test inflates ~3x); converges + on i.i.d. returns. + +Held-out confirmed (appraisals/22_results, HELDOUT.md): on the sealed short-sample x heavy-tail corner CED +has lower estimator variance (CV 0.30-0.32 vs max-drawdown 0.40-0.42) and better rank-recovery (Spearman +0.42-0.78 vs 0.31-0.69); the LW test holds nominal size (0.056) under AR(0.3) where the naive test rejects +0.161 (~3x), and converges on i.i.d. returns. CDaR@0.95 was NOT admitted (no variance/rank advantage over +max-drawdown as a descriptive statistic). Evidence and caveats: appraisals/22_verdict.md. + +References +---------- +Goldberg, L. R. and Mahmoud, O. (2017) Drawdown: from practice to theory and back again. + Quantitative Finance, 17(5), 745-761. (Conditional Expected Drawdown.) +Ledoit, O. and Wolf, M. (2008) Robust performance hypothesis testing with the Sharpe ratio. + Journal of Empirical Finance, 15(5), 850-859. +""" + +from __future__ import annotations + +from math import erfc, sqrt + +import numpy as np + + +# -------------------------------------------------------------------------------------------------- +# Conditional Expected Drawdown +# -------------------------------------------------------------------------------------------------- +def _wealth_from_returns(returns: np.ndarray) -> np.ndarray: + return np.cumprod(1.0 + np.asarray(returns, dtype=float)) + + +def _drawdown_series(wealth: np.ndarray) -> np.ndarray: + w = np.asarray(wealth, dtype=float) + return 1.0 - w / np.maximum.accumulate(w) + + +def _cvar_upper(values: np.ndarray, alpha: float) -> float: + """Empirical upper-tail CVaR: the mean of the worst ``1 - alpha`` fraction of ``values``.""" + v = np.sort(np.asarray(values, dtype=float)) + n = v.size + if n == 0: + return 0.0 + k = (1.0 - alpha) * n + if k <= 0: + return float(v[-1]) + n_full = int(np.floor(k)) + total = v[n - n_full :].sum() if n_full > 0 else 0.0 + frac = k - n_full + if frac > 0 and n - n_full - 1 >= 0: + total += frac * v[n - n_full - 1] + return float(total / k) + + +def _rolling_max_drawdowns(wealth: np.ndarray, horizon: int) -> np.ndarray: + w = np.asarray(wealth, dtype=float) + n = w.size + if n < horizon or horizon < 2: + d = _drawdown_series(w) + return np.array([float(d.max()) if d.size else 0.0], dtype=float) + windows = np.lib.stride_tricks.sliding_window_view(w, horizon) + high_water = np.maximum.accumulate(windows, axis=1) + return (1.0 - windows / high_water).max(axis=1) + + +def conditional_expected_drawdown( + returns: np.ndarray, + horizon: int, + alpha: float = 0.90, +) -> float: + r""" + Conditional Expected Drawdown (CED; Goldberg-Mahmoud 2017): the CVaR of the maximum-drawdown + distribution at a fixed horizon. + + From a single return track it is estimated by the upper-tail (CVaR at level ``alpha``) mean of the + maximum drawdowns within overlapping windows of length ``horizon``. Because it is a tail MEAN of the + max-drawdown distribution rather than the single most-extreme realization, it has lower estimator + variance and ranks true drawdown risk better than max-drawdown, especially on short / heavy-tailed + tracks, and it converges to agree with max-drawdown on benign returns. It is coherent and + factor-attributable. + + Prefer CED over max-drawdown as a drawdown-risk statistic (see the module docstring for the full + verbatim scope tag and appraisals/22_verdict.md). The de Prado max-drawdown / time-under-water + statistics are unchanged. + + Parameters + ---------- + returns : np.ndarray + Per-period simple returns of the track. + horizon : int + The rolling-window length (the drawdown-measurement horizon, in periods). + alpha : float, default 0.90 + The CVaR tail level (the mean is taken over the worst ``1 - alpha`` fraction of window maxima). + + Returns + ------- + float + The conditional expected drawdown in ``[0, 1)``. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> r = rng.standard_normal(2000) * 0.01 + >>> ced = conditional_expected_drawdown(r, horizon=60) + >>> 0.0 <= ced < 1.0 + True + """ + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must be in (0, 1)") + wealth = _wealth_from_returns(returns) + return _cvar_upper(_rolling_max_drawdowns(wealth, int(horizon)), alpha) + + +# -------------------------------------------------------------------------------------------------- +# Sharpe-difference tests +# -------------------------------------------------------------------------------------------------- +def _sharpe_influence(r_a: np.ndarray, r_b: np.ndarray): + """Influence-function series of the Sharpe difference SR_A - SR_B and the point estimate.""" + a = np.asarray(r_a, dtype=float) + b = np.asarray(r_b, dtype=float) + mu_a, mu_b = a.mean(), b.mean() + g_a, g_b = (a * a).mean(), (b * b).mean() + s_a = np.sqrt(max(g_a - mu_a * mu_a, 1e-300)) + s_b = np.sqrt(max(g_b - mu_b * mu_b, 1e-300)) + delta = mu_a / s_a - mu_b / s_b + ga0, ga1 = g_a / s_a**3, -mu_a / (2.0 * s_a**3) + gb0, gb1 = g_b / s_b**3, -mu_b / (2.0 * s_b**3) + influence = ( + ga0 * (a - mu_a) + + ga1 * (a * a - g_a) + - (gb0 * (b - mu_b) + gb1 * (b * b - g_b)) + ) + return float(delta), influence + + +def _bartlett_hac_var(influence: np.ndarray, bandwidth: int) -> float: + """Bartlett-kernel HAC variance of the mean of the influence series (= Var(delta_hat)).""" + x = np.asarray(influence, dtype=float) + x = x - x.mean() + t = x.size + if t < 2: + return float("nan") + s = np.dot(x, x) / t + h = max(0, int(bandwidth)) + for j in range(1, h + 1): + if j >= t: + break + s += 2.0 * (1.0 - j / (h + 1.0)) * (np.dot(x[j:], x[:-j]) / t) + return float(max(s, 1e-300) / t) + + +def _iid_var(influence: np.ndarray) -> float: + x = np.asarray(influence, dtype=float) + t = x.size + if t < 2: + return float("nan") + return float(max(x.var(), 1e-300) / t) + + +def _circular_block_indices( + t: int, block_len: int, rng: np.random.Generator +) -> np.ndarray: + n_blocks = int(np.ceil(t / block_len)) + starts = rng.integers(0, t, size=n_blocks) + offsets = np.arange(block_len) + return ((starts[:, None] + offsets[None, :]) % t).reshape(-1)[:t] + + +def sharpe_difference_test( + returns_a: np.ndarray, + returns_b: np.ndarray, + method: str = "ledoit_wolf", + block_length: int | None = None, + n_boot: int = 1000, + bandwidth: int | None = None, + random_state: int | None = 0, +) -> dict: + r""" + Test H0: SR_A = SR_B (equal Sharpe ratios), naive or Ledoit-Wolf robust. + + ``method="naive"`` is the influence-function delta-method z-test using the i.i.d. (lag-0) variance of + the Sharpe-difference influence function and the normal cutoff - it ignores serial dependence and + over-rejects under it. ``method="ledoit_wolf"`` (Ledoit-Wolf 2008) studentizes the same Sharpe + difference by a Bartlett-kernel HAC standard error and calibrates the two-sided p-value by a + studentized circular block bootstrap that preserves serial dependence, so it holds nominal size where + the naive test inflates (~3x under AR), and converges to the naive test on i.i.d. returns. + + Prefer the Ledoit-Wolf test when comparing two Sharpes under serial dependence / heavy tails (see the + module docstring for the full verbatim scope tag and appraisals/22_verdict.md). The Sharpe ratio uses + the population standard deviation, matching ``backtest_statistics.sharpe_ratio``. + + Parameters + ---------- + returns_a, returns_b : np.ndarray + The two equal-length per-period return series being compared. + method : {"ledoit_wolf", "naive"}, default "ledoit_wolf" + The test variant. + block_length : int, optional + Circular-block length for the LW bootstrap (also the default HAC bandwidth). Defaults to + ``ceil(T ** (1/3))``. Ignored for ``method="naive"``. + n_boot : int, default 1000 + Bootstrap resamples for the LW p-value. Ignored for ``method="naive"``. + bandwidth : int, optional + HAC (Bartlett) bandwidth; defaults to ``block_length``. Ignored for ``method="naive"``. + random_state : int, optional + Seed for the LW bootstrap. + + Returns + ------- + dict + ``delta`` (SR_A - SR_B), ``se``, ``stat``, ``pvalue``, and ``reject`` (at the 5% level). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> a = rng.standard_normal(1000) * 0.01 + 0.001 + >>> b = rng.standard_normal(1000) * 0.01 + >>> res = sharpe_difference_test(a, b, method="ledoit_wolf", n_boot=200) + >>> set(res) == {"delta", "se", "stat", "pvalue", "reject"} + True + """ + if method not in ("ledoit_wolf", "naive"): + raise ValueError("method must be 'ledoit_wolf' or 'naive'") + a = np.asarray(returns_a, dtype=float) + b = np.asarray(returns_b, dtype=float) + if a.size != b.size: + raise ValueError("returns_a and returns_b must have the same length") + delta, influence = _sharpe_influence(a, b) + + if method == "naive": + se = np.sqrt(_iid_var(influence)) + z = delta / se if se > 0 else 0.0 + pvalue = float(erfc(abs(z) / sqrt(2.0))) + return { + "delta": delta, + "se": float(se), + "stat": float(z), + "pvalue": pvalue, + "reject": bool(pvalue < 0.05), + } + + t = a.size + if block_length is None: + block_length = max(1, int(np.ceil(t ** (1.0 / 3.0)))) + bw = block_length if bandwidth is None else bandwidth + se_hat = np.sqrt(_bartlett_hac_var(influence, bw)) + z_hat = abs(delta / se_hat) if se_hat > 0 else 0.0 + rng = np.random.default_rng(random_state) + count = 0 + for _ in range(n_boot): + idx = _circular_block_indices(t, block_length, rng) + delta_b, influence_b = _sharpe_influence(a[idx], b[idx]) + se_b = np.sqrt(_bartlett_hac_var(influence_b, bw)) + if se_b <= 0: + continue + if abs((delta_b - delta) / se_b) >= z_hat: + count += 1 + pvalue = (count + 1.0) / (n_boot + 1.0) + return { + "delta": delta, + "se": float(se_hat), + "stat": float(z_hat), + "pvalue": float(pvalue), + "reject": bool(pvalue < 0.05), + } diff --git a/src/RiskLabAI/backtest/sharpe_inference.py b/src/RiskLabAI/backtest/sharpe_inference.py new file mode 100644 index 0000000..cdbdbb5 --- /dev/null +++ b/src/RiskLabAI/backtest/sharpe_inference.py @@ -0,0 +1,200 @@ +r""" +Sharpe-ratio inference under non-normal, autocorrelated returns (Lopez de Prado-Lipton-Zoonekynd 2025). + +The sampling variance of the Sharpe estimator depends on the higher moments AND the serial correlation +of returns. de Prado's Probabilistic Sharpe Ratio (`probabilistic_sharpe_ratio`) corrects the higher +moments but assumes serial independence, so under autocorrelation its confidence interval is too narrow +and its test over-rejects. The LPLZ (2025) method takes the heteroskedasticity-and-autocorrelation- +consistent (HAC, Newey-West) long-run variance of the Sharpe influence function, correcting for both at +once. + +The Sharpe influence function is + +.. math:: + + \mathrm{IF}_t = z_t - \tfrac{1}{2}\,\widehat{SR}\,(z_t^2 - 1), \qquad z_t = (r_t - \mu)/\sigma, + +and the asymptotic variance of :math:`\sqrt{T}(\widehat{SR} - SR)` is the long-run variance of +:math:`\mathrm{IF}_t`. Under i.i.d. returns this long-run variance equals +:math:`1 - S\,\widehat{SR} + \tfrac{K-1}{4}\widehat{SR}^2` (exactly the PSR denominator), so the method +converges to the PSR when there is no serial correlation. + +Admitted in Appraisal 08 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + Prefer the LPLZ (2025) Sharpe inference - a HAC of the Sharpe influence function - when returns + show material serial correlation and/or non-normality (estimable from the sample): it restores + near-nominal CI coverage and test size where the PSR under-covers and over-rejects (PSR size ~ 0.20 + vs nominal 0.05 under AR(1)). It converges to the PSR on near-normal i.i.d. returns with no + over-coverage. Caveats: mild small-sample over-rejection at short tracks, coverage restored toward + (not fully to) nominal at strong autocorrelation, and wider CIs - the honest power cost. Lo (2002) + is the autocorrelation-only intermediate, dominated by LPLZ under non-normality. + +Evidence and caveats: appraisals/08_verdict.md. + +References +---------- +Lopez de Prado, M., Lipton, A. and Zoonekynd, V. (2025) Sharpe-ratio inference under non-normal, + serially-correlated returns. (Open code at github.com/zoonek/2025-sharpe-ratio - NOT consulted or + vendored; this is reimplemented clean-room from the influence-function / HAC math, GOVERNANCE 3.) +Newey, W. K. and West, K. D. (1987) A simple, positive semi-definite, heteroskedasticity and + autocorrelation consistent covariance matrix. Econometrica, 55(3), 703-708. +Lo, A. W. (2002) The statistics of Sharpe ratios. Financial Analysts Journal, 58(4), 36-52. +""" + +from __future__ import annotations + +from typing import Optional + +import numpy as np +from scipy import stats as ss + + +def _sharpe(returns: np.ndarray) -> float: + r = np.asarray(returns, dtype=float) + sd = r.std(ddof=1) + return float(r.mean() / sd) if sd > 0 else 0.0 + + +def sharpe_ratio_influence_function(returns: np.ndarray) -> np.ndarray: + r""" + The Sharpe-ratio influence function :math:`\mathrm{IF}_t = z_t - \tfrac{1}{2}SR(z_t^2 - 1)`. + + Parameters + ---------- + returns : np.ndarray + The return series. + + Returns + ------- + np.ndarray + The per-observation influence function (mean ~ 0). + """ + r = np.asarray(returns, dtype=float) + mu = r.mean() + sigma = r.std(ddof=1) + if sigma == 0: + return np.zeros_like(r) + z = (r - mu) / sigma + sr = mu / sigma + return z - 0.5 * sr * (z**2 - 1.0) + + +def newey_west_long_run_variance(series: np.ndarray, lag: int) -> float: + r""" + Newey-West (Bartlett-kernel) long-run variance of a series. + + .. math:: + \widehat{\Omega} = \gamma_0 + 2 \sum_{k=1}^{L} \left(1 - \frac{k}{L+1}\right) \gamma_k, + + where :math:`\gamma_k` is the lag-k autocovariance. With ``lag == 0`` this is the sample variance. + + Parameters + ---------- + series : np.ndarray + The series whose long-run variance is estimated. + lag : int + The HAC bandwidth L (number of Bartlett lags). + + Returns + ------- + float + The estimated long-run variance (floored at a small positive number). + """ + x = np.asarray(series, dtype=float) + x = x - x.mean() + t = x.size + if t == 0: + return 1e-12 + total = float(x @ x) / t + for k in range(1, lag + 1): + weight = 1.0 - k / (lag + 1.0) + total += 2.0 * weight * float(x[k:] @ x[:-k]) / t + return max(total, 1e-12) + + +def newey_west_automatic_lag(number_of_returns: int) -> int: + r"""Newey-West automatic bandwidth :math:`\lfloor 4 (T/100)^{2/9} \rfloor` (at least 1).""" + return max(int(np.floor(4.0 * (number_of_returns / 100.0) ** (2.0 / 9.0))), 1) + + +def lplz_sharpe_inference( + returns: np.ndarray, + confidence_level: float = 0.95, + lag: Optional[int] = None, + null_sharpe_ratio: float = 0.0, +) -> dict[str, float]: + r""" + Lopez de Prado-Lipton-Zoonekynd (2025) Sharpe-ratio inference (HAC of the influence function). + + Returns the per-period Sharpe ratio, its autocorrelation-and-non-normality-robust standard error, + a confidence interval, and the test of ``null_sharpe_ratio``. The standard error is + :math:`\sqrt{\widehat{\Omega}/T}` where :math:`\widehat{\Omega}` is the Newey-West long-run variance + of the Sharpe influence function. + + Prefer this over the PSR when returns show material serial correlation and/or non-normality + (estimable from the sample): it restores near-nominal CI coverage and test size where the PSR + under-covers and over-rejects, and converges to the PSR on near-normal i.i.d. returns with no + over-coverage. The honest cost is a wider CI under autocorrelation. See the module docstring for the + full regime tag and appraisals/08_verdict.md. + + Parameters + ---------- + returns : np.ndarray + The strategy return series. + confidence_level : float, default 0.95 + The CI confidence level. + lag : int, optional + The HAC (Newey-West) bandwidth. Defaults to :func:`newey_west_automatic_lag`. + null_sharpe_ratio : float, default 0.0 + The Sharpe under the null hypothesis tested. + + Returns + ------- + dict + ``sharpe_ratio``, ``standard_error``, ``confidence_interval`` (a (lo, hi) tuple), + ``test_statistic``, ``p_value`` (two-sided), ``significant`` (reject the null at the level), + and ``lag`` (the bandwidth used). + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest import lplz_sharpe_inference + >>> rng = np.random.default_rng(0) + >>> result = lplz_sharpe_inference(0.1 + rng.standard_normal(240)) + >>> result["standard_error"] > 0 + True + """ + r = np.asarray(returns, dtype=float) + t = r.size + sr = _sharpe(r) + if t < 3 or r.std(ddof=1) == 0: + return { + "sharpe_ratio": sr, + "standard_error": float("nan"), + "confidence_interval": (float("nan"), float("nan")), + "test_statistic": float("nan"), + "p_value": float("nan"), + "significant": False, + "lag": 0, + } + if lag is None: + lag = newey_west_automatic_lag(t) + lrv = newey_west_long_run_variance(sharpe_ratio_influence_function(r), lag) + standard_error = float(np.sqrt(lrv / t)) + z = ss.norm.ppf(0.5 + confidence_level / 2.0) + ci = (sr - z * standard_error, sr + z * standard_error) + test_statistic = ( + (sr - null_sharpe_ratio) / standard_error + if standard_error > 0 + else float("nan") + ) + p_value = float(2.0 * ss.norm.sf(abs(test_statistic))) + return { + "sharpe_ratio": sr, + "standard_error": standard_error, + "confidence_interval": ci, + "test_statistic": float(test_statistic), + "p_value": p_value, + "significant": bool(p_value < (1.0 - confidence_level)), + "lag": int(lag), + } diff --git a/src/RiskLabAI/backtest/strategy_risk.py b/src/RiskLabAI/backtest/strategy_risk.py new file mode 100644 index 0000000..15e5a36 --- /dev/null +++ b/src/RiskLabAI/backtest/strategy_risk.py @@ -0,0 +1,379 @@ +""" +Functions for calculating strategy risk metrics, such as implied +precision, binomial Sharpe ratio, and probability of failure. +""" + +import logging +from typing import TYPE_CHECKING + +import numpy as np +import scipy.stats as ss + +if TYPE_CHECKING: + import sympy + +logger = logging.getLogger(__name__) + + +def sharpe_ratio_trials(p: float, n_run: int) -> tuple[float, float, float]: + r""" + Simulate binomial trials to estimate mean, std dev, and Sharpe ratio. + + Each trial is a binomial outcome (1 or -1) with probability `p`. + + .. math:: + \text{outcome} = \begin{cases} 1 & \text{with probability } p \\ + -1 & \text{with probability } 1-p + \end{cases} + SR = \frac{\mu}{\sigma} + + Parameters + ---------- + p : float + Probability of success (outcome 1). + n_run : int + Number of simulation runs (trials). + + Returns + ------- + Tuple[float, float, float] + (mean_outcome, std_outcome, sharpe_ratio) + """ + outcomes = np.random.binomial(n=1, p=p, size=n_run) * 2 - 1 + mean_outcome = np.mean(outcomes) + std_outcome = np.std(outcomes) + sharpe_ratio = mean_outcome / std_outcome if std_outcome > 0 else 0.0 + + return mean_outcome, std_outcome, sharpe_ratio + + +def target_sharpe_ratio_symbolic() -> "sympy.Expr": + r""" + Calculate the variance of a binomial trial symbolically. + + The variance is: + .. math:: + Var = E[X^2] - (E[X])^2 + Var = (p u^2 + (1 - p) d^2) - (p u + (1 - p) d)^2 + + This function simplifies this expression using SymPy. + + Returns + ------- + sympy.Expr + The symbolic expression for variance: `p*(1 - p)*(d - u)**2` + """ + try: + from sympy import factor, symbols + except ImportError as exc: + raise ImportError( + "Symbolic strategy-risk analysis requires the optional 'sympy' dependency." + ) from exc + + p, u, d = symbols("p u d") + + m2 = p * u**2 + (1 - p) * d**2 + m1 = p * u + (1 - p) * d + v = m2 - m1**2 + + return factor(v) + + +def implied_precision( + stop_loss: float, + profit_taking: float, + frequency: float, + target_sharpe_ratio: float, +) -> float: + r""" + Calculate the implied precision (probability) to achieve a target SR. + + Solves the quadratic equation for `p` from the binomial SR formula: + .. math:: + a = (f + S^2) (pt - sl)^2 + b = (2 f sl - S^2 (pt - sl)) (pt - sl) + c = f sl^2 + p = \frac{-b + \sqrt{b^2 - 4ac}}{2a} + + Where: + - \(f\) is the frequency (bets per year) + - \(S\) is the target_sharpe_ratio + - \(pt\) is the profit_taking + - \(sl\) is the stop_loss (as a positive value) + + Parameters + ---------- + stop_loss : float + Stop-loss threshold (positive value, e.g., 0.02 for 2%). + profit_taking : float + Profit-taking threshold (positive value, e.g., 0.04 for 4%). + frequency : float + Number of bets per year. + target_sharpe_ratio : float + Target annual Sharpe ratio. + + Returns + ------- + float + The required precision (probability of success). + """ + a = (frequency + target_sharpe_ratio**2) * (profit_taking - stop_loss) ** 2 + b = ( + 2 * frequency * stop_loss - target_sharpe_ratio**2 * (profit_taking - stop_loss) + ) * (profit_taking - stop_loss) + c = frequency * stop_loss**2 + + discriminant = b**2 - 4 * a * c + if discriminant < 0: + return np.nan # No real solution + + precision = (-b + np.sqrt(discriminant)) / (2.0 * a) + return precision + + +def bin_frequency( + stop_loss: float, + profit_taking: float, + precision: float, + target_sharpe_ratio: float, +) -> float: + r""" + Calculate the frequency needed to achieve a target SR. + + .. math:: + f = \frac{S^2 (pt - sl)^2 p (1 - p)}{((pt - sl) p - sl)^2} + + Note: The original formula had `((pt - sl) * p + sl)` in the denominator, + which corresponds to `sl` being negative (a loss). This + implementation assumes `stop_loss` is a positive value, so the + expected return is `(pt - sl) * p - sl * (1-p) = p*pt - p*sl - sl + p*sl = p*pt - sl`. + Wait, the original `binomial_sharpe_ratio` uses `(pt - sl) * p + sl`. + This implies `sl` is treated as a negative number. + Let's assume `stop_loss` is a positive threshold, so `d = -stop_loss`. + E[R] = p*pt + (1-p)*(-sl) = p*(pt+sl) - sl + Stdev = (pt - (-sl)) * sqrt(p(1-p)) = (pt+sl)*sqrt(p(1-p)) + SR = (p*(pt+sl) - sl) / ((pt+sl)*sqrt(p(1-p))) + Let's stick to the formula provided in `binomial_sharpe_ratio`. + + .. math:: + f = \frac{S^2 (pt - sl)^2 p (1-p)}{((pt - sl) p + sl)^2} + + Parameters + ---------- + stop_loss : float + Stop-loss threshold (e.g., -0.02 for 2% loss). + profit_taking : float + Profit-taking threshold (e.g., 0.04 for 4% gain). + precision : float + The precision (probability of a profitable trade). + target_sharpe_ratio : float + The target annual Sharpe ratio. + + Returns + ------- + float + The required number of bets per year (frequency). + """ + if precision <= 0 or precision >= 1: + return np.inf + + numerator = ( + (target_sharpe_ratio * (profit_taking - stop_loss)) ** 2 + * precision + * (1 - precision) + ) + denominator = ((profit_taking - stop_loss) * precision + stop_loss) ** 2 + + if denominator == 0: + return np.inf + + return numerator / denominator + + +def binomial_sharpe_ratio( + stop_loss: float, + profit_taking: float, + frequency: float, + probability: float, +) -> float: + r""" + Calculate the Sharpe Ratio for a binary outcome strategy. + + .. math:: + E[R] = p \cdot pt + (1-p) \cdot sl + \sigma[R] = (pt - sl) \sqrt{p(1-p)} + SR_{trade} = \frac{E[R]}{\sigma[R]} + SR_{annual} = SR_{trade} \times \sqrt{f} + + Parameters + ---------- + stop_loss : float + Stop-loss outcome (negative value, e.g., -0.02). + profit_taking : float + Profit-taking outcome (positive value, e.g., 0.04). + frequency : float + Number of bets per year. + probability : float + Probability of success (profit_taking). + + Returns + ------- + float + The annualized Sharpe Ratio. + """ + expected_return = (profit_taking * probability) + (stop_loss * (1 - probability)) + + p = probability + stdev_return = (profit_taking - stop_loss) * np.sqrt(p * (1 - p)) + + if stdev_return == 0: + return 0.0 if expected_return == 0 else np.inf * np.sign(expected_return) + + sr_trade = expected_return / stdev_return + sr_annual = sr_trade * np.sqrt(frequency) + + return sr_annual + + +def mix_gaussians( + mu1: float, + mu2: float, + sigma1: float, + sigma2: float, + probability: float, + n_obs: int, +) -> np.ndarray: + """ + Generate a mixture of two Gaussian-distributed outcomes. + + Parameters + ---------- + mu1 : float + Mean of the first Gaussian (e.g., winning trades). + mu2 : float + Mean of the second Gaussian (e.g., losing trades). + sigma1 : float + Standard deviation of the first Gaussian. + sigma2 : float + Standard deviation of the second Gaussian. + probability : float + Probability of drawing from the first Gaussian (e.g., prob of win). + n_obs : int + Total number of observations (trades) to generate. + + Returns + ------- + np.ndarray + Array of generated outcomes, shuffled. + """ + n_obs1 = int(n_obs * probability) + n_obs2 = n_obs - n_obs1 + + returns1 = np.random.normal(mu1, sigma1, size=n_obs1) + returns2 = np.random.normal(mu2, sigma2, size=n_obs2) + + returns = np.append(returns1, returns2, axis=0) + np.random.shuffle(returns) + + return returns + + +def failure_probability( + returns: np.ndarray, frequency: float, target_sharpe_ratio: float +) -> float: + """ + Calculate the probability that the strategy may fail. + + This compares the observed precision (from returns) with the + implied precision required to meet the target SR. + + Parameters + ---------- + returns : np.ndarray + Array of returns from a strategy. + frequency : float + Number of bets per year (observed). + target_sharpe_ratio : float + The target annual Sharpe ratio. + + Returns + ------- + float + Calculated failure probability (a Z-score CDF). + """ + positive_returns = returns[returns > 0] + negative_returns = returns[returns <= 0] + + if len(positive_returns) == 0 or len(negative_returns) == 0: + return 0.0 # Cannot calculate + + profit_taking = positive_returns.mean() + stop_loss = negative_returns.mean() # This will be negative + + # Observed precision + observed_precision = positive_returns.shape[0] / float(returns.shape[0]) + + # Required precision + required_precision = implied_precision( + abs(stop_loss), profit_taking, frequency, target_sharpe_ratio + ) + + if np.isnan(required_precision): + return 1.0 # Cannot achieve target SR + + # Probability that observed_precision < required_precision + # This is a test on a proportion + p_var = observed_precision * (1 - observed_precision) + if p_var == 0: + return 0.0 if observed_precision >= required_precision else 1.0 + + p_std = np.sqrt(p_var / returns.shape[0]) # Std dev of the proportion + + z_score = (observed_precision - required_precision) / p_std + risk = ss.norm.cdf(z_score) # Prob of being <= required_precision + + return risk + + +def calculate_strategy_risk( + mu1: float, + mu2: float, + sigma1: float, + sigma2: float, + probability: float, + n_obs: int, + frequency: float, + target_sharpe_ratio: float, +) -> float: + """ + Run a simulation to calculate the strategy risk. + + Parameters + ---------- + mu1 : float + Mean of winning trades. + mu2 : float + Mean of losing trades. + sigma1 : float + Standard deviation of winning trades. + sigma2 : float + Standard deviation of losing trades. + probability : float + Probability of a winning trade. + n_obs : int + Number of observations (trades) in the simulation. + frequency : float + Number of bets per year. + target_sharpe_ratio : float + Target annual Sharpe ratio. + + Returns + ------- + float + Calculated probability of strategy failure. + """ + returns = mix_gaussians(mu1, mu2, sigma1, sigma2, probability, n_obs) + probability_fail = failure_probability(returns, frequency, target_sharpe_ratio) + + logger.info("Probability that strategy will fail: %.2f%%", probability_fail * 100) + return probability_fail diff --git a/src/RiskLabAI/backtest/test_set_overfitting.py b/src/RiskLabAI/backtest/test_set_overfitting.py new file mode 100644 index 0000000..3c8afaf --- /dev/null +++ b/src/RiskLabAI/backtest/test_set_overfitting.py @@ -0,0 +1,320 @@ +""" +Functions for calculating metrics related to test-set overfitting, +including the expected maximum Sharpe Ratio and Type I/II errors. +""" + +import numpy as np +import pandas as pd +import scipy.stats as ss +from scipy.stats import norm + + +def expected_max_sharpe_ratio( + n_trials: int, mean_sharpe_ratio: float, std_sharpe_ratio: float +) -> float: + r""" + Calculate the expected maximum Sharpe Ratio from N trials. + + Uses the formula for the expected maximum of N standard normal variables: + .. math:: + E[\max(SR)] = \mu + \sigma \left[ (1 - \gamma) \Phi^{-1}(1 - \frac{1}{N}) + + \gamma \Phi^{-1}(1 - (N e)^{-1}) \right] + + Where: + - \(\mu\) is the mean_sharpe_ratio + - \(\sigma\) is the std_sharpe_ratio + - \(\gamma\) is the Euler-Mascheroni constant + - \(\Phi^{-1}\) is the inverse CDF of the standard normal distribution + - \(N\) is n_trials + + Parameters + ---------- + n_trials : int + Number of trials (e.g., number of strategies tested). + mean_sharpe_ratio : float + The mean Sharpe Ratio across all trials. + std_sharpe_ratio : float + The standard deviation of Sharpe Ratios across all trials. + + Returns + ------- + float + The expected maximum Sharpe Ratio. + """ + if n_trials == 0: + return 0.0 + if n_trials == 1: + return mean_sharpe_ratio + + euler_gamma = 0.5772156649 + + term1 = (1 - euler_gamma) * norm.ppf(1.0 - 1.0 / n_trials) + term2 = euler_gamma * norm.ppf(1.0 - (n_trials * np.e) ** -1) + + expected_max_sr = mean_sharpe_ratio + std_sharpe_ratio * (term1 + term2) + + return expected_max_sr + + +def generate_max_sharpe_ratios( + n_sims: int, + n_trials_list: list[int], + std_sharpe_ratio: float, + mean_sharpe_ratio: float, +) -> pd.DataFrame: + """ + Generate a DataFrame of maximum Sharpe Ratios from simulations. + + Parameters + ---------- + n_sims : int + Number of simulations to run for each `n_trials`. + n_trials_list : List[int] + A list of N-trials (e.g., [10, 50, 100] strategies). + std_sharpe_ratio : float + The standard deviation of Sharpe Ratios. + mean_sharpe_ratio : float + The mean of Sharpe Ratios. + + Returns + ------- + pd.DataFrame + A long-format DataFrame with columns ['max_SR', 'n_trials']. + """ + rng = np.random.default_rng() + output_list = [] + + for n_trials in n_trials_list: + # Generate all simulations for this n_trials + sr_sims = rng.normal(loc=0.0, scale=1.0, size=(n_sims, n_trials)) + + # Normalize (z-score) each simulation row + sr_sims = (sr_sims - sr_sims.mean(axis=1, keepdims=True)) / sr_sims.std( + axis=1, keepdims=True + ) + + # Scale by target mean and std + sr_sims = mean_sharpe_ratio + sr_sims * std_sharpe_ratio + + # Get max SR for each simulation + max_sr = sr_sims.max(axis=1) + + output_temp = pd.DataFrame({"max_SR": max_sr, "n_trials": n_trials}) + output_list.append(output_temp) + + return pd.concat(output_list, ignore_index=True) + + +def mean_std_error( + n_sims0: int, + n_sims1: int, + n_trials: list[int], + std_sharpe_ratio: float = 1.0, + mean_sharpe_ratio: float = 0.0, +) -> pd.DataFrame: + """ + Calculate the mean and standard deviation of the estimation error. + + This function compares the analytical E[max{SR}] with the simulated + average max{SR} over `n_sims1` repetitions. + + Parameters + ---------- + n_sims0 : int + Number of max{SR} simulations used to estimate E[max{SR}] (inner loop). + n_sims1 : int + Number of errors on which to compute the std dev (outer loop). + n_trials : List[int] + List of numbers of trials (e.g., [10, 20, 50]). + std_sharpe_ratio : float, default=1.0 + Standard deviation of Sharpe Ratios. + mean_sharpe_ratio : float, default=0.0 + Mean of Sharpe Ratios. + + Returns + ------- + pd.DataFrame + DataFrame indexed by 'nTrials' with columns ['meanErr', 'stdErr']. + """ + # 1. Analytical E[max{SR}] + expected_sr = pd.Series( + { + n: expected_max_sharpe_ratio(n, mean_sharpe_ratio, std_sharpe_ratio) + for n in n_trials + }, + name="E[max{SR}]", + ) + expected_sr.index.name = "nTrials" + + error_list = [] + + # 2. Run n_sims1 experiments + for _ in range(int(n_sims1)): + # 3. Generate simulated max SRs + simulated_sr = generate_max_sharpe_ratios( + n_sims=n_sims0, + n_trials_list=n_trials, + mean_sharpe_ratio=mean_sharpe_ratio, + std_sharpe_ratio=std_sharpe_ratio, + ) + # 4. Average simulated max SRs + avg_simulated_sr = simulated_sr.groupby("n_trials")["max_SR"].mean() + + # 5. Calculate error + error = avg_simulated_sr / expected_sr - 1.0 + error_list.append(error.rename("error")) + + all_errors = pd.concat(error_list, axis=1).T + + # 6. Compute mean and std of errors + output = pd.DataFrame( + { + "meanErr": all_errors.mean(), + "stdErr": all_errors.std(), + } + ) + + return output + + +def estimated_sharpe_ratio_z_statistics( + sharpe_ratio: float, + t: int, + true_sharpe_ratio: float = 0.0, + skew: float = 0.0, + kurt: int = 3, +) -> float: + r""" + Calculate the Z-statistic for an estimated Sharpe Ratio. + + This is the test statistic used in the Probabilistic Sharpe Ratio. + + .. math:: + Z = \frac{(\hat{SR} - SR_0) \sqrt{T - 1}} + {\sqrt{1 - S \hat{SR} + \frac{K - 1}{4} \hat{SR}^2}} + + Parameters + ---------- + sharpe_ratio : float + Estimated Sharpe Ratio (\(\hat{SR}\)). + t : int + Number of observations (T). + true_sharpe_ratio : float, default=0.0 + The null hypothesis Sharpe Ratio (\(SR_0\)). + skew : float, default=0.0 + Skewness of returns (S). + kurt : int, default=3 + Kurtosis of returns (K). + + Returns + ------- + float + The calculated Z-statistic. + """ + denominator = 1 - skew * sharpe_ratio + (kurt - 1) / 4.0 * sharpe_ratio**2 + if denominator <= 0: + return np.nan + + z = (sharpe_ratio - true_sharpe_ratio) * np.sqrt(t - 1) + z /= np.sqrt(denominator) + + return z + + +def strategy_type1_error_probability(z: float, k: int = 1) -> float: + r""" + Calculate the Type I error probability (alpha) for multiple tests. + + This is the probability of at least one false positive (rejecting + a true null) when conducting `k` independent tests. + + .. math:: + \alpha_k = 1 - (1 - \alpha)^k + + Where \(\alpha = N(-z)\) is the Type I error for a single test. + + Parameters + ---------- + z : float + Z-statistic for the significance threshold (e.g., 1.96). + k : int, default=1 + Number of independent tests. + + Returns + ------- + float + The family-wise Type I error rate. + """ + alpha_single_test = ss.norm.cdf(-z) + alpha_k = 1 - (1 - alpha_single_test) ** k + return alpha_k + + +def theta_for_type2_error( + sharpe_ratio: float, + t: int, + true_sharpe_ratio: float, + skew: float = 0.0, + kurt: int = 3, +) -> float: + r""" + Calculate the \(\theta\) parameter for Type II error probability. + + .. math:: + \theta = \frac{SR_{True} \sqrt{T - 1}} + {\sqrt{1 - S \hat{SR} + \frac{K - 1}{4} \hat{SR}^2}} + + Parameters + ---------- + sharpe_ratio : float + The estimated Sharpe Ratio (\(\hat{SR}\)). + t : int + Number of observations (T). + true_sharpe_ratio : float + The true Sharpe Ratio (\(SR_{True}\)) (the alternative hypothesis). + skew : float, default=0.0 + Skewness of returns (S). + kurt : int, default=3 + Kurtosis of returns (K). + + Returns + ------- + float + The \(\theta\) parameter. + """ + denominator = 1 - skew * sharpe_ratio + (kurt - 1) / 4.0 * sharpe_ratio**2 + if denominator <= 0: + return np.nan + + theta = true_sharpe_ratio * np.sqrt(t - 1) + theta /= np.sqrt(denominator) + return theta + + +def strategy_type2_error_probability(alpha_k: float, k: int, theta: float) -> float: + r""" + Calculate the Type II error probability (beta) for multiple tests. + + This is the probability of failing to reject a false null hypothesis. + + .. math:: + Z_{\alpha} = \Phi^{-1}((1 - \alpha_k)^{1/k}) + \beta = N(Z_{\alpha} - \theta) + + Parameters + ---------- + alpha_k : float + The family-wise Type I error rate. + k : int + Number of independent tests. + theta : float + The \(\theta\) parameter from `theta_for_type2_error`. + + Returns + ------- + float + The Type II error probability (\(\beta\)). + """ + z_alpha = ss.norm.ppf((1 - alpha_k) ** (1.0 / k)) + beta = ss.norm.cdf(z_alpha - theta) + return beta diff --git a/src/RiskLabAI/backtest/validation/__init__.py b/src/RiskLabAI/backtest/validation/__init__.py new file mode 100644 index 0000000..385b4a4 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/__init__.py @@ -0,0 +1,55 @@ +""" +RiskLabAI Cross-Validation Module + +Implements advanced financial cross-validation techniques from +"Advances in Financial Machine Learning" by de Prado, including: +- Purged K-Fold (for removing group leakage) +- Walk-Forward +- Combinatorial Purged K-Fold (CPSC-V) + +This module is built on an AbstractCrossValidator interface and +a CrossValidatorFactory for easy instantiation. +""" + +from .adaptive_combinatorial_purged import AdaptiveCombinatorialPurged +from .bagged_combinatorial_purged import BaggedCombinatorialPurged +from .combinatorial_purged import CombinatorialPurged +from .cross_validator_controller import CrossValidatorController +from .cross_validator_factory import CrossValidatorFactory +from .cross_validator_interface import CrossValidator +from .kfold import KFold +from .leakage_aware_hpo import deflated_sharpe_gate, leakage_aware_hpo +from .path_adaptive_cpcv import ( + adaptive_probability_of_backtest_overfitting, + estimate_volatility_regimes, +) +from .path_bagged_cpcv import ( + bagged_probability_of_backtest_overfitting, + moving_block_bootstrap_indices, +) +from .purged_kfold import PurgedKFold +from .walk_forward import WalkForward + +__all__ = [ + # Interface + "CrossValidator", + # Validators + "KFold", + "PurgedKFold", + "WalkForward", # <-- Fix + "CombinatorialPurged", # <-- Fix + "BaggedCombinatorialPurged", # <-- Fix + "AdaptiveCombinatorialPurged", # <-- Fix + # Path-level bagged CPCV PBO (Appraisal 09) + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices", + # Path-level adaptive (regime-weighted) CPCV PBO (Appraisal 09b) + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes", + # Leakage-aware HPO methodology (Appraisal 20) + "leakage_aware_hpo", + "deflated_sharpe_gate", + # Utilities + "CrossValidatorFactory", + "CrossValidatorController", +] diff --git a/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py b/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py new file mode 100644 index 0000000..7e81bc0 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py @@ -0,0 +1,472 @@ +""" +Implements an Adaptive Combinatorial Purged Cross-Validation (A-CPCV), +which adjusts split boundaries based on an external feature. +""" + +import warnings +from collections import ChainMap +from collections.abc import Generator +from itertools import combinations +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.base import clone +from sklearn.exceptions import ConvergenceWarning + +from .combinatorial_purged import CombinatorialPurged + +# For type hinting sklearn-like estimators +Estimator = Any + + +class AdaptiveCombinatorialPurged(CombinatorialPurged): + """ + Adaptive Combinatorial Purged Cross-Validation (A-CPCV). + + This method extends CPCV by making the group splits "adaptive". + Instead of fixed-size groups, it adjusts the group boundaries + based on the quantiles of an external feature (e.g., volatility, + market stress indicator). + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + n_subsplits : int, default=3 + Number of subsplits within each group to check for boundary + adjustment. Higher numbers allow for finer adjustments. + external_feature : pd.Series or Dict[str, pd.Series] + The external feature used to adjust split boundaries. + Must share the same index as `data` and `times`. + lower_quantile : float, default=0.25 + The lower quantile threshold. If a boundary point's + feature value is below this, the boundary is shifted. + upper_quantile : float, default=0.75 + The upper quantile threshold. If a boundary point's + feature value is above this, the boundary is shifted. + subtract_border_adjustments : bool, default=True + If True, shifts boundaries left (subtracts) for low + quantile values and right (adds) for high. If False, + does the opposite. + """ + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + n_subsplits: int = 3, + external_feature: Union[pd.Series, dict[str, pd.Series]] = None, + lower_quantile: float = 0.25, + upper_quantile: float = 0.75, + subtract_border_adjustments: bool = True, + ): + """ + Initialize the AdaptiveCombinatorialPurged class. + """ + super().__init__(n_splits, n_test_groups, times, embargo) + + if external_feature is None: + raise ValueError("external_feature must be provided for A-CPCV") + + self.n_subsplits = n_subsplits + self.external_feature = external_feature + self.lower_quantile = lower_quantile + self.upper_quantile = upper_quantile + self.subtract_border_adjustments = subtract_border_adjustments + + # Check for multiple datasets consistency + if self.is_multiple_datasets and not isinstance(external_feature, dict): + raise ValueError( + "If 'times' is a dict, 'external_feature' must also be a dict." + ) + if not self.is_multiple_datasets and not isinstance( + external_feature, pd.Series + ): + raise ValueError( + "If 'times' is a Series, 'external_feature' must be a Series." + ) + + def _validate_input( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: pd.Series, + ) -> None: + """ + Validate that data, times, and external feature share the same index. + """ + # Call parent validation + super()._validate_input(single_times, single_data) + + if not single_data.index.equals(single_external_feature.index): + raise ValueError("Data and external_feature must have the same index") + + def _single_adaptive_split_segments( + self, indices: np.ndarray, single_external_feature: pd.Series + ) -> list[np.ndarray]: + """ + Adaptively split data indices based on the external feature. + + Parameters + ---------- + indices : np.ndarray + Array of data indices to be split. + single_external_feature : pd.Series + The external feature for the entire dataset. + + Returns + ------- + List[np.ndarray] + List of adaptively split data indices (the `n_splits` groups). + """ + # Get feature values for the relevant indices + external_values = single_external_feature.iloc[indices].values + + # Calculate quantile thresholds + lower_threshold = np.percentile(external_values, self.lower_quantile * 100) + upper_threshold = np.percentile(external_values, self.upper_quantile * 100) + + # Partition indices into fine-grained subsplits + n_total_subsplits = self.n_splits * self.n_subsplits + subsplits = np.array_split(indices, n_total_subsplits) + + # Get the start index of each subsplit + subsplit_starts_loc = np.array( + [split[0] for split in subsplits if len(split) > 0] + ) + + # Get the feature values at the start of each subsplit + # We need to map iloc back to the feature series's index + subsplit_start_indices = single_external_feature.index[subsplit_starts_loc] + subsplit_start_features = single_external_feature[subsplit_start_indices].values + + # Determine the initial (non-adaptive) borders + # These are the indices of the *subsplits* that start a new group + borders = np.arange(self.n_subsplits, n_total_subsplits, self.n_subsplits) + + # Vectorized comparison of external feature values at borders + border_feature_values = subsplit_start_features[borders] + + # Determine adjustments + border_adjustments = np.zeros_like(borders) + border_adjustments[border_feature_values < lower_threshold] = -1 + border_adjustments[border_feature_values > upper_threshold] = 1 + + if self.subtract_border_adjustments: + adjusted_borders = borders - border_adjustments + else: + adjusted_borders = borders + border_adjustments + + # Ensure borders are within valid range + adjusted_borders = np.clip(adjusted_borders, 1, len(subsplit_starts_loc) - 1) + + # Get the integer-location split points + split_points = subsplit_starts_loc[adjusted_borders] + + # Create the final N-split segments + # We split the original `indices` array at the calculated `split_points` + # `np.searchsorted` maps the split point values back to their locations in `indices` + split_point_locs = np.searchsorted(indices, split_points) + split_segments = np.split(indices, split_point_locs) + + return split_segments + + def _get_split_segments( + self, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> list[np.ndarray]: + """Override to use adaptive splitting.""" + if single_external_feature is None: + raise ValueError("_get_split_segments requires external_feature") + + indices = np.arange(single_data.shape[0]) + return self._single_adaptive_split_segments(indices, single_external_feature) + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into C(n, k) adaptively purged indices. + """ + if single_external_feature is None: + raise ValueError("_single_split requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + split_segments = self._get_split_segments(single_data, single_external_feature) + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + + all_combinatorial_splits = self._combinatorial_splits( + combinations_list, split_segments + ) + + for test_indices in all_combinatorial_splits: + train_indices = self._get_train_indices( + test_indices, single_times, continous_test_times=False + ) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into adaptively purged indices. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must be a dict.") + for key in self.times: + for train_indices, test_indices in self._single_split( + self.times[key], data[key], self.external_feature[key] + ): + yield key, (train_indices, test_indices) + else: + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + for train_indices, test_indices in self._single_split( + self.times, data, self.external_feature + ): + yield train_indices, test_indices + + def _combinations_and_path_locations_and_split_segments( + self, data: pd.DataFrame, single_external_feature: Optional[pd.Series] = None + ) -> tuple[ + list[tuple[int, ...]], dict[int, list[tuple[int, int]]], list[np.ndarray] + ]: + """Helper to compute all components, now including adaptive splits.""" + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + locations = self._path_locations(self.n_splits, combinations_list) + split_segments = self._get_split_segments(data, single_external_feature) + + return combinations_list, locations, split_segments + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate all adaptive combinatorial backtest paths. + """ + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + paths = {} + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments( + single_data, single_external_feature + ) + ) + + all_combinatorial_splits = list( + self._combinatorial_splits(combinations_list, split_segments) + ) + + for path_num, locs in locations.items(): + path_data = [] + for group_idx, split_idx in locs: + combinatorial_test_indices = all_combinatorial_splits[split_idx] + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + test_indices_segment = split_segments[group_idx] + + path_data.append( + { + "Train": train_indices, + "Test": test_indices_segment, + } + ) + + paths[f"Path {path_num}"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate adaptive backtest paths for data or a dictionary of data. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must be a dict.") + multiple_paths = {} + for key in self.times: + multiple_paths[key] = self._single_backtest_paths( + self.times[key], data[key], self.external_feature[key] + ) + return multiple_paths + + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + return self._single_backtest_paths(self.times, data, self.external_feature) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + single_external_feature: Optional[pd.Series] = None, # New arg + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all A-CPCV paths. + """ + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments( + single_data, single_external_feature + ) + ) + + def train_single_estimator( + estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except (TypeError, ValueError): + estimator_.fit(X_train, y_train) + + return estimator_ + + # 1. Train all C(n, k) estimators in parallel + combinatorial_trained_estimators = Parallel(n_jobs=n_jobs)( + delayed(train_single_estimator)(clone(single_estimator), test_indices) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + for group_idx, split_idx in locs: + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + # 2. Assemble predictions for all paths + path_results = Parallel(n_jobs=n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate adaptive backtest predictions. + """ + if self.is_multiple_datasets: + if not ( + isinstance(data, dict) + and isinstance(estimator, dict) + and isinstance(labels, dict) + ): + raise ValueError( + "If 'times' is a dict, 'data', 'estimator', and 'labels' " + "must also be dicts." + ) + + multiple_paths_predictions = {} + for key in self.times: + s_weights = sample_weights[key] if sample_weights else None + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + self.times[key], + data[key], + labels[key], + s_weights, + self.external_feature[key], + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If 'times' is a Series, 'data' must be a DataFrame " + "and 'labels' must be a Series." + ) + + return self._single_backtest_predictions( + estimator, + self.times, + data, + labels, + sample_weights, + self.external_feature, + predict_probability, + n_jobs, + ) diff --git a/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py b/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py new file mode 100644 index 0000000..27d8d45 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py @@ -0,0 +1,247 @@ +""" +Implements Bagged Combinatorial Purged Cross-Validation (B-CPCV). +""" + +import warnings +from collections import ChainMap +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.base import clone +from sklearn.ensemble import BaggingClassifier, BaggingRegressor +from sklearn.exceptions import ConvergenceWarning + +from .combinatorial_purged import CombinatorialPurged + +# For type hinting sklearn-like estimators +Estimator = Any + + +class BaggedCombinatorialPurged(CombinatorialPurged): + """ + Bagged Combinatorial Purged Cross-Validation (B-CPCV). + + This class extends CPCV by applying a bagging (Bootstrap Aggregating) + wrapper to the estimator for each combinatorial split. This helps + to reduce variance and improve model stability. + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + classifier : bool, default=True + If True, use `sklearn.ensemble.BaggingClassifier`. + If False, use `sklearn.ensemble.BaggingRegressor`. + n_estimators : int, default=10 + The number of base estimators in the ensemble. + max_samples : float, default=1.0 + The number of samples to draw to train each base estimator. + max_features : float, default=1.0 + The number of features to draw to train each base estimator. + bootstrap : bool, default=True + Whether samples are drawn with replacement. + bootstrap_features : bool, default=False + Whether features are drawn with replacement. + random_state : int, optional + Controls the randomness of the bootstrapping. + """ + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + classifier: bool = True, + n_estimators: int = 10, + max_samples: float = 1.0, + max_features: float = 1.0, + bootstrap: bool = True, + bootstrap_features: bool = False, + random_state: int = None, + ): + """ + Initialize the BaggedCombinatorialPurged class. + """ + super().__init__(n_splits, n_test_groups, times, embargo) + self.classifier = classifier + self.n_estimators = n_estimators + self.max_samples = max_samples + self.max_features = max_features + self.bootstrap = bootstrap + self.bootstrap_features = bootstrap_features + self.random_state = random_state + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all B-CPCV paths. + + This method overrides the parent by wrapping the `single_estimator` + in a `BaggingClassifier` or `BaggingRegressor` before training. + + Parameters + ---------- + single_estimator : Estimator + The *base* estimator to be used inside the bagging ensemble. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + Only valid if `classifier=True`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are "Path {id}" and values are + the contiguous arrays of out-of-sample predictions for each path. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + if predict_probability and not self.classifier: + raise ValueError( + "Cannot use predict_probability=True when classifier=False" + ) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + def train_single_bagging_estimator( + base_estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one Bagging estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + # --- Bagging Wrapper --- + if self.classifier: + bagging_estimator = BaggingClassifier( + estimator=base_estimator_, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + bootstrap=self.bootstrap, + bootstrap_features=self.bootstrap_features, + random_state=self.random_state, + n_jobs=n_jobs, # Parallelize bagging itself + ) + else: + bagging_estimator = BaggingRegressor( + estimator=base_estimator_, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + bootstrap=self.bootstrap, + bootstrap_features=self.bootstrap_features, + random_state=self.random_state, + n_jobs=n_jobs, # Parallelize bagging itself + ) + # --- End Bagging Wrapper --- + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + bagging_estimator.fit(X_train, y_train, sample_weight=weights_train) + except (TypeError, ValueError): + # Fallback for models without sample_weight + bagging_estimator.fit(X_train, y_train) + + return bagging_estimator + + # 1. Train all C(n, k) *bagging* estimators in parallel + # Note: We set n_jobs=1 for the *outer* parallel loop + # and let the *inner* bagging estimator use `n_jobs`. + # This avoids nested parallelization issues. + + # Determine parallelization strategy + if n_jobs > 1 or n_jobs == -1: + outer_n_jobs = n_jobs + inner_n_jobs = 1 + + # Update bagging params to use inner_n_jobs + if self.classifier: + BaggingClassifier.__init__ = lambda self, **kwargs: super( + BaggingClassifier, self + ).__init__(**kwargs, n_jobs=inner_n_jobs) + else: + BaggingRegressor.__init__ = lambda self, **kwargs: super( + BaggingRegressor, self + ).__init__(**kwargs, n_jobs=inner_n_jobs) + else: + # Let bagging use all cores if outer loop is serial + outer_n_jobs = 1 + inner_n_jobs = -1 # Use all + + combinatorial_trained_estimators = Parallel(n_jobs=outer_n_jobs)( + delayed(train_single_bagging_estimator)( + clone(single_estimator), test_indices + ) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + # Restore default constructors + BaggingClassifier.__init__ = BaggingClassifier.__init__ + BaggingRegressor.__init__ = BaggingRegressor.__init__ + + # 2. Assemble predictions (this is fast, can be serial or parallel) + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + for group_idx, split_idx in locs: + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + path_results = Parallel(n_jobs=outer_n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions diff --git a/src/RiskLabAI/backtest/validation/combinatorial_purged.py b/src/RiskLabAI/backtest/validation/combinatorial_purged.py new file mode 100644 index 0000000..e6a0a25 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/combinatorial_purged.py @@ -0,0 +1,413 @@ +""" +Implements Combinatorial Purged Cross-Validation (CPCV) as described by +Marcos Lopez de Prado. +""" + +import warnings +from collections import ChainMap, defaultdict +from collections.abc import Generator +from copy import deepcopy +from itertools import combinations +from math import comb +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .purged_kfold import PurgedKFold + +# For type hinting sklearn-like estimators +Estimator = Any + + +class CombinatorialPurged(PurgedKFold): + """ + Combinatorial Purged Cross-Validation (CPCV). + + This method generates multiple backtest paths by creating all possible + combinations of train/test splits, given `n_splits` groups and + `n_test_groups` for the test set size. + + It inherits from `PurgedKFold` to ensure that all generated training + sets are properly purged and embargoed against their corresponding + test set. + + Parameters + ---------- + n_splits : int + Total number of groups to partition the data into. + n_test_groups : int + The number of groups to use for testing in each combination. + e.g., n_splits=8, n_test_groups=2 -> C(8, 2) = 28 splits. + times : pd.Series or Dict[str, pd.Series] + A Series where the index is the observation start time and + the value is the observation end time. + embargo : float, default=0 + The embargo fraction (e.g., 0.01 for 1%). + """ + + @staticmethod + def _path_locations( + n_splits: int, combinations_list: list[tuple[int, ...]] + ) -> dict[int, list[tuple[int, int]]]: + """ + Generate a labeled path matrix to map splits to backtest paths. + + Parameters + ---------- + n_splits : int + Total number of groups. + combinations_list : List[Tuple[int, ...]] + The list of all C(n_splits, n_test_groups) combinations. + + Returns + ------- + Dict[int, List[Tuple[int, int]]] + A dictionary mapping each path ID (int) to a list of its + coordinates (group, split_num) in the path matrix. + """ + n_combinations = len(combinations_list) + + # Initialize a zero matrix + matrix = np.zeros((n_splits, n_combinations), dtype=int) + + # Populate matrix: 1 if group `i` is in combination `j` + for j, indices in enumerate(combinations_list): + matrix[indices, j] = 1 + + # Label path numbers + def label_path_row(row: np.ndarray) -> np.ndarray: + """Assigns a path number to each test split in a group.""" + path_counter = 1 + labeled_row = np.zeros_like(row) + for i, val in enumerate(row): + if val == 1: + labeled_row[i] = path_counter + path_counter += 1 + return labeled_row + + path_numbers = np.apply_along_axis(label_path_row, 1, matrix) + + # Extract path locations + locations = defaultdict(list) + for group_idx in range(n_splits): + for split_idx in range(n_combinations): + path_id = path_numbers[group_idx, split_idx] + if path_id != 0: + locations[path_id].append((group_idx, split_idx)) + + return dict(locations) + + @staticmethod + def _combinatorial_splits( + combinations_list: list[tuple[int, ...]], split_segments: list[np.ndarray] + ) -> Generator[np.ndarray, None, None]: + """ + Generate combinatorial test sets. + + Parameters + ---------- + combinations_list : List[Tuple[int, ...]] + List of all C(n_splits, n_test_groups) combinations. + split_segments : List[np.ndarray] + List of index arrays, one for each of the `n_splits` groups. + + Yields + ------- + Generator[np.ndarray, None, None] + A generator yielding the concatenated test set indices for + each combination. + """ + for test_groups in combinations_list: + test_sets = [ + split for i, split in enumerate(split_segments) if i in test_groups + ] + yield np.concatenate(test_sets) + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + ) -> None: + """ + Initialize the CombinatorialPurged class. + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + """ + super().__init__(n_splits, times, embargo) + if n_test_groups >= n_splits: + raise ValueError("n_test_groups must be strictly less than n_splits") + self.n_test_groups = n_test_groups + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the total number of combinatorial splits. + + This is C(n_splits, n_test_groups). + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The total number of splits, C(n_splits, n_test_groups). + """ + return comb(self.n_splits, self.n_test_groups) + + def _get_split_segments(self, single_data: pd.DataFrame) -> list[np.ndarray]: + """Helper to get the base K-Fold segments.""" + indices = np.arange(single_data.shape[0]) + return np.array_split(indices, self.n_splits) + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into C(n, k) purged train-test indices. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator of (train_indices, test_indices) for all + C(n_splits, n_test_groups) combinations. + """ + self._validate_input(single_times, single_data) + + split_segments = self._get_split_segments(single_data) + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + + all_combinatorial_splits = self._combinatorial_splits( + combinations_list, split_segments + ) + + for test_indices in all_combinatorial_splits: + # Purge against the *non-contiguous* test set + train_indices = self._get_train_indices( + test_indices, single_times, continous_test_times=False + ) + yield train_indices, test_indices + + def _combinations_and_path_locations_and_split_segments( + self, data: pd.DataFrame + ) -> tuple[ + list[tuple[int, ...]], dict[int, list[tuple[int, int]]], list[np.ndarray] + ]: + """ + Helper to compute all necessary components for CPCV. + + Parameters + ---------- + data : pd.DataFrame + Input dataset. + + Returns + ------- + Tuple[List, Dict, List] + - combinations_list: List of all C(n, k) combinations. + - locations: The path location dictionary from `_path_locations`. + - split_segments: List of index arrays for each of the `n_splits` groups. + """ + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + locations = self._path_locations(self.n_splits, combinations_list) + split_segments = self._get_split_segments(data) + + return combinations_list, locations, split_segments + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate all combinatorial backtest paths for a single dataset. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary where each key is a "Path {id}" and the value + is the list of (Train, Test) splits for that path. + """ + self._validate_input(single_times, single_data) + + paths = {} + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + all_combinatorial_splits = list( + self._combinatorial_splits(combinations_list, split_segments) + ) + + for path_num, locs in locations.items(): + path_data = [] + for group_idx, split_idx in locs: + # Get the full test set for this *combination* + combinatorial_test_indices = all_combinatorial_splits[split_idx] + + # Get the train set purged against this *combination* + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + # The test set for this *path segment* is just one group + test_indices_segment = split_segments[group_idx] + + path_data.append( + { + "Train": train_indices, + "Test": test_indices_segment, + } + ) + + # This path is complete + paths[f"Path {path_num}"] = path_data + + return paths + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all CPCV paths. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are "Path {id}" and values are + the contiguous arrays of out-of-sample predictions for each path. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + def train_single_estimator( + estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + return estimator_ + + # 1. Train all C(n, k) estimators in parallel + combinatorial_trained_estimators = Parallel(n_jobs=n_jobs)( + delayed(train_single_estimator)(deepcopy(single_estimator), test_indices) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + + for group_idx, split_idx in locs: + # Get the test segment for this path + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + + # Get the estimator trained for this split + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + # 2. Assemble predictions for all paths in parallel + path_results = Parallel(n_jobs=n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + # Combine list of dicts into one dict + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions diff --git a/src/RiskLabAI/backtest/validation/cross_validator_controller.py b/src/RiskLabAI/backtest/validation/cross_validator_controller.py new file mode 100644 index 0000000..56820a5 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_controller.py @@ -0,0 +1,38 @@ +""" +Deprecated controller for creating and holding a cross-validator. + +Use the core component registry (:data:`RiskLabAI.core.CROSS_VALIDATORS`) +directly instead. This controller is retained for backward compatibility and +delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +from .cross_validator_interface import CrossValidator + + +class CrossValidatorController: + """ + Deprecated. Use ``RiskLabAI.core.CROSS_VALIDATORS.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + def __init__(self, validator_type: str, **kwargs: Any): + warnings.warn( + "CrossValidatorController is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.CROSS_VALIDATORS.create(validator_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import CROSS_VALIDATORS + + self.cross_validator: CrossValidator = CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **kwargs + ) + + def get_validator(self) -> CrossValidator: + """Return the created cross-validator instance.""" + return self.cross_validator diff --git a/src/RiskLabAI/backtest/validation/cross_validator_factory.py b/src/RiskLabAI/backtest/validation/cross_validator_factory.py new file mode 100644 index 0000000..ee34e0a --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_factory.py @@ -0,0 +1,74 @@ +""" +Deprecated factory for creating cross-validator instances. + +The core component registry (:data:`RiskLabAI.core.CROSS_VALIDATORS`) is now the +single source of truth for creating cross-validators. This factory is retained +for backward compatibility and simply delegates to that registry; it is removed +in 4.0.0. +""" + +import warnings +from typing import Any + +from .adaptive_combinatorial_purged import AdaptiveCombinatorialPurged +from .bagged_combinatorial_purged import BaggedCombinatorialPurged +from .combinatorial_purged import CombinatorialPurged +from .cross_validator_interface import CrossValidator +from .kfold import KFold +from .purged_kfold import PurgedKFold +from .walk_forward import WalkForward + + +class CrossValidatorFactory: + """ + Deprecated. Use ``RiskLabAI.core.CROSS_VALIDATORS.create(...)`` instead. + + Retained for backward compatibility (scheduled for removal in 4.0.0). ``VALIDATORS`` is + kept so existing introspection keeps working; ``create_cross_validator`` + now delegates to the core registry. + """ + + VALIDATORS = { + "kfold": KFold, + "walkforward": WalkForward, + "purgedkfold": PurgedKFold, + "combinatorialpurged": CombinatorialPurged, + "baggedcombinatorialpurged": BaggedCombinatorialPurged, + "adaptivecombinatorialpurged": AdaptiveCombinatorialPurged, + } + + @staticmethod + def create_cross_validator(validator_type: str, **kwargs: Any) -> CrossValidator: + """ + Deprecated. Create a cross-validator via the core registry. + + Parameters + ---------- + validator_type : str + One of the keys in :attr:`VALIDATORS` (case-insensitive). + **kwargs : Any + Forwarded to the validator's constructor; arguments it does not + accept are dropped (matching the historical behaviour). + + Raises + ------ + ValueError + If ``validator_type`` is not a known validator. + """ + warnings.warn( + "CrossValidatorFactory is deprecated and will be removed in 4.0.0; " + "use RiskLabAI.core.CROSS_VALIDATORS.create(validator_type, ...) " + "instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import CROSS_VALIDATORS + + if validator_type.lower() not in CROSS_VALIDATORS: + raise ValueError( + f"Invalid validator_type: {validator_type}. " + f"Valid types are: {list(CrossValidatorFactory.VALIDATORS.keys())}" + ) + return CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **kwargs + ) diff --git a/src/RiskLabAI/backtest/validation/cross_validator_interface.py b/src/RiskLabAI/backtest/validation/cross_validator_interface.py new file mode 100644 index 0000000..b4cb587 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_interface.py @@ -0,0 +1,235 @@ +""" +Defines the abstract base class (ABC) for all cross-validation strategies +in the RiskLabAI library. +""" + +from abc import ABC, abstractmethod +from collections.abc import Generator +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd + +# For type hinting sklearn-like estimators +Estimator = Any + + +class CrossValidator(ABC): + """ + Abstract Base Class (ABC) for cross-validation strategies. + + This interface defines the common methods required for all cross-validators, + ensuring they can handle both single and multiple datasets (passed as dicts) + and provide methods for splitting, generating backtest paths, and + running backtest predictions. + """ + + @abstractmethod + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the total number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The total number of splits. + """ + raise NotImplementedError + + @abstractmethod + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into train-test indices. + + This function provides train-test indices to split a single dataset + into train/test sets. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + raise NotImplementedError + + @abstractmethod + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into train-test indices. + + This function returns a generator that yields train-test indices. If a + dictionary of data is provided, the generator yields a key followed + by the (train_indices, test_indices) tuple. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + raise NotImplementedError + + @abstractmethod + def _single_backtest_paths( + self, single_data: pd.DataFrame + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + + A "path" is a specific sequence of train/test folds. For simple + K-Fold, there is only one path. For combinatorial methods, there + can be multiple. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary where keys are path identifiers (e.g., "Path 1") + and values are lists of splits. Each split is a dictionary + with "Train" and "Test" keys holding their respective indices. + """ + raise NotImplementedError + + @abstractmethod + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of backtest paths. + """ + raise NotImplementedError + + @abstractmethod + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are path identifiers (e.g., "Path 1") + and values are the contiguous arrays of out-of-sample predictions. + """ + raise NotImplementedError + + @abstractmethod + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. Can be a single estimator or a + dictionary of estimators for multiple datasets. + data : pd.DataFrame or dict + Input data. Can be a single dataset or a dictionary of datasets. + labels : pd.Series or dict + Target labels. Can be a single series or a dictionary of series. + sample_weights : np.ndarray or dict, optional + Sample weights. Can be a single array or a dictionary of arrays. + Defaults to None (equal weights). + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of predictions. + """ + raise NotImplementedError diff --git a/src/RiskLabAI/backtest/validation/kfold.py b/src/RiskLabAI/backtest/validation/kfold.py new file mode 100644 index 0000000..9dbb7b4 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/kfold.py @@ -0,0 +1,359 @@ +""" +Implements a standard K-Fold cross-validator. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .cross_validator_interface import CrossValidator + +# For type hinting sklearn-like estimators +Estimator = Any + + +class KFold(CrossValidator): + """ + K-Fold cross-validator. + + This class implements the standard K-Fold cross-validation strategy, where + the dataset is divided into `k` consecutive folds. Each fold is then + used once as a validation set while the `k - 1` remaining folds + form the training set. + + This implementation allows for optional shuffling. + """ + + def __init__( + self, n_splits: int, shuffle: bool = False, random_seed: int = None + ) -> None: + """ + Initialize the K-Fold cross-validator. + + Parameters + ---------- + n_splits : int + Number of splits or folds for the cross-validation. + The dataset will be divided into `n_splits` consecutive parts. + shuffle : bool, default=False + Whether to shuffle the data before splitting it into folds. + random_seed : int, optional + Seed used for random shuffling. Only used when `shuffle` is True. + """ + self.n_splits = n_splits + self.shuffle = shuffle + self.random_seed = random_seed + self.rng_ = np.random.default_rng(random_seed) + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The number of splits (`n_splits`). + """ + return self.n_splits + + def _get_shuffled_indices(self, n_samples: int) -> np.ndarray: + """Helper to get shuffled or sequential indices.""" + indices = np.arange(n_samples) + if self.shuffle: + self.rng_.shuffle(indices) + return indices + + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into train-test indices. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + indices = self._get_shuffled_indices(single_data.shape[0]) + + for test_indices in np.array_split(indices, self.n_splits): + train_indices = np.setdiff1d(indices, test_indices) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into train-test indices. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + if isinstance(data, dict): + for key in data: + for train_indices, test_indices in self._single_split(data[key]): + yield key, (train_indices, test_indices) + else: + for train_indices, test_indices in self._single_split(data): + yield train_indices, test_indices + + def _single_backtest_paths( + self, single_data: pd.DataFrame + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + For K-Fold, there is only one "path". + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary with one key "Path 1", containing a list of all + (Train, Test) index dictionaries. + """ + path_data = [] + paths = {} + + for train_indices, test_indices in self._single_split(single_data): + path_data.append( + { + "Train": train_indices, + "Test": test_indices, + } + ) + + paths["Path 1"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of backtest paths. + """ + if isinstance(data, dict): + multiple_paths = {} + for key in data: + multiple_paths[key] = self._single_backtest_paths(data[key]) + return multiple_paths + + return self._single_backtest_paths(data) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions, ordered by the original + dataset index. + """ + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> tuple[np.ndarray, np.ndarray]: + """Train model and return (predictions, test_indices).""" + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + # Fallback for estimators without sample_weight + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + preds = estimator_.predict_proba(X_test) + else: + preds = estimator_.predict(X_test) + + return preds, test_indices + + # Run all folds in parallel + results = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split(single_data) + ) + + # Unpack predictions and their corresponding original indices + all_preds = [res[0] for res in results] + all_indices = [res[1] for res in results] + + # Concatenate results + path_data = np.concatenate(all_preds) + original_indices = np.concatenate(all_indices) + + # Re-order predictions to match the original dataset order + # This is critical if shuffle=True + if self.shuffle: + reorder_indices = np.argsort(original_indices) + path_data = path_data[reorder_indices] + + paths_predictions = {"Path 1": path_data} + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. + data : pd.DataFrame or dict + Input data. + labels : pd.Series or dict + Target labels. + sample_weights : np.ndarray or dict, optional + Sample weights. Defaults to None. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary of predictions. + """ + if isinstance(data, dict): + if ( + not isinstance(estimator, dict) + or not isinstance(labels, dict) + or (sample_weights and not isinstance(sample_weights, dict)) + ): + raise ValueError( + "If data is a dict, estimator, labels, " + "and sample_weights (if provided) must also be dicts." + ) + + multiple_paths_predictions = {} + for key in data: + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + data[key], + labels[key], + sample_weights[key] if sample_weights else None, + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If data is a DataFrame, estimator must be a single estimator " + "and labels must be a single Series." + ) + + return self._single_backtest_predictions( + estimator, data, labels, sample_weights, predict_probability, n_jobs + ) diff --git a/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py b/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py new file mode 100644 index 0000000..e299bd6 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py @@ -0,0 +1,240 @@ +r""" +Leakage-aware hyper-parameter optimization: Optuna wired through purged CV, gated by the Deflated Sharpe. + +de Prado tunes via grid / random search under purged cross-validation. Principled Bayesian / evolutionary +search (Optuna's TPE / CMA-ES) reaches good configurations in far fewer trials, but two hazards make naive +HPO dangerous on financial data: leakage-prone CV (a shuffled k-fold) massively inflates the apparent +score under overlapping labels, and every trial is a configuration, so the search itself overfits and the +in-sample HPO score must be deflated by the trial count before it can be trusted. + +This module wires Optuna through the repo's :class:`PurgedKFold` / :class:`CombinatorialPurged` (so the +per-trial score is leakage-controlled) and provides a Deflated-Sharpe gate that deflates the selected +model's out-of-sample Sharpe by the HPO-inclusive trial count. + +Admitted in Appraisal 20 (CONTRIBUTIONS_LEDGER 2026-06-27) as a methodology / infrastructure admit, NOT a +performance claim. Scope tag, verbatim from the verdict: + + Optuna (TPE/CMA-ES) wired through PurgedKFold/CPCV with PBO/DSR gating, preferred over grid/random for + search efficiency and over naive-CV tuning for leakage-safety, and carrying the honest, characterized + finding that tuning yields no OOS edge after deflation - so it must be gated by the Deflated Sharpe at + the trial count, never trusted on the in-sample HPO score. + +**Plainly: this improves search efficiency and leakage-safety, NOT out-of-sample performance. Tuning does +not create edge.** In the appraisal the principled-HPO real-leg gain did not survive the Deflated Sharpe at +the HPO-inclusive trial count (DSR 0.44 / 0.34 < 0.5, PBO 0.67 / 0.69, OOS at the base rate); the value is +that the search reaches the optimum in fewer trials and that purged CV removes the leakage a naive k-fold +inflates. Always gate the selected model by :func:`deflated_sharpe_gate` at the HPO trial count; never +trust the in-sample HPO score. Evidence and caveats: appraisals/20_verdict.md. + +References +---------- +Akiba, T. et al. (2019) Optuna: a next-generation hyperparameter optimization framework. KDD. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning (PurgedKFold/CPCV, PBO, the Deflated + Sharpe Ratio). +""" + +from __future__ import annotations + +from typing import Callable + +import numpy as np + +from RiskLabAI.backtest.probabilistic_sharpe_ratio import probabilistic_sharpe_ratio +from RiskLabAI.backtest.test_set_overfitting import expected_max_sharpe_ratio + +from .purged_kfold import PurgedKFold + + +def _purged_cv_score( + estimator_factory, params, x, y, times, n_splits, embargo, scoring, seed +): + """Mean leakage-controlled CV score of one configuration under PurgedKFold.""" + purged = PurgedKFold(n_splits=n_splits, times=times, embargo=embargo) + x_values = x.values if hasattr(x, "values") else np.asarray(x) + y_values = y.values if hasattr(y, "values") else np.asarray(y) + scores = [] + for train, test in purged.split(x): + if len(train) < 25 or len(test) < 5 or len(np.unique(y_values[train])) < 2: + continue + model = estimator_factory(params) + model.fit(x_values[train], y_values[train]) + if scoring is None: + scores.append( + float((model.predict(x_values[test]) == y_values[test]).mean()) + ) + else: + scores.append(float(scoring(model, x_values[test], y_values[test]))) + return float(np.mean(scores)) if scores else float("-inf") + + +def leakage_aware_hpo( + suggest_params: Callable, + estimator_factory: Callable, + x, + y, + times, + n_trials: int = 50, + sampler: str = "tpe", + n_splits: int = 5, + embargo: float = 0.0, + scoring: Callable | None = None, + random_state: int = 0, +) -> dict: + r""" + Optuna hyper-parameter search with every trial scored under :class:`PurgedKFold` (leakage-controlled). + + This improves SEARCH EFFICIENCY (TPE / CMA-ES reach the optimum in far fewer trials than grid / random) + and LEAKAGE-SAFETY (the per-trial score is purged, not a leaky shuffled k-fold). It does NOT improve + out-of-sample performance - tuning does not create edge - so the selected model must be gated by + :func:`deflated_sharpe_gate` at the returned ``n_trials``; never trust ``best_score`` directly. See the + module docstring for the verbatim scope tag and appraisals/20_verdict.md. + + Parameters + ---------- + suggest_params : callable + ``suggest_params(trial) -> dict``: maps an Optuna trial to a hyper-parameter dict. + estimator_factory : callable + ``estimator_factory(params) -> estimator``: builds a fresh scikit-learn-style estimator + (``fit`` / ``predict``). + x, y : array-like or pandas + Design matrix and labels. ``x`` must support ``PurgedKFold.split(x)`` (a DataFrame indexed by the + event start times). + times : pd.Series + Event start -> end times for purging / embargo (passed to :class:`PurgedKFold`). + n_trials : int, default 50 + Optuna trial budget (this is the HPO trial count to deflate by). + sampler : {"tpe", "cmaes"}, default "tpe" + The Optuna sampler. + n_splits : int, default 5 + Purged CV folds. + embargo : float, default 0.0 + PurgedKFold embargo fraction. + scoring : callable, optional + ``scoring(estimator, x_test, y_test) -> float`` (higher is better). Defaults to accuracy. + random_state : int, default 0 + Sampler seed. + + Returns + ------- + dict + ``best_params``, ``best_score`` (purged CV), ``n_trials``, ``trial_scores``, ``mean_trial_score``, + ``std_trial_score``. + + Raises + ------ + ImportError + If ``optuna`` is not installed (it is an optional dependency). + """ + try: + import optuna + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError( + "leakage_aware_hpo requires optuna (pip install optuna)" + ) from exc + + if sampler == "tpe": + optuna_sampler = optuna.samplers.TPESampler(seed=random_state) + elif sampler == "cmaes": + optuna_sampler = optuna.samplers.CmaEsSampler(seed=random_state) + else: + raise ValueError("sampler must be 'tpe' or 'cmaes'") + + optuna.logging.set_verbosity(optuna.logging.WARNING) + + def objective(trial): + params = suggest_params(trial) + return _purged_cv_score( + estimator_factory, + params, + x, + y, + times, + n_splits, + embargo, + scoring, + random_state, + ) + + study = optuna.create_study(direction="maximize", sampler=optuna_sampler) + study.optimize(objective, n_trials=n_trials, show_progress_bar=False) + trial_scores = [ + t.value for t in study.trials if t.value is not None and np.isfinite(t.value) + ] + return { + "best_params": study.best_params, + "best_score": float(study.best_value), + "n_trials": int(n_trials), + "trial_scores": trial_scores, + "mean_trial_score": ( + float(np.mean(trial_scores)) if trial_scores else float("nan") + ), + "std_trial_score": ( + float(np.std(trial_scores)) if trial_scores else float("nan") + ), + } + + +def deflated_sharpe_gate( + out_of_sample_returns: np.ndarray, + n_trials: int, + trial_sharpe_std: float, + threshold: float = 0.5, +) -> dict: + r""" + Gate a selected model's out-of-sample Sharpe by the Deflated Sharpe at the HPO trial count. + + The benchmark is the expected maximum Sharpe over ``n_trials`` independent trials (with the observed + cross-trial Sharpe dispersion ``trial_sharpe_std``); the Deflated Sharpe is the probability that the + realized OOS Sharpe exceeds that benchmark. A selection passes only if the Deflated Sharpe exceeds + ``threshold`` (0.5). This is the decisive control: in the appraisal the principled-HPO gain did not + pass it - tuning yielded no out-of-sample edge after deflation. See the module docstring for the + verbatim scope tag and appraisals/20_verdict.md. + + Parameters + ---------- + out_of_sample_returns : np.ndarray + The selected model's out-of-sample (e.g. CPCV) per-period return series. + n_trials : int + The HPO-inclusive trial count to deflate by (every configuration tried). + trial_sharpe_std : float + The standard deviation of the per-trial (per-period) Sharpe ratios across the search. + threshold : float, default 0.5 + The Deflated-Sharpe pass threshold. + + Returns + ------- + dict + ``observed_sharpe``, ``benchmark_sharpe`` (E[max SR] at ``n_trials``), ``n_trials``, + ``deflated_sharpe``, and ``passes``. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> r = rng.standard_normal(500) * 0.01 + >>> gate = deflated_sharpe_gate(r, n_trials=100, trial_sharpe_std=0.05) + >>> set(gate) >= {"deflated_sharpe", "passes", "benchmark_sharpe"} + True + """ + from scipy.stats import kurtosis, skew + + r = np.asarray(out_of_sample_returns, dtype=float) + sd = r.std() + observed_sharpe = float(r.mean() / sd) if sd > 0 else 0.0 + benchmark_sharpe = float(expected_max_sharpe_ratio(n_trials, 0.0, trial_sharpe_std)) + deflated = float( + probabilistic_sharpe_ratio( + observed_sharpe, + benchmark_sharpe, + r.size, + skewness_of_returns=float(skew(r)) if r.size > 2 else 0.0, + kurtosis_of_returns=float(kurtosis(r, fisher=False)) if r.size > 3 else 3.0, + ) + ) + return { + "observed_sharpe": observed_sharpe, + "benchmark_sharpe": benchmark_sharpe, + "n_trials": int(n_trials), + "deflated_sharpe": deflated, + "passes": bool(deflated > threshold), + } diff --git a/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py b/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py new file mode 100644 index 0000000..bdf8460 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py @@ -0,0 +1,219 @@ +""" +Path-level Adaptive Combinatorial Purged Cross-Validation PBO (Arian-Norouzi-Seco 2024). + +de Prado's CPCV / CSCV weights every combinatorial backtest path equally when it estimates the +Probability of Backtest Overfitting (PBO) and when it selects a configuration. When the train/test +regime shifts, that equal weighting biases the selection toward the wrong regime. Adaptive CPCV +weights each path by how much its TEST block resembles the FORWARD regime, where the regime is read +from an observable, point-in-time volatility signal (the cross-configuration dispersion). Down-weighting +paths that test on a non-representative regime lowers selection error when the regime is detectable and +selection is signal-bearing, and the weights collapse to uniform in a stationary single regime, so the +estimate converges to plain CPCV with no over-adaptation. + +This is the **path-level** mechanism from the Appraisal 09 pre-registration. It is distinct from, and +does NOT touch, the existing :class:`AdaptiveCombinatorialPurged`, which adapts the split *boundaries* +of an ML backtest (a different mechanism). The plain-CPCV baseline here is the repo's CSCV PBO: the +per-path overfit indicator is taken from +:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.performance_evaluation`, so with uniform +weights this function reproduces +:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.probability_of_backtest_overfitting` +exactly. + +Admitted in Appraisal 09b (CONTRIBUTIONS_LEDGER 2026-06-27; in-house method under a conflict of +interest, held to the identical bar, admitted only after a re-designed signal-bearing held-out). +Regime tag, verbatim from the 09b verdict: + + prefer Adaptive CPCV over plain CPCV for model selection when the train/test regime may shift, the + shift is identifiable at decision time from observable volatility, and there is adequate data for + selection (it does not help where selection is noise-dominated by too little data); it converges to + plain CPCV in a stationary regime with no over-adaptation. + +The advantage is, as of admission, Monte-Carlo evidence where the regime is observable through the +adaptive feature; a real-data regime-shift confirmation on a genuine crisis/volatility period is a +tracked obligation (``REAL_DATA_FOLLOWUPS.md``). Evidence and caveats: appraisals/09_verdict.md (the +09b re-appraisal). + +References +---------- +Arian, H., Norouzi M., L. and Seco, L. (2024) Bagged and Adaptive Combinatorial Purged + Cross-Validation. (Clean-room from the path-level mechanism; the verdict harness validated plain + CPCV against probability_of_backtest_overfitting exactly.) +Bailey, D. H., Borwein, J., Lopez de Prado, M. and Zhu, Q. J. (2017) The probability of backtest + overfitting. Journal of Computational Finance, 20(4), 39-69. +""" + +from __future__ import annotations + +from itertools import combinations +from typing import Callable + +import numpy as np + +from ..backtest_statistics import sharpe_ratio +from ..probability_of_backtest_overfitting import performance_evaluation + + +def estimate_volatility_regimes( + performances: np.ndarray, n_regimes: int = 2, window: int | None = None +) -> np.ndarray: + """ + Observable regime label per period from the cross-configuration volatility. + + The signal is the per-period dispersion across configurations (a market-stress proxy), smoothed by a + **trailing** moving average so the volatility feature at period ``t`` uses only returns up to ``t``. + Periods are then labelled high- vs low-volatility relative to the median of the observed signal (an + evaluation-time normalisation over the in-sample window). The estimate keys on volatility alone: it + is invariant to the sign of the returns and never uses the forward performance the method selects, so + it carries no look-ahead into the quantity being predicted. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-configurations matrix of per-period returns. + n_regimes : int, default 2 + The number of regimes (>= 2); 2 gives a low/high-volatility split at the observed median. + window : int, optional + The trailing smoothing window. Defaults to ``max(T // 20, 3)``. + + Returns + ------- + np.ndarray + An integer regime label in ``[0, n_regimes)`` for each of the T periods. + """ + performances = np.asarray(performances, dtype=float) + t_len = performances.shape[0] + dispersion = performances.std(axis=1) + if window is None: + window = max(t_len // 20, 3) + window = max(int(window), 1) + # trailing (causal) moving average: cumulative-sum difference over a back-window + cumulative = np.concatenate([[0.0], np.cumsum(dispersion)]) + counts = np.minimum(np.arange(1, t_len + 1), window) + starts = np.maximum(np.arange(1, t_len + 1) - window, 0) + smooth = (cumulative[1:] - cumulative[starts]) / counts + if n_regimes <= 2: + return (smooth > np.median(smooth)).astype(int) + edges = np.quantile(smooth, np.linspace(0.0, 1.0, n_regimes + 1)[1:-1]) + return np.digitize(smooth, edges) + + +def _cscv_path_stats( + performances: np.ndarray, + n_partitions: int, + metric: Callable[[np.ndarray, float], float], + risk_free_return: float, +): + """ + Per CSCV path: ``(is_overfit, test_config_metrics, test_partition_ids)``. + + The overfit indicator comes from the repo's + :func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.performance_evaluation`, over the same + symmetric train/test combinations the plain PBO uses, so the unweighted mean reproduces the repo PBO + exactly. The per-configuration test metrics are recomputed with the same metric for the (weighted) + selection and weighting. + """ + _, n_strategies = performances.shape + partitions = np.array_split(performances, n_partitions) + all_ids = list(range(n_partitions)) + stats = [] + for train_ids in combinations(all_ids, n_partitions // 2): + test_ids = [i for i in all_ids if i not in train_ids] + train = np.concatenate([partitions[i] for i in train_ids], axis=0) + test = np.concatenate([partitions[i] for i in test_ids], axis=0) + is_overfit, _logit = performance_evaluation( + train, test, n_strategies, metric, risk_free_return + ) + test_metrics = np.array( + [metric(test[:, j], risk_free_return) for j in range(n_strategies)] + ) + stats.append((1.0 if is_overfit else 0.0, test_metrics, test_ids)) + return stats + + +def adaptive_probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + target_fraction: float = 0.25, + risk_free_return: float = 0.0, + metric: Callable[[np.ndarray, float], float] | None = None, + n_regimes: int = 2, +) -> tuple[float, int]: + r""" + Adaptive (path-level) Probability of Backtest Overfitting and regime-aware configuration selection. + + Each CSCV path is weighted by how much its test block resembles the forward regime (the regime of + the most recent ``target_fraction`` of the sample, read from the observable volatility signal of + :func:`estimate_volatility_regimes`). The regime-weighted PBO and the configuration with the highest + regime-weighted out-of-sample metric are returned. In a stationary regime the weights are uniform, + so the PBO converges to plain CPCV and the selection matches it; see the module docstring for the + full regime tag and appraisals/09_verdict.md. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-configurations matrix of per-period returns. + n_partitions : int, default 16 + The number of CSCV partitions S (must be even). + target_fraction : float, default 0.25 + The trailing fraction of the sample whose regime defines the forward target (in ``(0, 1]``). + risk_free_return : float, default 0.0 + Passed through to the metric. + metric : Callable, optional + The performance metric (defaults to the PBO module's Sharpe). + n_regimes : int, default 2 + The number of volatility regimes to estimate. + + Returns + ------- + (float, int) + The regime-weighted PBO and the index of the selected configuration. + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest.validation import ( + ... adaptive_probability_of_backtest_overfitting, + ... ) + >>> rng = np.random.default_rng(0) + >>> # a regime shift: config 0 leads early, config 9 leads in the volatile forward block + >>> perf = rng.standard_normal((480, 10)) * 0.05 + >>> perf[:360, 0] += 0.02 + >>> perf[360:, 9] += 0.02 + rng.standard_normal((120, 1))[:, 0] * 0.05 + >>> pbo, selected = adaptive_probability_of_backtest_overfitting(perf, n_partitions=8) + >>> 0.0 <= pbo <= 1.0 and 0 <= selected < 10 + True + """ + performances = np.asarray(performances, dtype=float) + if n_partitions % 2 != 0: + raise ValueError("Number of partitions must be even.") + if performances.ndim != 2: + raise ValueError("performances must be a 2-D (T x N) matrix.") + t_len, _ = performances.shape + if t_len < n_partitions: + raise ValueError( + f"Too few observations ({t_len}) for {n_partitions} partitions." + ) + if not 0.0 < target_fraction <= 1.0: + raise ValueError("target_fraction must be in (0, 1].") + if metric is None: + metric = sharpe_ratio + + regimes = estimate_volatility_regimes(performances, n_regimes=n_regimes) + forward = regimes[int((1.0 - target_fraction) * t_len) :] + # majority regime label over the forward block (the target the selection is steered toward) + forward_label = int(np.bincount(forward).argmax()) if forward.size else 0 + + regime_partitions = np.array_split(regimes, n_partitions) + stats = _cscv_path_stats(performances, n_partitions, metric, risk_free_return) + + overfit = np.array([s[0] for s in stats]) + test_metrics = np.array([s[1] for s in stats]) # (n_paths, n_configs) + weights = np.empty(len(stats)) + for k, (_overfit, _metrics, test_ids) in enumerate(stats): + test_regime = np.concatenate([regime_partitions[i] for i in test_ids]) + # similarity = share of the test block in the forward regime; epsilon avoids all-zero weights + weights[k] = float(np.mean(test_regime == forward_label)) + 1e-3 + + pbo = float(np.average(overfit, weights=weights)) + selected = int(np.argmax(np.average(test_metrics, axis=0, weights=weights))) + return pbo, selected diff --git a/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py b/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py new file mode 100644 index 0000000..ab15b2c --- /dev/null +++ b/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py @@ -0,0 +1,137 @@ +""" +Path-level Bagged Combinatorial Purged Cross-Validation PBO (Arian-Norouzi-Seco 2024). + +de Prado's CPCV / CSCV estimates the Probability of Backtest Overfitting (PBO) from a finite set of +combinatorial backtest paths, so the estimate carries estimation variance. Bagging the PATHS - taking +a moving-block bootstrap of the performance series, recomputing the CSCV PBO on each resample, and +averaging - reduces that variance, giving a more accurate PBO estimate whenever the path set is small +or noisy, and converging to plain CPCV in the data-rich limit. + +This is the **path-level** mechanism from the Appraisal 09 pre-registration. It is distinct from, and +does NOT touch, the existing :class:`BaggedCombinatorialPurged`, which bags a sklearn *estimator* on an +ML backtest (a different mechanism). The plain-CPCV baseline here is the repo's CSCV PBO +(:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.probability_of_backtest_overfitting`), +which this function calls on each bootstrap resample. + +Admitted in Appraisal 09 (CONTRIBUTIONS_LEDGER 2026-06-27; in-house method, held to the identical bar). +Regime tag, verbatim from the verdict: + + For a more accurate, lower-variance overfitting (PBO) estimate whenever the CPCV path set is small + or noisy, converging to plain CPCV in the data-rich limit; neutral on model selection. Point-in-time + and mechanism-backed. + +Evidence and caveats: appraisals/09_verdict.md. + +References +---------- +Arian, H., Norouzi M., L. and Seco, L. (2024) Bagged and Adaptive Combinatorial Purged + Cross-Validation. (Clean-room from the path-level mechanism; the verdict harness validated plain + CPCV against probability_of_backtest_overfitting exactly.) +Bailey, D. H., Borwein, J., Lopez de Prado, M. and Zhu, Q. J. (2017) The probability of backtest + overfitting. Journal of Computational Finance, 20(4), 39-69. +""" + +from __future__ import annotations + +from typing import Callable, Optional + +import numpy as np + +from ..probability_of_backtest_overfitting import probability_of_backtest_overfitting + + +def moving_block_bootstrap_indices( + n_observations: int, block_size: int, rng: np.random.Generator +) -> np.ndarray: + """ + Moving-block bootstrap row indices for a length-n series (blocks preserve serial structure). + + Random block start points are tiled, each contributing ``block_size`` consecutive (wrap-around) + rows, truncated to ``n_observations``. + + Parameters + ---------- + n_observations : int + The series length n. + block_size : int + The block length. + rng : np.random.Generator + The random generator (seed it for reproducibility). + + Returns + ------- + np.ndarray + An index array of length ``n_observations``. + """ + block_size = max(int(block_size), 1) + n_blocks = int(np.ceil(n_observations / block_size)) + starts = rng.integers(0, n_observations, size=n_blocks) + offsets = np.arange(block_size) + return (starts[:, None] + offsets[None, :]).ravel()[ + :n_observations + ] % n_observations + + +def bagged_probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + n_bag: int = 30, + block_size: Optional[int] = None, + risk_free_return: float = 0.0, + metric: Optional[Callable[[np.ndarray, float], float]] = None, + seed: int = 0, +) -> tuple[float, np.ndarray]: + r""" + Bagged (path-level) Probability of Backtest Overfitting. + + Takes ``n_bag`` moving-block bootstrap resamples of the performance matrix, computes the CSCV PBO + (:func:`probability_of_backtest_overfitting`) on each, and returns the average - a lower-variance + estimate than a single CPCV PBO. Prefer it whenever the CPCV path set is small or noisy; it + converges to plain CPCV in the data-rich limit and is neutral on model selection. See the module + docstring for the full regime tag and appraisals/09_verdict.md. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-strategies matrix of per-period returns. + n_partitions : int, default 16 + The number of CSCV partitions S (must be even); passed through to the plain PBO. + n_bag : int, default 30 + The number of moving-block bootstrap resamples to aggregate. + block_size : int, optional + The bootstrap block length. Defaults to ``max(T // 20, 5)``. + risk_free_return : float, default 0.0 + Passed through to the metric. + metric : Callable, optional + The performance metric (defaults to the PBO module's Sharpe). + seed : int, default 0 + The bootstrap seed (for reproducibility). + + Returns + ------- + (float, np.ndarray) + The bagged PBO (the mean over resamples) and the array of per-resample PBOs. + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest.validation import bagged_probability_of_backtest_overfitting + >>> rng = np.random.default_rng(0) + >>> perf = rng.standard_normal((480, 10)) * 0.05 + np.linspace(0, 0.01, 10) + >>> pbo, per_resample = bagged_probability_of_backtest_overfitting(perf, n_partitions=8, n_bag=20) + >>> 0.0 <= pbo <= 1.0 + True + """ + performances = np.asarray(performances, dtype=float) + t_len = performances.shape[0] + if block_size is None: + block_size = max(t_len // 20, 5) + rng = np.random.default_rng(seed) + pbos = np.empty(n_bag) + for b in range(n_bag): + rows = moving_block_bootstrap_indices(t_len, block_size, rng) + pbo, _ = probability_of_backtest_overfitting( + performances[rows], n_partitions, risk_free_return, metric + ) + pbos[b] = pbo + return float(pbos.mean()), pbos diff --git a/src/RiskLabAI/backtest/validation/purged_kfold.py b/src/RiskLabAI/backtest/validation/purged_kfold.py new file mode 100644 index 0000000..9872834 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/purged_kfold.py @@ -0,0 +1,543 @@ +""" +Implements a Purged K-Fold cross-validator to prevent information leakage +in financial time-series data. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .cross_validator_interface import CrossValidator + +# For type hinting sklearn-like estimators +Estimator = Any + + +class PurgedKFold(CrossValidator): + """ + Purged K-Fold cross-validator. + + Implements a K-Fold strategy where each fold is purged of observations + that overlap in time with the test set. An "embargo" period can also be + added after the test set to prevent information leakage from the training + set immediately following the test set. + + This is essential for financial data where labels are defined over a + future period (e.g., T+k returns) and simple K-Fold would leak + information. + + Parameters + ---------- + n_splits : int + Number of splits or folds. + times : pd.Series or Dict[str, pd.Series] + A Series where the index is the time the observation starts and + the values are the time the observation ends. This defines the + "information range" of each observation. + embargo : float, default=0 + The embargo fraction. A fraction of the dataset length (e.g., 0.01) + to be removed *after* the end of the test set information range. + """ + + @staticmethod + def filtered_training_indices_with_embargo( + data_info_range: pd.Series, + test_time_range: pd.Series, + embargo_fraction: float = 0, + continous_test_times: bool = False, + ) -> pd.Series: + r""" + Purge observations in the training set with embargo. + + Finds training set indices that do not overlap with the test set's + information range. + + .. math:: + \text{embargo\_length} = \text{len(data\_info_range)} \times \text{embargo\_fraction} + + Parameters + ---------- + data_info_range : pd.Series + Series detailing the information range for *all* data. + - Index: Time when the information extraction started. + - Value: Time when the information extraction ended. + test_time_range : pd.Series + Series detailing the information range for *test set* data. + - Index: Time when the information extraction started. + - Value: Time when the information extraction ended. + embargo_fraction : float, default=0 + Fraction of the dataset trailing the test observations to + exclude from training. + continous_test_times : bool, default=False + If True, considers the test time range as one continuous block + from the earliest start to the latest end. + + Returns + ------- + pd.Series + A view of `data_info_range` containing only the filtered + training data (i.e., observations that do not overlap with + test + embargo). + """ + indices_to_drop: set[int] = set() + embargo_length = int(len(data_info_range) * embargo_fraction) + + if test_time_range.empty: + return data_info_range + + sorted_test_time_range = test_time_range.sort_index().copy() + + if not continous_test_times: + # Create blocks of contiguous test ranges + test_ranges = pd.DataFrame( + { + "start": sorted_test_time_range.index, + "end": sorted_test_time_range.values, + } + ) + gaps = test_ranges["start"] > test_ranges["end"].shift(1) + blocks = gaps.cumsum() + effective_test_time_range = test_ranges.groupby(blocks).agg( + {"start": "min", "end": "max"} + ) + effective_test_time_range = pd.Series( + effective_test_time_range["end"].values, + index=effective_test_time_range["start"], + ) + else: + effective_test_time_range = pd.Series( + sorted_test_time_range.values[-1], + index=[sorted_test_time_range.index[0]], + ) + + if embargo_length == 0: + embargoed_ranges = effective_test_time_range + else: + embargoed_values = [] + for end_val in effective_test_time_range.values: + end_iloc = data_info_range.index.searchsorted(end_val, side="left") + + if end_iloc >= len(data_info_range): + embargoed_values.append(end_val) + else: + embargoed_iloc = min( + end_iloc + embargo_length, len(data_info_range) - 1 + ) + embargoed_values.append(data_info_range.index[embargoed_iloc]) + + embargoed_ranges = pd.Series( + embargoed_values, index=effective_test_time_range.index + ) + # === END OF FIX === + + # Purge + for test_start, test_end_embargoed in embargoed_ranges.items(): + # 1. Overlap: train starts during test/embargo + cond1 = (data_info_range.index >= test_start) & ( + data_info_range.index <= test_end_embargoed + ) + # 2. Overlap: train ends during test/embargo + cond2 = (data_info_range.values >= test_start) & ( + data_info_range.values <= test_end_embargoed + ) + # 3. Overlap: train envelops test/embargo + cond3 = (data_info_range.index <= test_start) & ( + data_info_range.values >= test_end_embargoed + ) + + indices_to_drop.update(data_info_range[cond1 | cond2 | cond3].index) + + return data_info_range.drop(indices_to_drop) + + def __init__( + self, + n_splits: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + ) -> None: + """ + Initialize the PurgedKFold cross-validator. + + Parameters + ---------- + n_splits : int + Number of splits or folds. + times : pd.Series or Dict[str, pd.Series] + A Series (or dict of Series) where the index is the observation + start time and the value is the observation end time. + embargo : float, default=0 + The embargo fraction (e.g., 0.01 for 1%). + """ + self.n_splits = n_splits + self.times = times + self.embargo = embargo + self.is_multiple_datasets = isinstance(times, dict) + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The number of splits (`n_splits`). + """ + return self.n_splits + + def _validate_input( + self, single_times: pd.Series, single_data: pd.DataFrame + ) -> None: + """Validate that data and times indices match.""" + if not single_data.index.equals(single_times.index): + raise ValueError("Data and 'times' (info range) must have the same index") + + def _get_train_indices( + self, + test_indices: np.ndarray, + single_times: pd.Series, + continous_test_times: bool = False, + ) -> np.ndarray: + """ + Get training indices after purging and embargo. + + Parameters + ---------- + test_indices : np.ndarray + The integer-location indices of the test set. + single_times : pd.Series + The 'times' (info range) Series for *all* data. + continous_test_times : bool, default=False + Passed to `filtered_training_indices_with_embargo`. + + Returns + ------- + np.ndarray + The integer-location indices of the training set. + """ + if len(test_indices) == 0: + return np.arange(len(single_times)) + + test_time_range = single_times.iloc[test_indices] + + train_times = self.filtered_training_indices_with_embargo( + single_times, test_time_range, self.embargo, continous_test_times + ) + + # Convert index labels back to integer locations + train_indices = single_times.index.get_indexer(train_times.index) + return train_indices + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into purged train-test indices. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator of (train_indices, test_indices). + """ + self._validate_input(single_times, single_data) + + indices = np.arange(len(single_data)) + + for test_indices in np.array_split(indices, self.n_splits): + train_indices = self._get_train_indices(test_indices, single_times, True) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into purged train-test indices. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must also be a dict.") + for key in self.times: + for train_indices, test_indices in self._single_split( + self.times[key], data[key] + ): + yield key, (train_indices, test_indices) + else: + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + for train_indices, test_indices in self._single_split(self.times, data): + yield train_indices, test_indices + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + For PurgedKFold, there is only one "path". + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary with one key "Path 1", containing a list of all + (Train, Test) index dictionaries. + """ + self._validate_input(single_times, single_data) + + path_data = [] + paths = {} + + for train_indices, test_indices in self._single_split( + single_times, single_data + ): + path_data.append( + { + "Train": train_indices, + "Test": test_indices, + } + ) + + paths["Path 1"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary of backtest paths. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must also be a dict.") + multiple_paths = {} + for key in self.times: + multiple_paths[key] = self._single_backtest_paths( + self.times[key], data[key] + ) + return multiple_paths + + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + return self._single_backtest_paths(self.times, data) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> np.ndarray: + """Train model and return predictions.""" + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + return estimator_.predict_proba(X_test) + + return estimator_.predict(X_test) + + path_data = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split( + single_times, single_data + ) + ) + + # Since PurgedKFold is not shuffled, we can just concatenate + paths_predictions = {"Path 1": np.concatenate(path_data)} + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. + data : pd.DataFrame or dict + Input data. + labels : pd.Series or dict + Target labels. + sample_weights : np.ndarray or dict, optional + Sample weights. Defaults to None. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary of predictions. + """ + if self.is_multiple_datasets: + if not ( + isinstance(data, dict) + and isinstance(estimator, dict) + and isinstance(labels, dict) + ): + raise ValueError( + "If 'times' is a dict, 'data', 'estimator', and 'labels' " + "must also be dicts." + ) + + multiple_paths_predictions = {} + for key in self.times: + s_weights = sample_weights[key] if sample_weights else None + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + self.times[key], + data[key], + labels[key], + s_weights, + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If 'times' is a Series, 'data' must be a DataFrame " + "and 'labels' must be a Series." + ) + + return self._single_backtest_predictions( + estimator, + self.times, + data, + labels, + sample_weights, + predict_probability, + n_jobs, + ) diff --git a/src/RiskLabAI/backtest/validation/walk_forward.py b/src/RiskLabAI/backtest/validation/walk_forward.py new file mode 100644 index 0000000..8b64ab0 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/walk_forward.py @@ -0,0 +1,183 @@ +""" +Implements a Walk-Forward cross-validator for time-series data. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .kfold import KFold + +# For type hinting sklearn-like estimators +Estimator = Any + + +class WalkForward(KFold): + """ + Walk-Forward Cross-Validator for Time Series Data. + + This cross-validator splits time-series data in a "walk-forward" + manner. In each split, the training set progressively grows (up to + `max_train_size`) while the test set "walks" forward. + + This method respects the temporal order of data, ensuring the model is + always trained on past data and validated on future data. + + Parameters + ---------- + n_splits : int, default=5 + Number of splits/folds. Must be at least 2. + max_train_size : int, optional + Maximum number of observations allowed in the training dataset. + If provided, the most recent `max_train_size` observations are + used for training. + gap : int, default=0 + Number of observations to skip between the end of the training + data and the start of the test data. + """ + + def __init__( + self, n_splits: int = 5, max_train_size: Optional[int] = None, gap: int = 0 + ) -> None: + """ + Initialize the WalkForward cross-validator. + + Parameters + ---------- + n_splits : int, default=5 + Number of splits/folds. Must be at least 2. + max_train_size : int, optional + Maximum number of observations allowed in the training dataset. + gap : int, default=0 + Number of observations to skip between the end of the training + data and the start of the test data. + """ + # WalkForward cannot be shuffled + super().__init__(n_splits=n_splits, shuffle=False) + self.max_train_size = max_train_size + self.gap = gap + + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into walk-forward train-test indices. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + indices = np.arange(single_data.shape[0]) + + # np.array_split handles non-divisible splits + for test_indices in np.array_split(indices, self.n_splits): + # The first test index + first_test_idx_loc = test_indices[0] + + # Train indices end `gap` samples before the test set + train_end_loc = first_test_idx_loc - self.gap + + if train_end_loc < 0: + # No training data possible + train_indices = np.array([], dtype=int) + else: + if self.max_train_size and train_end_loc > self.max_train_size: + train_start_loc = train_end_loc - self.max_train_size + train_indices = indices[train_start_loc:train_end_loc] + else: + train_indices = indices[:train_end_loc] + + yield train_indices, test_indices + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions. + """ + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> np.ndarray: + """Train model and return predictions.""" + + if len(train_indices) == 0: + # No training data, return NaNs + n_classes = len(np.unique(single_labels)) if predict_probability else 1 + shape = ( + (len(test_indices), n_classes) + if predict_probability + else (len(test_indices),) + ) + return np.full(shape, np.nan) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + return estimator_.predict_proba(X_test) + + return estimator_.predict(X_test) + + path_data = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split(single_data) + ) + + # Since shuffle=False, we can just concatenate + paths_predictions = {"Path 1": np.concatenate(path_data)} + return paths_predictions diff --git a/src/RiskLabAI/causal_factor_analysis/__init__.py b/src/RiskLabAI/causal_factor_analysis/__init__.py new file mode 100644 index 0000000..e2b800e --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/__init__.py @@ -0,0 +1,135 @@ +"""Causal-factor portfolio, graph, specification-error, and protocol methods. + +This package implements externally published Causal Factor Analysis equations +and graphical identification criteria together with the seven-stage research +protocol, without importing the broader causal or portfolio-optimization +namespaces. +""" + +from .factor_mirage import ( + ColliderCoefficients, + ColliderDiagnostics, + StrategyPerformance, + collider_factor_return, + collider_forecast_return, + collider_model_diagnostics, + collider_overcontrolled_coefficients, + confounder_factor_return, + confounder_forecast_return, + confounder_undercontrolled_coefficient, +) +from .graph_identification import ( + BackdoorAdjustmentEvidence, + CausalDAG, + DSeparationEvidence, + FrontdoorAdjustmentEvidence, + InstrumentEvidence, + NodeRoleEvidence, + PathEvidence, + causal_role_evidence, + check_backdoor_adjustment_set, + check_frontdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) +from .optimizer import minimum_variance_factor_weights +from .protocol import ( + BacktestStage, + CausalAdjustmentSetStage, + CausalDiscoveryStage, + CausalExplanatoryAndPredictivePowerStage, + CausalFactorProtocolReport, + CausalPortfolioConstructionStage, + EventHorizon, + FoldEvidence, + MultipleTestingAdjustmentsStage, + ValidationEvidence, + VariableSelectionStage, + validate_causal_factor_protocol, +) +from .specification_diagnostics import ( + PopulationRegressionDiagnostics, + RegressionDiagnostics, + SpecificationExperimentResult, + SpecificationPopulationResult, + collider_population_diagnostics, + collider_specification_experiment, + confounded_mediator_population_diagnostics, + confounded_mediator_specification_experiment, + fork_population_diagnostics, + fork_specification_experiment, +) +from .treatment_effects import ( + DifferenceInDifferencesEstimate, + TreatmentEffectDecomposition, + average_treatment_effect, + backdoor_adjusted_average_treatment_effect, + backdoor_adjusted_expectation, + difference_in_differences, + frontdoor_adjusted_probability, + linear_instrumental_variable_effect, + randomized_mean_difference, + treatment_effect_decomposition, +) + +__all__ = [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol", +] diff --git a/src/RiskLabAI/causal_factor_analysis/factor_mirage.py b/src/RiskLabAI/causal_factor_analysis/factor_mirage.py new file mode 100644 index 0000000..08ddade --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/factor_mirage.py @@ -0,0 +1,399 @@ +"""Closed-form analytics for specification errors in factor investing. + +The functions implement the standardized linear-Gaussian results in Lopez de +Prado and Zoonekynd (2026), Equations 2, 5-19 and Appendices D-E. Factor +returns condition on the supplied exposures. Forecast returns integrate over +the exposure distribution assumed by the article. + +Appendix F is not exposed as a separate parameter-shift model because its +equations contain only the training-regime parameters. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from decimal import Decimal, localcontext +from fractions import Fraction +from numbers import Integral, Real +from typing import Optional + +__all__ = [ + "ColliderCoefficients", + "ColliderDiagnostics", + "StrategyPerformance", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", +] + + +@dataclass(frozen=True) +class StrategyPerformance: + """Expected returns from the correct and misspecified strategies.""" + + correct: float + misspecified: float + + +@dataclass(frozen=True) +class ColliderCoefficients: + """Population coefficients after conditioning on the collider.""" + + beta_hat: float + theta_hat: float + + +@dataclass(frozen=True) +class ColliderDiagnostics: + """Population model-selection diagnostics for collider conditioning. + + The t-statistic fields are the expected signals in Appendix D. They are + not p-values. In particular, the article assigns different degrees of + freedom to the correct and overcontrolled regressions. + """ + + residual_variance: float + outcome_variance: float + correct_r_squared: float + overcontrolled_r_squared: float + correct_adjusted_r_squared: float + overcontrolled_adjusted_r_squared: float + correct_beta_variance: float + overcontrolled_beta_variance: float + collider_coefficient_variance: float + correct_beta_t_statistic: float + overcontrolled_beta_t_statistic: float + collider_t_statistic: float + adjusted_r_squared_prefers_overcontrolled: bool + absolute_beta_t_prefers_overcontrolled: bool + + +def _as_finite_real(name: str, value: Real) -> float: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{name} must be a real scalar") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must be a representable real scalar") from error + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _as_sample_size(value: Integral) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError("n_observations must be an integer") + result = int(value) + if result <= 3: + raise ValueError("n_observations must be greater than 3") + return result + + +def _fraction(value: float) -> Fraction: + return Fraction.from_float(value) + + +def _as_result(name: str, value: Fraction) -> float: + try: + result = float(value) + except (OverflowError, ValueError) as error: + raise ValueError(f"{name} cannot be represented as a finite float") from error + if not math.isfinite(result): + raise ValueError(f"{name} cannot be represented as a finite float") + return result + + +def _as_decimal(value: Fraction) -> Decimal: + return Decimal(value.numerator) / Decimal(value.denominator) + + +def _coefficient_times_square_root( + name: str, coefficient: Fraction, radicand: Fraction +) -> float: + if coefficient == 0: + return 0.0 + try: + with localcontext() as context: + context.prec = 80 + value = _as_decimal(coefficient) * _as_decimal(radicand).sqrt() + result = float(value) + except (ArithmeticError, OverflowError, ValueError) as error: + raise ValueError(f"{name} cannot be represented as a finite float") from error + if not math.isfinite(result): + raise ValueError(f"{name} cannot be represented as a finite float") + return result + + +def _confounder_coefficient( + beta: Fraction, gamma: Fraction, delta: Fraction +) -> Fraction: + return beta + gamma * delta / (1 + delta * delta) + + +def _collider_coefficients( + beta: Fraction, gamma: Fraction, delta: Fraction +) -> tuple[Fraction, Fraction]: + denominator = 1 + gamma * gamma + return (beta - delta * gamma) / denominator, gamma / denominator + + +def confounder_undercontrolled_coefficient( + beta: Real, gamma: Real, delta: Real +) -> float: + r"""Return the coefficient obtained when the confounder is omitted. + + For ``X = delta * Z + v`` and ``Y = beta * X + gamma * Z + u``, the + undercontrolled regression coefficient is + ``beta + gamma * delta / (1 + delta**2)`` (Equation 2). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + return _as_result( + "undercontrolled coefficient", + _confounder_coefficient(beta_value, gamma_value, delta_value), + ) + + +def confounder_factor_return( + x: Real, + z: Real, + beta: Real, + gamma: Real, + delta_estimated: Real, + *, + delta_realized: Optional[Real] = None, # noqa: UP045, Python 3.9 API +) -> StrategyPerformance: + r"""Return correct and undercontrolled factor-strategy performance. + + With one correlation regime, omit ``delta_realized`` to evaluate Equations + 5 and 7. Supply it to evaluate the train-to-execution shift in Appendix E, + where ``delta_estimated`` is the training value and ``delta_realized`` is + the execution value. + """ + + x_value = _fraction(_as_finite_real("x", x)) + z_value = _fraction(_as_finite_real("z", z)) + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + estimated = _fraction(_as_finite_real("delta_estimated", delta_estimated)) + realized = ( + estimated + if delta_realized is None + else _fraction(_as_finite_real("delta_realized", delta_realized)) + ) + + correct_signal = x_value * beta_value + z_value * gamma_value + estimated_coefficient = _confounder_coefficient(beta_value, gamma_value, estimated) + realized_coefficient = _confounder_coefficient(beta_value, gamma_value, realized) + correct = correct_signal * correct_signal + misspecified = x_value * x_value * estimated_coefficient * realized_coefficient + return StrategyPerformance( + correct=_as_result("correct factor return", correct), + misspecified=_as_result("undercontrolled factor return", misspecified), + ) + + +def confounder_forecast_return( + beta: Real, + gamma: Real, + delta_estimated: Real, + *, + delta_realized: Optional[Real] = None, # noqa: UP045, Python 3.9 API +) -> StrategyPerformance: + r"""Return correct and undercontrolled forecasting performance. + + Omitting ``delta_realized`` evaluates Equations 8 and 9. Supplying it + evaluates Equations E-3 and E-4 after a change in the confounder link. + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + estimated = _fraction(_as_finite_real("delta_estimated", delta_estimated)) + realized = ( + estimated + if delta_realized is None + else _fraction(_as_finite_real("delta_realized", delta_realized)) + ) + + estimated_coefficient = _confounder_coefficient(beta_value, gamma_value, estimated) + realized_coefficient = _confounder_coefficient(beta_value, gamma_value, realized) + second_signal = beta_value * realized + gamma_value + correct = beta_value * beta_value + second_signal * second_signal + misspecified = ( + (1 + realized * realized) * estimated_coefficient * realized_coefficient + ) + return StrategyPerformance( + correct=_as_result("correct forecast return", correct), + misspecified=_as_result("undercontrolled forecast return", misspecified), + ) + + +def collider_overcontrolled_coefficients( + beta: Real, gamma: Real, delta: Real +) -> ColliderCoefficients: + r"""Return the two coefficients after conditioning on the collider. + + The coefficient on ``X`` is ``(beta - delta*gamma)/(1 + gamma**2)`` and + the coefficient on ``Z`` is ``gamma/(1 + gamma**2)`` (Equations 11-12). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + beta_hat, theta_hat = _collider_coefficients(beta_value, gamma_value, delta_value) + return ColliderCoefficients( + beta_hat=_as_result("overcontrolled beta coefficient", beta_hat), + theta_hat=_as_result("collider coefficient", theta_hat), + ) + + +def collider_factor_return( + x: Real, + collider_proxy: Real, + beta: Real, + gamma: Real, + delta: Real, +) -> StrategyPerformance: + r"""Return correct and overcontrolled factor-strategy performance. + + ``collider_proxy`` is the article's pre-outcome proxy ``Z tilde``. It is + not the contemporaneous collider, which contains the outcome. The return + maps to Equations 16 and 17. + """ + + x_value = _fraction(_as_finite_real("x", x)) + proxy_value = _fraction(_as_finite_real("collider_proxy", collider_proxy)) + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + + beta_x = beta_value * x_value + denominator = 1 + gamma_value * gamma_value + overcontrolled_signal = ( + beta_value - delta_value * gamma_value + ) * x_value + gamma_value * proxy_value + correct = beta_x * beta_x + misspecified = beta_x * overcontrolled_signal / denominator + return StrategyPerformance( + correct=_as_result("correct collider factor return", correct), + misspecified=_as_result("overcontrolled factor return", misspecified), + ) + + +def collider_forecast_return( + beta: Real, gamma: Real, delta: Real +) -> StrategyPerformance: + r"""Return correct and overcontrolled forecasting performance. + + The two values are ``beta**2`` and + ``beta * (beta - gamma*delta) / (1 + gamma**2)`` (Equations 18-19). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + denominator = 1 + gamma_value * gamma_value + correct = beta_value * beta_value + misspecified = beta_value * (beta_value - gamma_value * delta_value) / denominator + return StrategyPerformance( + correct=_as_result("correct collider forecast return", correct), + misspecified=_as_result("overcontrolled forecast return", misspecified), + ) + + +def collider_model_diagnostics( + beta: Real, + gamma: Real, + delta: Real, + n_observations: Integral, +) -> ColliderDiagnostics: + r"""Return the Appendix D diagnostics for conditioning on a collider. + + ``n_observations`` must be an integer greater than three. The returned + booleans implement the strict adjusted-R-squared and absolute expected + beta t-statistic comparisons. The latter is not an exact p-value test. + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + n_integer = _as_sample_size(n_observations) + n_value = Fraction(n_integer) + + gamma_denominator = 1 + gamma_value * gamma_value + outcome_variance = 1 + beta_value * beta_value + residual_variance = 1 / gamma_denominator + correct_r_squared = 1 - 1 / outcome_variance + overcontrolled_r_squared = 1 - 1 / (gamma_denominator * outcome_variance) + correct_adjusted_r_squared = 1 - ( + Fraction(n_integer - 1, n_integer - 2) / outcome_variance + ) + overcontrolled_adjusted_r_squared = 1 - ( + Fraction(n_integer - 1, n_integer - 3) / (gamma_denominator * outcome_variance) + ) + + diagnostic_sum = ( + (beta_value * gamma_value + delta_value) ** 2 + gamma_value * gamma_value + 1 + ) + correct_beta_variance = 1 / n_value + overcontrolled_beta_variance = diagnostic_sum / ( + n_value * gamma_denominator * gamma_denominator + ) + collider_coefficient_variance = 1 / ( + n_value * gamma_denominator * gamma_denominator + ) + + overcontrolled_numerator = beta_value - delta_value * gamma_value + correct_beta_t_statistic = _coefficient_times_square_root( + "correct beta t-statistic", beta_value, n_value + ) + overcontrolled_beta_t_statistic = _coefficient_times_square_root( + "overcontrolled beta t-statistic", + overcontrolled_numerator, + n_value / diagnostic_sum, + ) + collider_t_statistic = _coefficient_times_square_root( + "collider t-statistic", gamma_value, n_value + ) + + return ColliderDiagnostics( + residual_variance=_as_result("residual variance", residual_variance), + outcome_variance=_as_result("outcome variance", outcome_variance), + correct_r_squared=_as_result("correct R-squared", correct_r_squared), + overcontrolled_r_squared=_as_result( + "overcontrolled R-squared", overcontrolled_r_squared + ), + correct_adjusted_r_squared=_as_result( + "correct adjusted R-squared", correct_adjusted_r_squared + ), + overcontrolled_adjusted_r_squared=_as_result( + "overcontrolled adjusted R-squared", + overcontrolled_adjusted_r_squared, + ), + correct_beta_variance=_as_result( + "correct beta variance", correct_beta_variance + ), + overcontrolled_beta_variance=_as_result( + "overcontrolled beta variance", overcontrolled_beta_variance + ), + collider_coefficient_variance=_as_result( + "collider coefficient variance", collider_coefficient_variance + ), + correct_beta_t_statistic=correct_beta_t_statistic, + overcontrolled_beta_t_statistic=overcontrolled_beta_t_statistic, + collider_t_statistic=collider_t_statistic, + adjusted_r_squared_prefers_overcontrolled=( + gamma_value * gamma_value * (n_integer - 3) > 1 + ), + absolute_beta_t_prefers_overcontrolled=( + overcontrolled_numerator * overcontrolled_numerator + > beta_value * beta_value * diagnostic_sum + ), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/graph_identification.py b/src/RiskLabAI/causal_factor_analysis/graph_identification.py new file mode 100644 index 0000000..02ec948 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/graph_identification.py @@ -0,0 +1,1051 @@ +# ruff: noqa: UP045 +"""Graphical identification evidence for causal-factor research. + +The routines in this module operate on an explicit, immutable directed acyclic +graph. They implement path activation, d-separation, the back-door and +front-door criteria, and two distinct graphical instrument profiles. The +results describe implications of an accepted graph; they do not discover a +graph, verify its causal assumptions, establish positivity, or estimate an +effect. +""" + +from __future__ import annotations + +from collections import namedtuple +from collections.abc import Callable, Iterable, Sequence +from typing import NamedTuple, Optional + +__all__ = [ + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", +] + + +NodeTuple = tuple[str, ...] +EdgeTuple = tuple[tuple[str, str], ...] +PathTuple = tuple[str, ...] +_DEFAULT_MAX_PATHS = 10_000 +_DEFAULT_MAX_PATH_STATES = 100_000 +_DEFAULT_MAX_CANDIDATES = 65_536 + + +def _materialize_collection(value, subject: str) -> tuple: + if isinstance(value, (str, bytes)) or not isinstance(value, Iterable): + raise TypeError(f"{subject} must be a collection, not a scalar string.") + try: + return tuple(value) + except TypeError as exc: + raise TypeError(f"{subject} must be an iterable collection.") from exc + + +def _require_node(value, subject: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{subject} must be a string node identifier.") + if not value.strip(): + raise ValueError(f"{subject} must be nonblank.") + return value + + +def _normalize_nodes(value, subject: str, *, allow_empty: bool) -> NodeTuple: + items = _materialize_collection(value, subject) + normalized = tuple(_require_node(item, f"{subject} member") for item in items) + if not allow_empty and not normalized: + raise ValueError(f"{subject} must be nonempty.") + if len(set(normalized)) != len(normalized): + raise ValueError(f"{subject} must not contain duplicate nodes.") + return tuple(sorted(normalized)) + + +def _normalize_edges(value) -> EdgeTuple: + items = _materialize_collection(value, "directed_edges") + normalized = [] + for raw_edge in items: + if isinstance(raw_edge, (str, bytes)) or not isinstance(raw_edge, Sequence): + raise TypeError("Each directed edge must be an ordered two-node sequence.") + edge = _materialize_collection(raw_edge, "each directed edge") + if len(edge) != 2: + raise ValueError("Each directed edge must contain exactly two nodes.") + source = _require_node(edge[0], "directed-edge source") + target = _require_node(edge[1], "directed-edge target") + normalized.append((source, target)) + result = tuple(normalized) + if len(set(result)) != len(result): + raise ValueError("directed_edges must not contain duplicate edges.") + return tuple(sorted(result)) + + +def _children(nodes: NodeTuple, edges: EdgeTuple): + result = {node: set() for node in nodes} + for source, target in edges: + result[source].add(target) + return result + + +def _parents(nodes: NodeTuple, edges: EdgeTuple): + result = {node: set() for node in nodes} + for source, target in edges: + result[target].add(source) + return result + + +def _is_acyclic(nodes: NodeTuple, edges: EdgeTuple) -> bool: + children = _children(nodes, edges) + indegree = {node: 0 for node in nodes} + for _, target in edges: + indegree[target] += 1 + ready = sorted(node for node, degree in indegree.items() if degree == 0) + visited = 0 + while ready: + node = ready.pop() + visited += 1 + for child in sorted(children[node], reverse=True): + indegree[child] -= 1 + if indegree[child] == 0: + ready.append(child) + return visited == len(nodes) + + +_CausalDAGBase = namedtuple("CausalDAG", ("nodes", "directed_edges", "observed_nodes")) + + +class CausalDAG(_CausalDAGBase): + """An immutable DAG with explicit observed and latent nodes. + + Nodes absent from ``observed_nodes`` remain in every graphical calculation + but are never eligible for an adjustment or mediator set. + """ + + __slots__ = () + + def __new__(cls, nodes, directed_edges, observed_nodes): + nodes = _normalize_nodes(nodes, "nodes", allow_empty=False) + edges = _normalize_edges(directed_edges) + observed = _normalize_nodes(observed_nodes, "observed_nodes", allow_empty=True) + node_set = set(nodes) + unknown_observed = set(observed) - node_set + if unknown_observed: + raise ValueError("observed_nodes must be a subset of nodes.") + for source, target in edges: + if source not in node_set or target not in node_set: + raise ValueError("Every directed edge endpoint must be a graph node.") + if source == target: + raise ValueError("Directed self-loops are not admissible in a DAG.") + if not _is_acyclic(nodes, edges): + raise ValueError("directed_edges must define an acyclic graph.") + return super().__new__(cls, nodes, edges, observed) + + @classmethod + def _make(cls, iterable): + values = tuple(iterable) + if len(values) != 3: + raise TypeError("CausalDAG._make requires exactly three values.") + return cls(*values) + + def _replace(self, **changes): + unknown = set(changes) - set(self._fields) + if unknown: + names = ", ".join(sorted(unknown)) + raise TypeError(f"Unexpected CausalDAG field names: {names}.") + return type(self)( + changes.get("nodes", self.nodes), + changes.get("directed_edges", self.directed_edges), + changes.get("observed_nodes", self.observed_nodes), + ) + + +class PathEvidence(NamedTuple): + """Path-relative collider, blocking, and direction evidence.""" + + nodes: PathTuple + colliders: NodeTuple + noncolliders: NodeTuple + conditioned_noncolliders: NodeTuple + activated_colliders: NodeTuple + blocking_colliders: NodeTuple + is_open: bool + is_directed_from_left: bool + is_backdoor_from_left: bool + + +class DSeparationEvidence(NamedTuple): + """Complete simple-path evidence for one d-separation query.""" + + left: NodeTuple + right: NodeTuple + conditioned: NodeTuple + separated: bool + paths: tuple[PathEvidence, ...] + + +class BackdoorAdjustmentEvidence(NamedTuple): + """Evidence for the ordinary total-effect back-door criterion.""" + + treatment: str + outcome: str + adjustment_set: NodeTuple + all_observed: bool + endpoints_in_set: NodeTuple + descendants_in_set: NodeTuple + open_backdoor_paths: tuple[PathEvidence, ...] + admissible: bool + + +class FrontdoorAdjustmentEvidence(NamedTuple): + """Graphical front-door evidence; positivity remains an external premise.""" + + treatment: str + outcome: str + mediator_set: NodeTuple + all_observed: bool + endpoints_in_set: NodeTuple + directed_path_exists: bool + unintercepted_directed_paths: tuple[PathEvidence, ...] + open_treatment_mediator_backdoor_paths: tuple[PathEvidence, ...] + open_mediator_outcome_backdoor_paths_given_treatment: tuple[PathEvidence, ...] + graphically_admissible: bool + positivity_required: bool + + +class InstrumentEvidence(NamedTuple): + """Separate Pearl-graphical and simple CFI instrument premises.""" + + instrument: str + treatment: str + outcome: str + conditioned: NodeTuple + all_observed: bool + conditioning_disjoint: bool + conditioning_unaffected_by_treatment: bool + pearl_relevance: bool + pearl_exclusion_exogeneity: bool + pearl_graphical: bool + cfi_direct_relevance: bool + cfi_full_mediation: bool + cfi_exogeneity: bool + cfi_simple: bool + + +class NodeRoleEvidence(NamedTuple): + """Non-exclusive, path-relative evidence for one internal graph node.""" + + node: str + treatment: str + outcome: str + collider_on_paths: tuple[PathTuple, ...] + noncollider_on_paths: tuple[PathTuple, ...] + mediator_on_directed_paths: tuple[PathTuple, ...] + backdoor_noncollider_on_paths: tuple[PathTuple, ...] + common_cause_paths: tuple[PathTuple, ...] + descendant_of_treatment: bool + ancestor_of_outcome: bool + + +def _validate_dag(dag) -> CausalDAG: + if not isinstance(dag, CausalDAG): + raise TypeError("dag must be a CausalDAG.") + return dag + + +def _known_node_set(dag: CausalDAG, value, subject: str, *, allow_empty: bool): + nodes = _normalize_nodes(value, subject, allow_empty=allow_empty) + if not set(nodes).issubset(dag.nodes): + raise ValueError(f"{subject} must contain only known graph nodes.") + return nodes + + +def _validate_endpoints(dag: CausalDAG, treatment, outcome) -> tuple[str, str]: + treatment = _require_node(treatment, "treatment") + outcome = _require_node(outcome, "outcome") + if treatment == outcome: + raise ValueError("treatment and outcome must be distinct nodes.") + if treatment not in dag.nodes or outcome not in dag.nodes: + raise ValueError("treatment and outcome must be known graph nodes.") + observed = set(dag.observed_nodes) + if treatment not in observed or outcome not in observed: + raise ValueError("treatment and outcome must both be observed nodes.") + return treatment, outcome + + +def _descendants(dag: CausalDAG, node: str, edges: Optional[EdgeTuple] = None): + active_edges = dag.directed_edges if edges is None else edges + children = _children(dag.nodes, active_edges) + visited = set() + pending = list(children[node]) + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + pending.extend(children[current]) + return visited + + +def _ancestors(dag: CausalDAG, node: str): + parents = _parents(dag.nodes, dag.directed_edges) + visited = set() + pending = list(parents[node]) + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + pending.extend(parents[current]) + return visited + + +def _normalize_limit(value, subject: str) -> Optional[int]: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{subject} must be a positive integer or None.") + if value < 1: + raise ValueError(f"{subject} must be positive.") + return value + + +def _has_directed_path( + dag: CausalDAG, + left: str, + right: str, + edges: Optional[EdgeTuple] = None, + blocked: Iterable[str] = (), +) -> bool: + active_edges = dag.directed_edges if edges is None else edges + blocked_nodes = set(blocked) + if left in blocked_nodes or right in blocked_nodes: + return False + children = _children(dag.nodes, active_edges) + pending = [left] + visited = set() + while pending: + node = pending.pop() + if node == right: + return True + if node in visited: + continue + visited.add(node) + pending.extend( + child + for child in children[node] + if child not in visited and child not in blocked_nodes + ) + return False + + +def _is_d_separated( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, +) -> bool: + """Test d-separation through ancestral-subgraph moralization.""" + + active_edges = dag.directed_edges if edges is None else edges + parents = _parents(dag.nodes, active_edges) + relevant = set(left).union(right, conditioned) + pending = list(relevant) + while pending: + node = pending.pop() + for parent in parents[node]: + if parent not in relevant: + relevant.add(parent) + pending.append(parent) + + moral = {node: set() for node in relevant} + for source, target in active_edges: + if source in relevant and target in relevant: + moral[source].add(target) + moral[target].add(source) + for child in relevant: + child_parents = sorted( + parent for parent in parents[child] if parent in relevant + ) + for index, first in enumerate(child_parents): + for second in child_parents[index + 1 :]: + moral[first].add(second) + moral[second].add(first) + + blocked = set(conditioned) + targets = set(right) + reachable = [node for node in left if node not in blocked] + visited = set() + while reachable: + node = reachable.pop() + if node in targets: + return False + if node in visited: + continue + visited.add(node) + reachable.extend( + neighbor + for neighbor in moral[node] + if neighbor not in blocked and neighbor not in visited + ) + return True + + +def _without_outgoing(edges: EdgeTuple, nodes) -> EdgeTuple: + removed = set(nodes) + return tuple(edge for edge in edges if edge[0] not in removed) + + +def _without_incoming(edges: EdgeTuple, nodes) -> EdgeTuple: + removed = set(nodes) + return tuple(edge for edge in edges if edge[1] not in removed) + + +def _simple_paths( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + edges: Optional[EdgeTuple] = None, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathTuple, ...]: + active_edges = dag.directed_edges if edges is None else edges + adjacency = {node: set() for node in dag.nodes} + for source, target in active_edges: + adjacency[source].add(target) + adjacency[target].add(source) + paths = [] + states = 0 + for start in left: + for finish in right: + stack = [(start, (start,), frozenset((start,)))] + while stack: + states += 1 + if max_path_states is not None and states > max_path_states: + raise ValueError( + "Complete path search exceeds max_path_states; " + "increase the explicit budget or simplify the query." + ) + node, path, visited = stack.pop() + if node == finish: + paths.append(path) + if max_paths is not None and len(paths) > max_paths: + raise ValueError( + "Complete path evidence exceeds max_paths; " + "increase the explicit budget or simplify the query." + ) + continue + for neighbor in sorted(adjacency[node], reverse=True): + if neighbor not in visited: + stack.append( + (neighbor, path + (neighbor,), visited | {neighbor}) + ) + return tuple(sorted(paths, key=lambda path: (len(path), path))) + + +def _path_evidence( + dag: CausalDAG, + path: PathTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, +) -> PathEvidence: + active_edges = dag.directed_edges if edges is None else edges + edge_set = set(active_edges) + conditioned_set = set(conditioned) + colliders = [] + noncolliders = [] + for index in range(1, len(path) - 1): + previous, node, following = path[index - 1 : index + 2] + if (previous, node) in edge_set and (following, node) in edge_set: + colliders.append(node) + else: + noncolliders.append(node) + activated = [] + blocking = [] + for collider in colliders: + collider_family = {collider} | _descendants(dag, collider, active_edges) + if collider_family.intersection(conditioned_set): + activated.append(collider) + else: + blocking.append(collider) + conditioned_noncolliders = tuple( + node for node in noncolliders if node in conditioned_set + ) + is_directed = all( + (path[index], path[index + 1]) in edge_set for index in range(len(path) - 1) + ) + is_backdoor = len(path) > 1 and (path[1], path[0]) in edge_set + return PathEvidence( + nodes=path, + colliders=tuple(colliders), + noncolliders=tuple(noncolliders), + conditioned_noncolliders=conditioned_noncolliders, + activated_colliders=tuple(activated), + blocking_colliders=tuple(blocking), + is_open=not conditioned_noncolliders and not blocking, + is_directed_from_left=is_directed, + is_backdoor_from_left=is_backdoor, + ) + + +def _d_separation_from_normalized( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> DSeparationEvidence: + paths = tuple( + _path_evidence(dag, path, conditioned, edges) + for path in _simple_paths( + dag, + left, + right, + edges, + max_paths, + max_path_states, + ) + ) + return DSeparationEvidence( + left=left, + right=right, + conditioned=conditioned, + separated=all(not path.is_open for path in paths), + paths=paths, + ) + + +def d_separation( + dag: CausalDAG, + left: Iterable[str], + right: Iterable[str], + conditioned: Iterable[str] = (), + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> DSeparationEvidence: + """Return complete simple-path evidence for a d-separation query. + + The three node sets must be pairwise disjoint. ``separated`` describes a + graphical implication for distributions compatible with ``dag``; its + negation is not a sample-level dependence test. Complete path evidence is + output-sensitive. ``max_paths`` bounds returned witnesses and + ``max_path_states`` bounds search expansions. ``None`` explicitly opts + out of the respective bound. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + left_nodes = _known_node_set(dag, left, "left", allow_empty=False) + right_nodes = _known_node_set(dag, right, "right", allow_empty=False) + conditioned_nodes = _known_node_set( + dag, conditioned, "conditioned", allow_empty=True + ) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + if ( + set(left_nodes).intersection(right_nodes) + or set(left_nodes).intersection(conditioned_nodes) + or set(right_nodes).intersection(conditioned_nodes) + ): + raise ValueError("left, right, and conditioned must be pairwise disjoint.") + return _d_separation_from_normalized( + dag, + left_nodes, + right_nodes, + conditioned_nodes, + max_paths=max_paths, + max_path_states=max_path_states, + ) + + +def check_backdoor_adjustment_set( + dag: CausalDAG, + treatment: str, + outcome: str, + adjustment_set: Iterable[str] = (), + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> BackdoorAdjustmentEvidence: + """Check one observed set against the ordinary back-door criterion. + + ``max_paths`` bounds returned witnesses and ``max_path_states`` bounds + search expansions. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + adjustment = _known_node_set( + dag, adjustment_set, "adjustment_set", allow_empty=True + ) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + observed = set(dag.observed_nodes) + endpoints = tuple(sorted(set(adjustment).intersection((treatment, outcome)))) + descendants = tuple( + sorted(set(adjustment).intersection(_descendants(dag, treatment))) + ) + backdoor_edges = _without_outgoing(dag.directed_edges, (treatment,)) + separation = _d_separation_from_normalized( + dag, + (treatment,), + (outcome,), + adjustment, + backdoor_edges, + max_paths, + max_path_states, + ) + open_paths = tuple(path for path in separation.paths if path.is_open) + all_observed = set(adjustment).issubset(observed) + admissible = all_observed and not endpoints and not descendants and not open_paths + return BackdoorAdjustmentEvidence( + treatment=treatment, + outcome=outcome, + adjustment_set=adjustment, + all_observed=all_observed, + endpoints_in_set=endpoints, + descendants_in_set=descendants, + open_backdoor_paths=open_paths, + admissible=admissible, + ) + + +def _inclusion_minimal_sets(valid_sets): + ordered = sorted(valid_sets, key=lambda item: (len(item), item)) + result = [] + for candidate in ordered: + candidate_set = set(candidate) + if not any(set(existing).issubset(candidate_set) for existing in result): + result.append(candidate) + return tuple(result) + + +def _search_inclusion_minimal_sets( + candidates: NodeTuple, + is_valid: Callable[[NodeTuple], bool], + max_candidates: Optional[int], +) -> tuple[NodeTuple, ...]: + valid_sets = [] + selected = [] + frames = [[0, False]] + work = 0 + while frames: + next_index, evaluated = frames[-1] + candidate = tuple(selected) + if not evaluated: + work += 1 + len(candidate) + if max_candidates is not None and work > max_candidates: + raise ValueError( + "Exact minimal-set search exceeds max_candidates; " + "increase the explicit budget or simplify the graph." + ) + candidate_set = set(candidate) + if any(set(existing).issubset(candidate_set) for existing in valid_sets): + frames.pop() + if selected: + selected.pop() + continue + if is_valid(candidate): + if not candidate: + return ((),) + valid_sets.append(candidate) + frames.pop() + selected.pop() + continue + frames[-1][1] = True + continue + if next_index >= len(candidates): + frames.pop() + if selected: + selected.pop() + continue + frames[-1][0] += 1 + selected.append(candidates[next_index]) + frames.append([next_index + 1, False]) + return _inclusion_minimal_sets(valid_sets) + + +def minimal_backdoor_adjustment_sets( + dag: CausalDAG, + treatment: str, + outcome: str, + *, + max_candidates: Optional[int] = _DEFAULT_MAX_CANDIDATES, +) -> tuple[NodeTuple, ...]: + """Return all inclusion-minimal observed back-door adjustment sets. + + ``((),)`` means that the empty set is admissible. ``()`` means that no + admissible set exists among the observed, non-descendant graph nodes. + Search is exact and stops with ``ValueError`` before exceeding the explicit + ``max_candidates`` budget; no partial result is returned. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + max_candidates = _normalize_limit(max_candidates, "max_candidates") + descendants = _descendants(dag, treatment) + candidates = tuple( + node + for node in dag.observed_nodes + if node not in {treatment, outcome} and node not in descendants + ) + backdoor_edges = _without_outgoing(dag.directed_edges, (treatment,)) + + def is_valid(candidate: NodeTuple) -> bool: + return _is_d_separated(dag, (treatment,), (outcome,), candidate, backdoor_edges) + + return _search_inclusion_minimal_sets(candidates, is_valid, max_candidates) + + +def _directed_path_tuples( + dag: CausalDAG, + left: str, + right: str, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathTuple, ...]: + children = _children(dag.nodes, dag.directed_edges) + parents = _parents(dag.nodes, dag.directed_edges) + can_reach_right = {right} + pending = [right] + while pending: + node = pending.pop() + for parent in parents[node]: + if parent not in can_reach_right: + can_reach_right.add(parent) + pending.append(parent) + paths = [] + stack = [(left, (left,))] + states = 0 + while stack: + states += 1 + if max_path_states is not None and states > max_path_states: + raise ValueError( + "Complete directed-path search exceeds max_path_states; " + "increase the explicit budget or simplify the query." + ) + node, path = stack.pop() + if node == right: + paths.append(path) + if max_paths is not None and len(paths) > max_paths: + raise ValueError( + "Complete directed-path evidence exceeds max_paths; " + "increase the explicit budget or simplify the query." + ) + continue + for child in sorted(children[node], reverse=True): + if child in can_reach_right: + stack.append((child, path + (child,))) + return tuple(sorted(paths, key=lambda path: (len(path), path))) + + +def _directed_paths( + dag: CausalDAG, + left: str, + right: str, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathEvidence, ...]: + return tuple( + _path_evidence(dag, path, ()) + for path in _directed_path_tuples(dag, left, right, max_paths, max_path_states) + ) + + +def check_frontdoor_adjustment_set( + dag: CausalDAG, + treatment: str, + outcome: str, + mediator_set: Iterable[str], + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> FrontdoorAdjustmentEvidence: + """Check Pearl's front-door conditions under PROTOCOL-01 eligibility. + + The policy layer requires a nonempty observed mediator set and at least one + directed treatment-to-outcome path, avoiding vacuous eligibility when no + causal path exists. A positive result is deliberately called + ``graphically_admissible``. Positivity is required by the identification + theorem but cannot be checked from a DAG. ``max_paths`` bounds returned + witnesses and ``max_path_states`` bounds search expansions. Evidence is + never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + mediators = _known_node_set(dag, mediator_set, "mediator_set", allow_empty=True) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + observed = set(dag.observed_nodes) + endpoints = tuple(sorted(set(mediators).intersection((treatment, outcome)))) + directed_paths = _directed_paths( + dag, treatment, outcome, max_paths, max_path_states + ) + unintercepted = tuple( + path + for path in directed_paths + if not set(path.nodes[1:-1]).intersection(mediators) + ) + treatment_edges = _without_outgoing(dag.directed_edges, (treatment,)) + treatment_mediator = ( + _d_separation_from_normalized( + dag, + (treatment,), + mediators, + (), + treatment_edges, + max_paths, + max_path_states, + ) + if mediators and not endpoints + else None + ) + mediator_edges = _without_outgoing(dag.directed_edges, mediators) + mediator_outcome = ( + _d_separation_from_normalized( + dag, + mediators, + (outcome,), + (treatment,), + mediator_edges, + max_paths, + max_path_states, + ) + if mediators and not endpoints + else None + ) + open_treatment_mediator = ( + tuple(path for path in treatment_mediator.paths if path.is_open) + if treatment_mediator is not None + else () + ) + open_mediator_outcome = ( + tuple(path for path in mediator_outcome.paths if path.is_open) + if mediator_outcome is not None + else () + ) + all_observed = set(mediators).issubset(observed) + directed_path_exists = bool(directed_paths) + admissible = ( + bool(mediators) + and all_observed + and not endpoints + and directed_path_exists + and not unintercepted + and not open_treatment_mediator + and not open_mediator_outcome + ) + return FrontdoorAdjustmentEvidence( + treatment=treatment, + outcome=outcome, + mediator_set=mediators, + all_observed=all_observed, + endpoints_in_set=endpoints, + directed_path_exists=directed_path_exists, + unintercepted_directed_paths=unintercepted, + open_treatment_mediator_backdoor_paths=open_treatment_mediator, + open_mediator_outcome_backdoor_paths_given_treatment=open_mediator_outcome, + graphically_admissible=admissible, + positivity_required=True, + ) + + +def minimal_frontdoor_adjustment_sets( + dag: CausalDAG, + treatment: str, + outcome: str, + *, + max_candidates: Optional[int] = _DEFAULT_MAX_CANDIDATES, +) -> tuple[NodeTuple, ...]: + """Return all inclusion-minimal observed PROTOCOL-eligible front-door sets. + + Search is exact and stops with ``ValueError`` before exceeding the explicit + ``max_candidates`` budget; no partial result is returned. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + max_candidates = _normalize_limit(max_candidates, "max_candidates") + if not _has_directed_path(dag, treatment, outcome): + return () + candidates = tuple( + node for node in dag.observed_nodes if node not in {treatment, outcome} + ) + treatment_edges = _without_outgoing(dag.directed_edges, (treatment,)) + + def is_valid(candidate: NodeTuple) -> bool: + if not candidate: + return False + if _has_directed_path(dag, treatment, outcome, blocked=candidate): + return False + if not _is_d_separated(dag, (treatment,), candidate, (), treatment_edges): + return False + mediator_edges = _without_outgoing(dag.directed_edges, candidate) + return _is_d_separated(dag, candidate, (outcome,), (treatment,), mediator_edges) + + return _search_inclusion_minimal_sets(candidates, is_valid, max_candidates) + + +def check_instrument( + dag: CausalDAG, + instrument: str, + treatment: str, + outcome: str, + conditioned: Iterable[str] = (), +) -> InstrumentEvidence: + """Check Pearl's conditional and CFI's simple instrument profiles. + + ``pearl_graphical`` uses d-connected relevance and a conditioning set. + ``conditioned`` applies only to that Pearl profile. Every ``cfi_*`` field + evaluates the stricter, unconditional simple CFI profile on the same DAG + and can remain true when ``conditioned`` is nonempty. Neither Boolean is + an effect-estimation or nonparametric-identification claim. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + instrument = _require_node(instrument, "instrument") + if instrument not in dag.nodes: + raise ValueError("instrument must be a known graph node.") + if instrument in {treatment, outcome}: + raise ValueError("instrument, treatment, and outcome must be distinct.") + controls = _known_node_set(dag, conditioned, "conditioned", allow_empty=True) + forbidden = {instrument, treatment, outcome} + conditioning_disjoint = not set(controls).intersection(forbidden) + observed = set(dag.observed_nodes) + all_observed = {instrument, treatment, outcome}.union(controls).issubset(observed) + unaffected = not set(controls).intersection(_descendants(dag, treatment)) + if conditioning_disjoint: + treatment_edges = _without_incoming(dag.directed_edges, (treatment,)) + pearl_relevance = not _is_d_separated( + dag, (instrument,), (treatment,), controls + ) + pearl_exclusion = _is_d_separated( + dag, (instrument,), (outcome,), controls, treatment_edges + ) + else: + pearl_relevance = False + pearl_exclusion = False + pearl_graphical = ( + all_observed + and conditioning_disjoint + and unaffected + and pearl_relevance + and pearl_exclusion + ) + + edge_set = set(dag.directed_edges) + cfi_direct = (instrument, treatment) in edge_set + instrument_outcome_path = _has_directed_path(dag, instrument, outcome) + bypasses_treatment = _has_directed_path( + dag, instrument, outcome, blocked=(treatment,) + ) + cfi_full_mediation = instrument_outcome_path and not bypasses_treatment + instrument_edges = _without_outgoing(dag.directed_edges, (instrument,)) + cfi_exogeneity = _is_d_separated( + dag, (instrument,), (outcome,), (), instrument_edges + ) + cfi_simple = ( + instrument in observed and cfi_direct and cfi_full_mediation and cfi_exogeneity + ) + return InstrumentEvidence( + instrument=instrument, + treatment=treatment, + outcome=outcome, + conditioned=controls, + all_observed=all_observed, + conditioning_disjoint=conditioning_disjoint, + conditioning_unaffected_by_treatment=unaffected, + pearl_relevance=pearl_relevance, + pearl_exclusion_exogeneity=pearl_exclusion, + pearl_graphical=pearl_graphical, + cfi_direct_relevance=cfi_direct, + cfi_full_mediation=cfi_full_mediation, + cfi_exogeneity=cfi_exogeneity, + cfi_simple=cfi_simple, + ) + + +def causal_role_evidence( + dag: CausalDAG, + treatment: str, + outcome: str, + node: str, + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> NodeRoleEvidence: + """Return non-exclusive path witnesses for one internal graph node. + + ``max_paths`` bounds returned witnesses and ``max_path_states`` bounds + search expansions. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + node = _require_node(node, "node") + if node not in dag.nodes: + raise ValueError("node must be a known graph node.") + if node in {treatment, outcome}: + raise ValueError("node must be distinct from treatment and outcome.") + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + paths = tuple( + _path_evidence(dag, path, ()) + for path in _simple_paths( + dag, + (treatment,), + (outcome,), + max_paths=max_paths, + max_path_states=max_path_states, + ) + ) + collider_paths = tuple(path.nodes for path in paths if node in path.colliders) + noncollider_paths = tuple(path.nodes for path in paths if node in path.noncolliders) + mediator_paths = tuple( + path.nodes + for path in paths + if path.is_directed_from_left and node in path.nodes[1:-1] + ) + backdoor_noncollider_paths = tuple( + path.nodes + for path in paths + if path.is_backdoor_from_left and node in path.noncolliders + ) + edge_set = set(dag.directed_edges) + common_cause_paths = [] + for path in paths: + if node not in path.noncolliders: + continue + index = path.nodes.index(node) + directed_to_treatment = all( + (path.nodes[position], path.nodes[position - 1]) in edge_set + for position in range(index, 0, -1) + ) + directed_to_outcome = all( + (path.nodes[position], path.nodes[position + 1]) in edge_set + for position in range(index, len(path.nodes) - 1) + ) + if directed_to_treatment and directed_to_outcome: + common_cause_paths.append(path.nodes) + return NodeRoleEvidence( + node=node, + treatment=treatment, + outcome=outcome, + collider_on_paths=collider_paths, + noncollider_on_paths=noncollider_paths, + mediator_on_directed_paths=mediator_paths, + backdoor_noncollider_on_paths=backdoor_noncollider_paths, + common_cause_paths=tuple(common_cause_paths), + descendant_of_treatment=node in _descendants(dag, treatment), + ancestor_of_outcome=node in _ancestors(dag, outcome), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/optimizer.py b/src/RiskLabAI/causal_factor_analysis/optimizer.py new file mode 100644 index 0000000..c5ade99 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/optimizer.py @@ -0,0 +1,217 @@ +r"""Minimum-variance allocation subject to target factor exposures. + +The optimization problem is + +.. math:: + + \min_{\omega}\; \omega^\mathsf{T} V \omega + \quad\text{subject to}\quad + B^\mathsf{T}\omega = c, + +where ``V`` is an asset covariance matrix, ``B`` contains asset-by-factor +exposures, and ``c`` is the requested factor-exposure vector. Equations (3)-(6) +and (A-1)-(A-7) of López de Prado, Lipton, and Zoonekynd (2025) give + +.. math:: + + \omega^* = V^{-1}B(B^\mathsf{T}V^{-1}B)^{-1}c. + +This implementation uses linear solves rather than forming either inverse. It +implements only the published allocation problem; factor-exposure estimation +and causal-model selection are separate responsibilities. + +References +---------- +López de Prado, M., Lipton, A., and Zoonekynd, V. (2025). "Causal Factor +Analysis Is a Necessary Condition for Investment Efficiency." Equations +(3)-(6) and Appendix A. SSRN 5131050. +""" + +from __future__ import annotations + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +__all__ = ["minimum_variance_factor_weights"] + +_SYMMETRY_RTOL = 1e-10 +_SYMMETRY_ATOL = 1e-12 +_FEASIBILITY_RTOL = 1e-10 +_FEASIBILITY_ATOL = 1e-12 + + +def _as_real_array(values: ArrayLike, name: str) -> NDArray[np.float64]: + """Convert an input to a real ``float64`` array with a stable error.""" + try: + raw = np.asarray(values) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be a numeric array") from exc + if np.iscomplexobj(raw): + raise ValueError(f"{name} must be real-valued") + try: + return np.asarray(values, dtype=np.float64) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be a numeric array") from exc + + +def minimum_variance_factor_weights( + covariance: ArrayLike, + factor_exposures: ArrayLike, + target_exposures: ArrayLike, +) -> NDArray[np.float64]: + r"""Compute minimum-variance weights with prescribed factor exposures. + + Parameters + ---------- + covariance : array-like of shape (n_assets, n_assets) + Symmetric positive-definite asset covariance matrix :math:`V`. + factor_exposures : array-like of shape (n_assets, n_factors) + Full-column-rank asset-by-factor exposure matrix :math:`B`. + target_exposures : array-like of shape (n_factors,) + Requested factor exposures :math:`c`. + + Returns + ------- + numpy.ndarray + One-dimensional ``float64`` vector of optimal asset weights. + + Raises + ------ + ValueError + If an input has the wrong shape, contains a non-finite or complex value, + if ``covariance`` is not symmetric positive definite, or if + ``factor_exposures`` is not full column rank or its constraints cannot + be resolved to numerical precision. + + Notes + ----- + For square full-rank exposure systems, Equation (5) is evaluated directly + by solving :math:`B^\mathsf{T}\omega=c`. Otherwise, the published inverse + formula is evaluated by Cholesky-whitening the exposures and solving the + resulting minimum-norm problem with reduced QR. This avoids both explicit + inverses and the condition-number squaring of a normal-equation Gram matrix. + Covariance and constraint units are normalized by positive scalars without + changing the optimizer, and the requested exposures are verified before + return. No diagonal jitter or pseudoinverse is applied: singular covariance + matrices and redundant factor constraints are outside the stated assumptions + of the model. + + Examples + -------- + >>> covariance = np.diag([1.0, 2.0, 4.0]) + >>> exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + >>> targets = np.array([0.0, 1.0]) + >>> minimum_variance_factor_weights(covariance, exposures, targets) + array([-0.28571429, 0.71428571, 0.28571429]) + """ + covariance_array = _as_real_array(covariance, "covariance") + exposure_array = _as_real_array(factor_exposures, "factor_exposures") + target_array = _as_real_array(target_exposures, "target_exposures") + + if covariance_array.ndim != 2: + raise ValueError("covariance must be a 2-D square matrix") + n_assets, covariance_columns = covariance_array.shape + if n_assets == 0 or covariance_columns != n_assets: + raise ValueError("covariance must be a non-empty square matrix") + + if exposure_array.ndim != 2: + raise ValueError("factor_exposures must be a 2-D matrix") + if exposure_array.shape[0] != n_assets: + raise ValueError( + "factor_exposures must have the same number of rows as covariance" + ) + n_factors = exposure_array.shape[1] + if n_factors == 0: + raise ValueError("factor_exposures must contain at least one factor column") + if n_factors > n_assets: + raise ValueError("factor_exposures cannot have more columns than rows") + + if target_array.ndim != 1: + raise ValueError("target_exposures must be a 1-D vector") + if target_array.shape[0] != n_factors: + raise ValueError( + "target_exposures length must equal the number of factor columns" + ) + + if not np.all(np.isfinite(covariance_array)): + raise ValueError("covariance must contain only finite values") + if not np.all(np.isfinite(exposure_array)): + raise ValueError("factor_exposures must contain only finite values") + if not np.all(np.isfinite(target_array)): + raise ValueError("target_exposures must contain only finite values") + + covariance_scale = float(np.max(np.abs(covariance_array))) + if covariance_scale == 0.0: + raise ValueError("covariance must be positive definite") + scaled_covariance = covariance_array / covariance_scale + if not np.allclose( + scaled_covariance, + scaled_covariance.T, + rtol=_SYMMETRY_RTOL, + atol=_SYMMETRY_ATOL, + ): + raise ValueError("covariance must be symmetric") + scaled_covariance = 0.5 * scaled_covariance + 0.5 * scaled_covariance.T + + try: + covariance_cholesky = np.linalg.cholesky(scaled_covariance) + except np.linalg.LinAlgError as exc: + raise ValueError("covariance must be positive definite") from exc + + maximum_exposures = np.max(np.abs(exposure_array), axis=0) + if np.any(maximum_exposures == 0.0): + raise ValueError("factor_exposures must have full column rank") + target_scales = np.abs(target_array) / (np.finfo(np.float64).max / 2.0) + constraint_scales = np.maximum(maximum_exposures, target_scales) + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + scaled_exposures = exposure_array / constraint_scales + scaled_targets = target_array / constraint_scales + target_underflow = (target_array != 0.0) & (scaled_targets == 0.0) + if ( + np.any(target_underflow) + or not np.all(np.isfinite(scaled_exposures)) + or not np.all(np.isfinite(scaled_targets)) + ): + raise ValueError( + "factor constraints could not be represented at numerical precision" + ) + + try: + whitened_exposures = np.linalg.solve(covariance_cholesky, scaled_exposures) + if not np.all(np.isfinite(whitened_exposures)): + raise ValueError("factor_exposures produce non-finite whitened values") + if np.linalg.matrix_rank(whitened_exposures) != n_factors: + raise ValueError("factor_exposures must have full column rank") + + if n_factors == n_assets: + weights = np.linalg.solve(scaled_exposures.T, scaled_targets) + else: + orthonormal_basis, triangular_factor = np.linalg.qr( + whitened_exposures, mode="reduced" + ) + transformed_targets = np.linalg.solve(triangular_factor.T, scaled_targets) + whitened_weights = orthonormal_basis @ transformed_targets + weights = np.linalg.solve(covariance_cholesky.T, whitened_weights) + except np.linalg.LinAlgError as exc: + raise ValueError("factor_exposures must have full column rank") from exc + + weights = np.asarray(weights, dtype=np.float64).reshape(n_assets) + if not np.all(np.isfinite(weights)): + raise ValueError("the allocation produced non-finite weights") + + with np.errstate(over="ignore", invalid="ignore"): + achieved_exposures = scaled_exposures.T @ weights + absolute_residuals = np.abs(achieved_exposures - scaled_targets) + feasibility_tolerances = np.where( + scaled_targets != 0.0, + _FEASIBILITY_RTOL * np.abs(scaled_targets), + _FEASIBILITY_ATOL, + ) + if not np.all(np.isfinite(achieved_exposures)) or np.any( + absolute_residuals > feasibility_tolerances + ): + raise ValueError( + "factor constraints could not be resolved to numerical precision" + ) + + return weights.copy() diff --git a/src/RiskLabAI/causal_factor_analysis/protocol.py b/src/RiskLabAI/causal_factor_analysis/protocol.py new file mode 100644 index 0000000..35edfcf --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/protocol.py @@ -0,0 +1,1780 @@ +# ruff: noqa: UP045 +"""Immutable evidence contract for the causal-factor research protocol. + +The module represents the seven stages in Lopez de Prado and Zoonekynd +(2026), PDF pages 18-22 (journal pages 27-31) and Exhibit 10. It validates a +completed research record; it does not execute discovery, estimation, +portfolio construction, backtesting, or multiple-testing algorithms. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from numbers import Integral, Real +from typing import Optional + +__all__ = [ + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol", +] + + +_STAGE_NAMES = ( + "Variable Selection", + "Causal Discovery", + "Causal Adjustment Set", + "Causal Explanatory and Predictive Power", + "Causal Portfolio Construction", + "Backtest", + "Multiple Testing Adjustments", +) + +_VARIABLE_SELECTION_METHODS = { + "MUTUAL_INFORMATION", + "SHAPLEY_VALUES", + "MEAN_DECREASE_IMPURITY", + "PERMUTATION_FEATURE_IMPORTANCE", +} +_RESEARCH_PURPOSES = {"CAUSAL_ATTRIBUTION", "RISK_PREMIA_HARVESTING"} +_DISCOVERY_METHODS = { + "PC", + "LINGAM", + "ECONOMIC_REASONING", + "EX_ANTE_VIEWS", + "OBSERVED_OUTCOMES", + "PEER_REVIEWED_ASSUMPTIONS", + "DOMAIN_EXPERTISE", +} +_ADJUSTMENT_METHODS = { + "BACKDOOR_ADJUSTMENT", + "FRONT_DOOR_ADJUSTMENT", + "INSTRUMENTAL_VARIABLES", +} +_VALIDATION_METHODS = { + "PURGED_CROSS_VALIDATION", + "WALK_FORWARD", + "RESAMPLING", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", +} +_PURGED_VALIDATION_METHODS = { + "PURGED_CROSS_VALIDATION", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", +} +_METRICS_BY_TASK = { + "PROBABILITY": {"LOG_LOSS", "BRIER_SCORE"}, + "RANKING": { + "ROC_CURVE", + "PRECISION_RECALL_CURVE", + "MEAN_RECIPROCAL_RANK", + "CLASSIFICATION_ACCURACY", + }, + "RETURN_SIZE": { + "MEAN_SQUARED_ERROR", + "R_SQUARED", + "SPEARMAN_CORRELATION", + }, +} +_PORTFOLIO_METHODS = { + "POSITION_SIZING", + "EXPOSURE_CONTROL", + "ECONOMIC_RATIONALE", + "FRAGILITY_STRESS_TEST", + "TRANSACTION_COST_OPTIMIZATION", + "TRANSFER_COEFFICIENT", +} +_BACKTEST_METHODS = { + "WALK_FORWARD", + "RESAMPLING", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", + "MONTE_CARLO", +} +_MULTIPLE_TESTING_METHODS = { + "HOLM", + "HOCHBERG", + "BENJAMINI_HOCHBERG", + "DEFLATED_SHARPE_RATIO", +} +_P_VALUE_METHODS = {"HOLM", "HOCHBERG", "BENJAMINI_HOCHBERG"} +_REFIT_COMPONENTS = { + "VARIABLE_SELECTION", + "CAUSAL_ESTIMATION", + "PORTFOLIO_CONSTRUCTION", +} + + +def _freeze_tuple(value): + if isinstance(value, list): + return tuple(value) + return value + + +def _freeze_nested_tuple(value): + outer = _freeze_tuple(value) + if isinstance(outer, tuple): + return tuple(_freeze_tuple(item) for item in outer) + return outer + + +def _freeze_partitions(value): + outer = _freeze_tuple(value) + if not isinstance(outer, tuple): + return outer + result = [] + for item in outer: + pair = _freeze_tuple(item) + if isinstance(pair, tuple) and len(pair) == 2: + pair = (pair[0], _freeze_tuple(pair[1])) + result.append(pair) + return tuple(result) + + +def _store_normalized_field(instance, field_name, value): + """Store a normalized value while a frozen record is being constructed.""" + + instance.__dict__[field_name] = value + + +@dataclass(frozen=True) +class EventHorizon: + """Observation identifier and its half-open return interval.""" + + observation_id: str + start: Real + end: Real + + +@dataclass(frozen=True) +class FoldEvidence: + """Membership and training-only pipeline components for one fold.""" + + fold_id: str + path_id: str + train_ids: tuple[str, ...] + test_ids: tuple[str, ...] + fit_ids: tuple[str, ...] + prediction_ids: tuple[str, ...] + refit_id: str + refit_components: tuple[str, ...] + + def __post_init__(self): + for name in ( + "train_ids", + "test_ids", + "fit_ids", + "prediction_ids", + "refit_components", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + + +@dataclass(frozen=True) +class ValidationEvidence: + """Temporal validation evidence shared by source stages that require it.""" + + method_labels: tuple[str, ...] + event_horizons: tuple[EventHorizon, ...] + folds: tuple[FoldEvidence, ...] + embargo: Real = 0.0 + + def __post_init__(self): + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + _store_normalized_field( + self, "event_horizons", _freeze_tuple(self.event_horizons) + ) + _store_normalized_field(self, "folds", _freeze_tuple(self.folds)) + + +@dataclass(frozen=True) +class VariableSelectionStage: + """Stage 1 evidence for candidate-variable selection and leakage controls.""" + + purpose: str + selected_variables: tuple[str, ...] + method_labels: tuple[str, ...] + overlapping_returns: bool + strong_time_dependence: bool + validation: Optional[ValidationEvidence] + + def __post_init__(self): + _store_normalized_field( + self, "selected_variables", _freeze_tuple(self.selected_variables) + ) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + + +@dataclass(frozen=True) +class CausalDiscoveryStage: + """Stage 2 evidence for an explicit, resolved causal graph.""" + + method_labels: tuple[str, ...] + graph_id: str + graph_nodes: tuple[str, ...] + directed_edges: tuple[tuple[str, str], ...] + graph_kind: str + ambiguous_edges: tuple[tuple[str, str], ...] + assumptions: tuple[str, ...] + + def __post_init__(self): + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + _store_normalized_field(self, "graph_nodes", _freeze_tuple(self.graph_nodes)) + _store_normalized_field( + self, "directed_edges", _freeze_nested_tuple(self.directed_edges) + ) + _store_normalized_field( + self, "ambiguous_edges", _freeze_nested_tuple(self.ambiguous_edges) + ) + _store_normalized_field(self, "assumptions", _freeze_tuple(self.assumptions)) + + +@dataclass(frozen=True) +class CausalAdjustmentSetStage: + """Stage 3 evidence for identification and admissible ordinary controls.""" + + treatment: str + outcome: str + method_label: str + identified: bool + admissible_adjustment_sets: tuple[tuple[str, ...], ...] + selected_adjustment_set: tuple[str, ...] + confounders: tuple[str, ...] + descendants: tuple[str, ...] + mediators: tuple[str, ...] + colliders: tuple[str, ...] + instruments: tuple[str, ...] + control_justifications: tuple[tuple[str, str], ...] + open_backdoor_paths: tuple[str, ...] + frontdoor_criteria_satisfied: Optional[bool] = None + instrument_relevance_satisfied: Optional[bool] = None + instrument_exclusion_satisfied: Optional[bool] = None + instrument_exogeneity_satisfied: Optional[bool] = None + + def __post_init__(self): + _store_normalized_field( + self, + "admissible_adjustment_sets", + _freeze_nested_tuple(self.admissible_adjustment_sets), + ) + for name in ( + "selected_adjustment_set", + "confounders", + "descendants", + "mediators", + "colliders", + "instruments", + "open_backdoor_paths", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + _store_normalized_field( + self, + "control_justifications", + _freeze_nested_tuple(self.control_justifications), + ) + + +@dataclass(frozen=True) +class CausalExplanatoryAndPredictivePowerStage: + """Stage 4 generalization evidence, separated by research purpose.""" + + task_types: tuple[str, ...] + estimator_label: str + explanatory_metric_labels: tuple[str, ...] + predictive_metric_labels: tuple[str, ...] + explanatory_evidence_id: Optional[str] + predictive_evidence_id: Optional[str] + naive_benchmark_id: str + validation: ValidationEvidence + multiclass_encoding: Optional[str] = None + averaging_method: Optional[str] = None + + def __post_init__(self): + _store_normalized_field(self, "task_types", _freeze_tuple(self.task_types)) + _store_normalized_field( + self, + "explanatory_metric_labels", + _freeze_tuple(self.explanatory_metric_labels), + ) + _store_normalized_field( + self, + "predictive_metric_labels", + _freeze_tuple(self.predictive_metric_labels), + ) + + +@dataclass(frozen=True) +class CausalPortfolioConstructionStage: + """Stage 5 evidence for a causally motivated implemented portfolio.""" + + method_labels: tuple[str, ...] + causal_exposures: tuple[str, ...] + controlled_unintended_exposures: tuple[str, ...] + cost_model_id: str + constraint_set_id: str + economic_rationale: str + fragility_scenarios: tuple[str, ...] + transfer_coefficient: Real + + def __post_init__(self): + for name in ( + "method_labels", + "causal_exposures", + "controlled_unintended_exposures", + "fragility_scenarios", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + + +@dataclass(frozen=True) +class BacktestStage: + """Stage 6 evidence for one or more source-listed backtest types.""" + + method_labels: tuple[str, ...] + trial_family_id: str + declared_trial_ids: tuple[str, ...] + validation: Optional[ValidationEvidence] = None + monte_carlo_dgp: Optional[str] = None + + def __post_init__(self): + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + + +@dataclass(frozen=True) +class MultipleTestingAdjustmentsStage: + """Stage 7 evidence for the complete declared family of trials.""" + + method_labels: tuple[str, ...] + trial_family_id: str + declared_trial_ids: tuple[str, ...] + family_partitions: tuple[tuple[str, tuple[str, ...]], ...] + alpha: Real + backtests_independent: bool + p_value_estimator: Optional[str] = None + time_dependence_model: Optional[str] = None + p_value_inputs_assume_independence: Optional[bool] = None + sharpe_variance: Optional[Real] = None + effective_trials: Optional[Real] = None + sample_length: Optional[Integral] = None + skewness: Optional[Real] = None + kurtosis: Optional[Real] = None + selection_bias_evidence_id: Optional[str] = None + + def __post_init__(self): + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + _store_normalized_field( + self, "family_partitions", _freeze_partitions(self.family_partitions) + ) + + +@dataclass(frozen=True) +class CausalFactorProtocolReport: + """Complete immutable record of the seven published protocol stages.""" + + trial_family_id: str + declared_trial_ids: tuple[str, ...] + variable_selection: VariableSelectionStage + causal_discovery: CausalDiscoveryStage + causal_adjustment_set: CausalAdjustmentSetStage + causal_explanatory_and_predictive_power: CausalExplanatoryAndPredictivePowerStage + causal_portfolio_construction: CausalPortfolioConstructionStage + backtest: BacktestStage + multiple_testing_adjustments: MultipleTestingAdjustmentsStage + source_oracle: str = field(default="PROTOCOL-01", init=False) + source_locator: str = field( + default="PDF pages 18-22 (journal pages 27-31), Exhibit 10", + init=False, + ) + + def __post_init__(self): + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + + @property + def stages(self): + """Return the seven named stage records in the source order.""" + + return ( + (_STAGE_NAMES[0], self.variable_selection), + (_STAGE_NAMES[1], self.causal_discovery), + (_STAGE_NAMES[2], self.causal_adjustment_set), + (_STAGE_NAMES[3], self.causal_explanatory_and_predictive_power), + (_STAGE_NAMES[4], self.causal_portfolio_construction), + (_STAGE_NAMES[5], self.backtest), + (_STAGE_NAMES[6], self.multiple_testing_adjustments), + ) + + @property + def stage_names(self): + """Return the exact seven source stage names.""" + + return _STAGE_NAMES + + +def _add_error(errors: list[str], message: str) -> None: + if message not in errors: + errors.append(message) + + +def _nonblank(value) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _safe_label_set(value) -> set[str]: + if not isinstance(value, tuple) or not all(isinstance(item, str) for item in value): + return set() + return set(value) + + +def _finite_real(value) -> bool: + if isinstance(value, bool) or not isinstance(value, Real): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError, OverflowError): + return False + + +def _validate_string_tuple( + value, + stage: str, + subject: str, + errors: list[str], + *, + allow_empty: bool = False, +) -> bool: + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: {subject} must be a tuple.") + return False + if not allow_empty and not value: + _add_error(errors, f"{stage}: {subject} must not be empty.") + return False + if not all(_nonblank(item) for item in value): + _add_error(errors, f"{stage}: {subject} contains an invalid identifier.") + return False + if len(set(value)) != len(value): + _add_error(errors, f"{stage}: {subject} contains duplicate identifiers.") + return False + return True + + +def _validate_labels( + value, + allowed: set[str], + stage: str, + subject: str, + errors: list[str], + *, + allow_empty: bool = False, +) -> bool: + valid = _validate_string_tuple( + value, stage, subject, errors, allow_empty=allow_empty + ) + if valid and not set(value).issubset(allowed): + _add_error(errors, f"{stage}: {subject} contains a non-source label.") + return False + return valid + + +def _validate_optional_identifier( + value, stage: str, subject: str, errors: list[str] +) -> bool: + if value is None: + return True + if not _nonblank(value): + _add_error(errors, f"{stage}: {subject} is invalid.") + return False + return True + + +def _validate_event_horizons( + evidence: ValidationEvidence, stage: str, errors: list[str] +) -> dict[str, tuple[float, float]]: + horizons = evidence.event_horizons + if not isinstance(horizons, tuple) or not horizons: + _add_error(errors, f"{stage}: event horizons must be a nonempty tuple.") + return {} + + result: dict[str, tuple[float, float]] = {} + for horizon in horizons: + if not isinstance(horizon, EventHorizon): + _add_error(errors, f"{stage}: an event-horizon record has an invalid type.") + continue + if not _nonblank(horizon.observation_id): + _add_error(errors, f"{stage}: an event horizon has an invalid identifier.") + continue + if horizon.observation_id in result: + _add_error(errors, f"{stage}: event-horizon identifiers must be unique.") + continue + if not _finite_real(horizon.start) or not _finite_real(horizon.end): + _add_error(errors, f"{stage}: event-horizon bounds must be finite numbers.") + continue + start = float(horizon.start) + end = float(horizon.end) + if not start < end: + _add_error( + errors, f"{stage}: every event horizon must have positive length." + ) + continue + result[horizon.observation_id] = (start, end) + return result + + +def _intervals_overlap(left: tuple[float, float], right: tuple[float, float]) -> bool: + return left[0] < right[1] and right[0] < left[1] + + +def _has_any_overlap(horizons: dict[str, tuple[float, float]]) -> bool: + intervals = list(horizons.values()) + for left_index, left in enumerate(intervals): + for right in intervals[left_index + 1 :]: + if _intervals_overlap(left, right): + return True + return False + + +def _validate_fold_evidence( + evidence: ValidationEvidence, + stage: str, + errors: list[str], + *, + required_refit_components: set[str], +) -> dict[str, tuple[float, float]]: + _validate_labels( + evidence.method_labels, + _VALIDATION_METHODS, + stage, + "validation method labels", + errors, + ) + horizon_map = _validate_event_horizons(evidence, stage, errors) + if not _finite_real(evidence.embargo) or float(evidence.embargo) < 0.0: + _add_error(errors, f"{stage}: embargo must be a finite nonnegative number.") + embargo = 0.0 + else: + embargo = float(evidence.embargo) + + folds = evidence.folds + if not isinstance(folds, tuple) or not folds: + _add_error(errors, f"{stage}: validation folds must be a nonempty tuple.") + return horizon_map + + fold_ids: set[str] = set() + refit_training_sets: dict[str, frozenset] = {} + path_predictions: set[tuple[str, str]] = set() + path_test_ids: dict[str, set[str]] = {} + path_partitions: dict[str, list[tuple[str, ...]]] = {} + nonresampled_test_ids: set[str] = set() + known_ids = set(horizon_map) + validation_methods = _safe_label_set(evidence.method_labels) + + for fold in folds: + if not isinstance(fold, FoldEvidence): + _add_error(errors, f"{stage}: a fold record has an invalid type.") + continue + if not _nonblank(fold.fold_id) or not _nonblank(fold.path_id): + _add_error(errors, f"{stage}: fold and path identifiers must be nonblank.") + elif fold.fold_id in fold_ids: + _add_error(errors, f"{stage}: fold identifiers must be unique.") + else: + fold_ids.add(fold.fold_id) + if not _nonblank(fold.refit_id): + _add_error(errors, f"{stage}: every fold must record a refit identifier.") + components_valid = _validate_labels( + fold.refit_components, + _REFIT_COMPONENTS, + stage, + "refit components", + errors, + ) + if components_valid and not required_refit_components.issubset( + set(fold.refit_components) + ): + _add_error( + errors, + f"{stage}: every fold must refit all required pipeline components.", + ) + + train_valid = _validate_string_tuple( + fold.train_ids, stage, "training identifiers", errors + ) + test_valid = _validate_string_tuple( + fold.test_ids, stage, "test identifiers", errors + ) + fit_valid = _validate_string_tuple( + fold.fit_ids, stage, "fit identifiers", errors + ) + prediction_valid = _validate_string_tuple( + fold.prediction_ids, stage, "prediction identifiers", errors + ) + if not (train_valid and test_valid and fit_valid and prediction_valid): + continue + + train_set = set(fold.train_ids) + test_set = set(fold.test_ids) + fit_set = set(fold.fit_ids) + prediction_set = set(fold.prediction_ids) + if train_set.intersection(test_set): + _add_error( + errors, f"{stage}: training and test membership must be disjoint." + ) + if fit_set != train_set or fit_set.intersection(test_set): + _add_error( + errors, + f"{stage}: every fold must refit on its complete training membership only.", + ) + if prediction_set != test_set: + _add_error(errors, f"{stage}: predictions must match the test membership.") + if not (train_set | test_set | fit_set | prediction_set).issubset(known_ids): + _add_error( + errors, f"{stage}: every fold identifier must have an event horizon." + ) + if validation_methods.intersection({"PURGED_CROSS_VALIDATION", "WALK_FORWARD"}): + if nonresampled_test_ids.intersection(test_set): + _add_error( + errors, + f"{stage}: ordinary temporal validation cannot duplicate test predictions.", + ) + nonresampled_test_ids.update(test_set.intersection(known_ids)) + + if _nonblank(fold.refit_id): + training_signature = frozenset(fold.train_ids) + previous = refit_training_sets.get(fold.refit_id) + if previous is not None and previous != training_signature: + _add_error( + errors, + f"{stage}: a refit identifier cannot represent different training sets.", + ) + refit_training_sets[fold.refit_id] = training_signature + + if _nonblank(fold.path_id): + path_test_ids.setdefault(fold.path_id, set()) + path_partitions.setdefault(fold.path_id, []) + path_partitions[fold.path_id].append(tuple(sorted(test_set))) + for observation_id in fold.test_ids: + contribution = (fold.path_id, observation_id) + if contribution in path_predictions: + _add_error( + errors, + f"{stage}: a path cannot contain duplicate test contributions.", + ) + path_predictions.add(contribution) + path_test_ids[fold.path_id].add(observation_id) + + if not (train_set | test_set).issubset(known_ids): + continue + for train_id in fold.train_ids: + train_horizon = horizon_map[train_id] + for test_id in fold.test_ids: + test_horizon = horizon_map[test_id] + if _intervals_overlap(train_horizon, test_horizon): + _add_error( + errors, + f"{stage}: train and test event horizons must be purged.", + ) + if ( + embargo > 0.0 + and train_horizon[0] >= test_horizon[1] + and train_horizon[0] < test_horizon[1] + embargo + ): + _add_error( + errors, + f"{stage}: post-test embargo observations must be excluded.", + ) + if "WALK_FORWARD" in validation_methods and train_set and test_set: + latest_train_end = max(horizon_map[item][1] for item in train_set) + earliest_test_start = min(horizon_map[item][0] for item in test_set) + if latest_train_end > earliest_test_start: + _add_error( + errors, + f"{stage}: walk-forward training must precede the test interval.", + ) + + if "COMBINATORIAL_PURGED_CROSS_VALIDATION" in validation_methods: + if len(path_test_ids) < 2: + _add_error(errors, f"{stage}: CPCV requires multiple complete paths.") + if known_ids and any(ids != known_ids for ids in path_test_ids.values()): + _add_error( + errors, + f"{stage}: every CPCV path must predict each declared observation once.", + ) + partition_signatures = [ + tuple(sorted(partition)) for partition in path_partitions.values() + ] + if len(set(partition_signatures)) != len(partition_signatures): + _add_error( + errors, f"{stage}: CPCV paths must have distinct test partitions." + ) + if ( + "PURGED_CROSS_VALIDATION" in validation_methods + and known_ids + and nonresampled_test_ids != known_ids + ): + _add_error( + errors, + f"{stage}: purged cross-validation must predict every observation once.", + ) + return horizon_map + + +def _valid_edge_tuple(value, stage: str, subject: str, errors: list[str]) -> bool: + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: {subject} must be a tuple.") + return False + valid = True + seen: set[tuple[str, str]] = set() + for edge in value: + if ( + not isinstance(edge, tuple) + or len(edge) != 2 + or not _nonblank(edge[0]) + or not _nonblank(edge[1]) + ): + _add_error(errors, f"{stage}: {subject} contains an invalid edge.") + valid = False + continue + if edge in seen: + _add_error(errors, f"{stage}: {subject} contains duplicate edges.") + valid = False + seen.add(edge) + return valid + + +def _adjacency(nodes: set[str], edges: tuple[tuple[str, str], ...]): + result = {node: set() for node in nodes} + for source, target in edges: + if source in result and target in result: + result[source].add(target) + return result + + +def _is_acyclic(nodes: set[str], edges: tuple[tuple[str, str], ...]) -> bool: + indegree = {node: 0 for node in nodes} + adjacency = _adjacency(nodes, edges) + for targets in adjacency.values(): + for target in targets: + indegree[target] += 1 + ready = [node for node, degree in indegree.items() if degree == 0] + visited = 0 + while ready: + node = ready.pop() + visited += 1 + for target in adjacency[node]: + indegree[target] -= 1 + if indegree[target] == 0: + ready.append(target) + return visited == len(nodes) + + +def _reachable(adjacency, start: str) -> set[str]: + visited: set[str] = set() + pending = list(adjacency.get(start, ())) + while pending: + node = pending.pop() + if node in visited: + continue + visited.add(node) + pending.extend(adjacency.get(node, ())) + return visited + + +def _reachable_avoiding(adjacency, start: str, blocked: set[str]) -> set[str]: + if start in blocked: + return set() + visited: set[str] = set() + pending = [node for node in adjacency.get(start, ()) if node not in blocked] + while pending: + node = pending.pop() + if node in visited or node in blocked: + continue + visited.add(node) + pending.extend( + target for target in adjacency.get(node, ()) if target not in blocked + ) + return visited + + +def _d_separated( + edges: tuple[tuple[str, str], ...], + left: str, + right: str, + conditioned: set[str], +) -> bool: + nodes = {left, right} | conditioned + for source, target in edges: + nodes.add(source) + nodes.add(target) + parents = {node: set() for node in nodes} + for source, target in edges: + parents[target].add(source) + + ancestors = {left, right} | conditioned + pending = list(ancestors) + while pending: + node = pending.pop() + for parent in parents.get(node, ()): + if parent not in ancestors: + ancestors.add(parent) + pending.append(parent) + + moral = {node: set() for node in ancestors} + for source, target in edges: + if source in ancestors and target in ancestors: + moral[source].add(target) + moral[target].add(source) + for child in ancestors: + child_parents = [parent for parent in parents[child] if parent in ancestors] + for index, first in enumerate(child_parents): + for second in child_parents[index + 1 :]: + moral[first].add(second) + moral[second].add(first) + + if left in conditioned or right in conditioned: + return True + visited = set(conditioned) + pending = [left] + while pending: + node = pending.pop() + if node == right: + return False + if node in visited: + continue + visited.add(node) + pending.extend(neighbor for neighbor in moral[node] if neighbor not in visited) + return True + + +def _backdoor_d_separated( + edges: tuple[tuple[str, str], ...], + exposure: str, + outcome: str, + conditioned: set[str], +) -> bool: + backdoor_edges = tuple(edge for edge in edges if edge[0] != exposure) + return _d_separated(backdoor_edges, exposure, outcome, conditioned) + + +def _validate_discovery( + stage: CausalDiscoveryStage, + selected_variables: set[str], + outcome: str, + errors: list[str], +): + label = "Stage 2" + _validate_labels( + stage.method_labels, + _DISCOVERY_METHODS, + label, + "causal-discovery method labels", + errors, + ) + if not _nonblank(stage.graph_id): + _add_error(errors, f"{label}: graph identifier must be nonblank.") + nodes_valid = _validate_string_tuple( + stage.graph_nodes, label, "graph nodes", errors + ) + nodes = set(stage.graph_nodes) if nodes_valid else set() + if nodes_valid and nodes != selected_variables | {outcome}: + _add_error( + errors, + f"{label}: graph nodes must match the selected variables and outcome.", + ) + edges_valid = _valid_edge_tuple( + stage.directed_edges, label, "directed edges", errors + ) + if edges_valid: + for source, target in stage.directed_edges: + if source not in nodes or target not in nodes: + _add_error( + errors, f"{label}: every directed edge must use graph nodes." + ) + if source == target: + _add_error(errors, f"{label}: directed self-loops are not admissible.") + if nodes and not _is_acyclic(nodes, stage.directed_edges): + _add_error(errors, f"{label}: the resolved graph must be acyclic.") + if stage.graph_kind != "DAG": + _add_error(errors, f"{label}: the accepted graph must be a resolved DAG.") + if not isinstance(stage.ambiguous_edges, tuple): + _add_error(errors, f"{label}: ambiguous edges must be a tuple.") + elif stage.ambiguous_edges: + _valid_edge_tuple(stage.ambiguous_edges, label, "ambiguous edges", errors) + _add_error( + errors, f"{label}: identification-relevant ambiguity must be resolved." + ) + _validate_string_tuple(stage.assumptions, label, "graph assumptions", errors) + safe_edges = stage.directed_edges if edges_valid else () + return nodes, _adjacency(nodes, safe_edges), safe_edges + + +def _validate_adjustment_sets( + value, stage: str, errors: list[str] +) -> tuple[set[frozenset], bool]: + if not isinstance(value, tuple) or not value: + _add_error(errors, f"{stage}: admissible adjustment sets must be nonempty.") + return set(), False + result: set[frozenset] = set() + valid = True + for adjustment_set in value: + if not _validate_string_tuple( + adjustment_set, + stage, + "an admissible adjustment set", + errors, + allow_empty=True, + ): + valid = False + continue + frozen = frozenset(adjustment_set) + if frozen in result: + _add_error(errors, f"{stage}: admissible adjustment sets must be distinct.") + valid = False + result.add(frozen) + return result, valid + + +def _validate_justifications(value, stage: str, errors: list[str]): + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: control justifications must be a tuple.") + return {} + result = {} + for item in value: + if ( + not isinstance(item, tuple) + or len(item) != 2 + or not _nonblank(item[0]) + or not _nonblank(item[1]) + ): + _add_error(errors, f"{stage}: a control justification is invalid.") + continue + if item[0] in result: + _add_error(errors, f"{stage}: every control must have one justification.") + result[item[0]] = item[1] + return result + + +def _validate_adjustment( + stage: CausalAdjustmentSetStage, + graph_nodes: set[str], + adjacency, + directed_edges: tuple[tuple[str, str], ...], + errors: list[str], +) -> None: + label = "Stage 3" + treatment = stage.treatment if _nonblank(stage.treatment) else "" + outcome = stage.outcome if _nonblank(stage.outcome) else "" + if not _nonblank(stage.treatment) or not _nonblank(stage.outcome): + _add_error(errors, f"{label}: treatment and outcome must be nonblank.") + elif treatment == outcome: + _add_error(errors, f"{label}: treatment and outcome must be distinct.") + if treatment not in graph_nodes or outcome not in graph_nodes: + _add_error(errors, f"{label}: treatment and outcome must be graph nodes.") + if ( + not _nonblank(stage.method_label) + or stage.method_label not in _ADJUSTMENT_METHODS + ): + _add_error(errors, f"{label}: adjustment method is not source-listed.") + if not isinstance(stage.identified, bool) or not stage.identified: + _add_error(errors, f"{label}: the causal effect must be identified.") + + selected_valid = _validate_string_tuple( + stage.selected_adjustment_set, + label, + "selected adjustment set", + errors, + allow_empty=True, + ) + admissible_sets, _ = _validate_adjustment_sets( + stage.admissible_adjustment_sets, label, errors + ) + selected = set(stage.selected_adjustment_set) if selected_valid else set() + if selected_valid and frozenset(selected) not in admissible_sets: + _add_error( + errors, + f"{label}: the selected adjustment set must be declared admissible.", + ) + + role_names = {} + for field_name, subject in ( + ("confounders", "confounders"), + ("descendants", "descendants"), + ("mediators", "mediators"), + ("colliders", "colliders"), + ("instruments", "instruments"), + ): + value = getattr(stage, field_name) + valid = _validate_string_tuple(value, label, subject, errors, allow_empty=True) + role_names[field_name] = set(value) if valid else set() + if valid and not set(value).issubset(graph_nodes): + _add_error( + errors, f"{label}: every declared causal role must be a graph node." + ) + if valid and {treatment, outcome}.intersection(value): + _add_error( + errors, f"{label}: treatment and outcome cannot be role variables." + ) + + computed_descendants = _reachable(adjacency, treatment) + computed_descendants.discard(outcome) + if role_names["descendants"] != computed_descendants: + _add_error( + errors, + f"{label}: declared descendants must exactly match the graph.", + ) + for mediator in role_names["mediators"]: + if mediator not in computed_descendants or outcome not in _reachable( + adjacency, mediator + ): + _add_error( + errors, f"{label}: declared mediators must lie on a causal path." + ) + incoming = {node: 0 for node in graph_nodes} + for targets in adjacency.values(): + for target in targets: + incoming[target] += 1 + if any(incoming.get(collider, 0) < 2 for collider in role_names["colliders"]): + _add_error(errors, f"{label}: declared colliders must have converging arrows.") + if outcome not in _reachable(adjacency, treatment): + _add_error( + errors, f"{label}: the graph must retain a treatment-to-outcome path." + ) + + graph_confounders = { + node + for node in graph_nodes - {treatment, outcome} + if treatment in _reachable(adjacency, node) + and outcome in _reachable_avoiding(adjacency, node, {treatment}) + } + if role_names["confounders"] != graph_confounders: + _add_error( + errors, + f"{label}: declared confounders must match graph-implied common causes.", + ) + graph_colliders = { + node + for node, degree in incoming.items() + if degree >= 2 and node not in {treatment, outcome} + } + if role_names["colliders"] != graph_colliders: + _add_error( + errors, + f"{label}: declared colliders must match graph-implied converging nodes.", + ) + graph_mediators = { + node + for node in graph_nodes - {treatment, outcome} + if node in _reachable(adjacency, treatment) + and outcome in _reachable(adjacency, node) + } + if role_names["mediators"] != graph_mediators: + _add_error( + errors, + f"{label}: declared mediators must match graph-implied causal-path nodes.", + ) + if role_names["confounders"].intersection(role_names["instruments"]): + _add_error( + errors, + f"{label}: a variable cannot be both a confounder and an instrument.", + ) + instrument_edges = tuple(edge for edge in directed_edges if edge[0] != treatment) + for instrument in role_names["instruments"]: + if treatment not in _reachable_avoiding(adjacency, instrument, selected): + _add_error( + errors, + f"{label}: every declared instrument must be relevant in the graph.", + ) + if outcome in _reachable_avoiding(adjacency, instrument, {treatment}): + _add_error( + errors, + f"{label}: every declared instrument must satisfy graph-level exclusion.", + ) + if not _d_separated(instrument_edges, instrument, outcome, selected): + _add_error( + errors, + f"{label}: every declared instrument must be graph-separated from outcome shocks.", + ) + + forbidden_controls = ( + computed_descendants + | role_names["descendants"] + | role_names["mediators"] + | role_names["colliders"] + | role_names["instruments"] + | {treatment, outcome} + ) + if selected.intersection(forbidden_controls): + _add_error( + errors, + f"{label}: ordinary controls cannot include descendants, mediators, colliders, or instruments.", + ) + for admissible in admissible_sets: + candidate = set(admissible) + if not candidate.issubset(graph_nodes): + _add_error( + errors, + f"{label}: every declared admissible set must use graph nodes.", + ) + if candidate.intersection(forbidden_controls): + _add_error( + errors, + f"{label}: no declared admissible set may contain a descendant, mediator, collider, or instrument.", + ) + if stage.method_label == "BACKDOOR_ADJUSTMENT" and not _backdoor_d_separated( + directed_edges, + treatment, + outcome, + candidate, + ): + _add_error( + errors, + f"{label}: every declared admissible set must block graph-implied backdoor paths.", + ) + if not selected.issubset(graph_nodes): + _add_error(errors, f"{label}: every selected control must be a graph node.") + justifications = _validate_justifications( + stage.control_justifications, label, errors + ) + if set(justifications) != selected: + _add_error(errors, f"{label}: every selected control needs one justification.") + if not _validate_string_tuple( + stage.open_backdoor_paths, + label, + "open backdoor paths", + errors, + allow_empty=True, + ): + pass + elif stage.open_backdoor_paths: + _add_error(errors, f"{label}: no backdoor path may remain open.") + + if ( + stage.method_label != "FRONT_DOOR_ADJUSTMENT" + and stage.frontdoor_criteria_satisfied is not None + ): + _add_error( + errors, + f"{label}: front-door evidence requires the front-door method.", + ) + instrument_premises = ( + stage.instrument_relevance_satisfied, + stage.instrument_exclusion_satisfied, + stage.instrument_exogeneity_satisfied, + ) + if stage.method_label != "INSTRUMENTAL_VARIABLES" and any( + value is not None for value in instrument_premises + ): + _add_error( + errors, + f"{label}: instrumental-variable premises require the IV method.", + ) + + if stage.method_label == "BACKDOOR_ADJUSTMENT": + if not _backdoor_d_separated( + directed_edges, + treatment, + outcome, + selected, + ): + _add_error( + errors, + f"{label}: selected controls must block every graph-implied backdoor path.", + ) + elif stage.method_label == "FRONT_DOOR_ADJUSTMENT": + if ( + not role_names["mediators"] + or stage.frontdoor_criteria_satisfied is not True + ): + _add_error( + errors, f"{label}: front-door criteria must be explicitly satisfied." + ) + elif outcome in _reachable_avoiding( + adjacency, treatment, role_names["mediators"] + ): + _add_error( + errors, + f"{label}: front-door mediators must intercept every directed causal path.", + ) + for mediator in role_names["mediators"]: + if not _backdoor_d_separated( + directed_edges, treatment, mediator, selected + ) or not _backdoor_d_separated( + directed_edges, mediator, outcome, {treatment} | selected + ): + _add_error( + errors, + f"{label}: graph-implied front-door backdoor criteria must hold.", + ) + elif stage.method_label == "INSTRUMENTAL_VARIABLES": + if not role_names["instruments"] or any( + value is not True for value in instrument_premises + ): + _add_error( + errors, f"{label}: instrumental-variable premises must all hold." + ) + + +def _same_horizons( + left: dict[str, tuple[float, float]], right: dict[str, tuple[float, float]] +) -> bool: + return left == right + + +def validate_causal_factor_protocol( + report: CausalFactorProtocolReport, +) -> CausalFactorProtocolReport: + """Validate and return a complete seven-stage protocol evidence record. + + The validation is structural and fail-closed. A successful result means + the record conforms to the published stage contract; it does not certify + that the caller's empirical evidence or causal assumptions are true. + + Raises + ------ + ValueError + If any stage is missing, has the wrong record type, uses a method label + outside the source contract, or fails a cross-stage safety invariant. + """ + + if not isinstance(report, CausalFactorProtocolReport): + raise ValueError("Protocol: report type is invalid.") + + errors: list[str] = [] + if not _nonblank(report.trial_family_id): + _add_error(errors, "Protocol: trial-family identifier must be nonblank.") + trials_valid = _validate_string_tuple( + report.declared_trial_ids, + "Protocol", + "declared trial identifiers", + errors, + ) + + stage_types = ( + ("Stage 1", report.variable_selection, VariableSelectionStage), + ("Stage 2", report.causal_discovery, CausalDiscoveryStage), + ("Stage 3", report.causal_adjustment_set, CausalAdjustmentSetStage), + ( + "Stage 4", + report.causal_explanatory_and_predictive_power, + CausalExplanatoryAndPredictivePowerStage, + ), + ( + "Stage 5", + report.causal_portfolio_construction, + CausalPortfolioConstructionStage, + ), + ("Stage 6", report.backtest, BacktestStage), + ( + "Stage 7", + report.multiple_testing_adjustments, + MultipleTestingAdjustmentsStage, + ), + ) + for label, value, expected in stage_types: + if not isinstance(value, expected): + _add_error(errors, f"{label}: stage record type is invalid.") + if any(not isinstance(value, expected) for _, value, expected in stage_types): + raise ValueError("Causal factor protocol is invalid. " + " ".join(errors)) + + selection = report.variable_selection + label = "Stage 1" + if not _nonblank(selection.purpose) or selection.purpose not in _RESEARCH_PURPOSES: + _add_error(errors, f"{label}: research purpose is not source-listed.") + selected_valid = _validate_string_tuple( + selection.selected_variables, label, "selected variables", errors + ) + selected_variables = set(selection.selected_variables) if selected_valid else set() + selection_methods_valid = _validate_labels( + selection.method_labels, + _VARIABLE_SELECTION_METHODS, + label, + "variable-selection method labels", + errors, + ) + if not isinstance(selection.overlapping_returns, bool): + _add_error(errors, f"{label}: overlapping-returns flag must be boolean.") + if not isinstance(selection.strong_time_dependence, bool): + _add_error(errors, f"{label}: time-dependence flag must be boolean.") + selection_methods = ( + set(selection.method_labels) if selection_methods_valid else set() + ) + if selection.validation is None: + selection_horizons = {} + if ( + selection.overlapping_returns is True + or selection.strong_time_dependence is True + or "PERMUTATION_FEATURE_IMPORTANCE" in selection_methods + ): + _add_error( + errors, + f"{label}: the declared selection design requires temporal validation.", + ) + elif not isinstance(selection.validation, ValidationEvidence): + _add_error( + errors, f"{label}: temporal validation evidence has an invalid type." + ) + selection_horizons = {} + else: + selection_horizons = _validate_fold_evidence( + selection.validation, + label, + errors, + required_refit_components={"VARIABLE_SELECTION"}, + ) + actual_overlap = _has_any_overlap(selection_horizons) + if isinstance(selection.overlapping_returns, bool) and ( + selection.overlapping_returns != actual_overlap + ): + _add_error( + errors, f"{label}: overlap declaration must match event horizons." + ) + validation_methods = _safe_label_set(selection.validation.method_labels) + if actual_overlap and not validation_methods.intersection( + _PURGED_VALIDATION_METHODS + ): + _add_error( + errors, f"{label}: overlapping returns require purged validation." + ) + if selection.strong_time_dependence and ( + not _finite_real(selection.validation.embargo) + or float(selection.validation.embargo) <= 0.0 + ): + _add_error(errors, f"{label}: strong time dependence requires an embargo.") + + adjustment = report.causal_adjustment_set + safe_outcome = adjustment.outcome if _nonblank(adjustment.outcome) else "" + graph_nodes, graph_adjacency, graph_edges = _validate_discovery( + report.causal_discovery, + selected_variables, + safe_outcome, + errors, + ) + _validate_adjustment( + adjustment, + graph_nodes, + graph_adjacency, + graph_edges, + errors, + ) + + stage4 = report.causal_explanatory_and_predictive_power + label = "Stage 4" + task_types_valid = _validate_labels( + stage4.task_types, + set(_METRICS_BY_TASK), + label, + "task types", + errors, + ) + allowed_metrics = set() + if task_types_valid: + for task_type in stage4.task_types: + allowed_metrics.update(_METRICS_BY_TASK[task_type]) + if not _nonblank(stage4.estimator_label): + _add_error(errors, f"{label}: estimator label must be nonblank.") + explanatory_valid = _validate_labels( + stage4.explanatory_metric_labels, + allowed_metrics, + label, + "explanatory metric labels", + errors, + allow_empty=True, + ) + predictive_valid = _validate_labels( + stage4.predictive_metric_labels, + allowed_metrics, + label, + "predictive metric labels", + errors, + allow_empty=True, + ) + has_explanatory = explanatory_valid and bool(stage4.explanatory_metric_labels) + has_predictive = predictive_valid and bool(stage4.predictive_metric_labels) + if not has_explanatory and not has_predictive: + _add_error(errors, f"{label}: at least one performance aspect is required.") + reported_metrics = set() + if explanatory_valid: + reported_metrics.update(stage4.explanatory_metric_labels) + if predictive_valid: + reported_metrics.update(stage4.predictive_metric_labels) + if task_types_valid: + for task_type in stage4.task_types: + if not reported_metrics.intersection(_METRICS_BY_TASK[task_type]): + _add_error( + errors, + f"{label}: every declared task needs a compatible metric.", + ) + if has_explanatory and not _nonblank(stage4.explanatory_evidence_id): + _add_error(errors, f"{label}: explanatory evidence identifier is required.") + if not has_explanatory and stage4.explanatory_evidence_id is not None: + _add_error( + errors, f"{label}: explanatory evidence must match reported metrics." + ) + if has_predictive and not _nonblank(stage4.predictive_evidence_id): + _add_error(errors, f"{label}: predictive evidence identifier is required.") + if not has_predictive and stage4.predictive_evidence_id is not None: + _add_error(errors, f"{label}: predictive evidence must match reported metrics.") + _validate_optional_identifier( + stage4.explanatory_evidence_id, + label, + "explanatory evidence identifier", + errors, + ) + _validate_optional_identifier( + stage4.predictive_evidence_id, + label, + "predictive evidence identifier", + errors, + ) + if ( + has_explanatory + and has_predictive + and stage4.explanatory_evidence_id == stage4.predictive_evidence_id + ): + _add_error( + errors, f"{label}: explanatory and predictive evidence must be separate." + ) + if selection.purpose == "CAUSAL_ATTRIBUTION" and not has_explanatory: + _add_error( + errors, f"{label}: causal attribution requires explanatory evidence." + ) + if selection.purpose == "RISK_PREMIA_HARVESTING" and not has_predictive: + _add_error( + errors, f"{label}: risk-premia harvesting requires predictive evidence." + ) + if not _nonblank(stage4.naive_benchmark_id): + _add_error(errors, f"{label}: a naive benchmark identifier is required.") + multiclass_fields = (stage4.multiclass_encoding, stage4.averaging_method) + if any(value is not None for value in multiclass_fields): + if not isinstance(stage4.multiclass_encoding, str) or ( + stage4.multiclass_encoding != "ONE_VS_REST" + ): + _add_error(errors, f"{label}: multiclass encoding is not source-listed.") + if not isinstance(stage4.averaging_method, str) or ( + stage4.averaging_method not in {"MICRO", "MACRO", "WEIGHTED", "SAMPLES"} + ): + _add_error(errors, f"{label}: multiclass averaging is not source-listed.") + if not _safe_label_set(stage4.task_types).intersection( + {"PROBABILITY", "RANKING"} + ): + _add_error( + errors, f"{label}: multiclass settings require a classification task." + ) + if not isinstance(stage4.validation, ValidationEvidence): + _add_error(errors, f"{label}: purged validation evidence is required.") + else: + stage4_horizons = _validate_fold_evidence( + stage4.validation, + label, + errors, + required_refit_components={"VARIABLE_SELECTION", "CAUSAL_ESTIMATION"}, + ) + if not _safe_label_set(stage4.validation.method_labels).intersection( + _PURGED_VALIDATION_METHODS + ): + _add_error(errors, f"{label}: purged cross-validation is required.") + if selection_horizons and not _same_horizons( + selection_horizons, stage4_horizons + ): + _add_error(errors, f"{label}: event horizons must match Stage 1.") + if selection.strong_time_dependence and ( + not _finite_real(stage4.validation.embargo) + or float(stage4.validation.embargo) <= 0.0 + ): + _add_error(errors, f"{label}: strong time dependence requires an embargo.") + + portfolio = report.causal_portfolio_construction + label = "Stage 5" + portfolio_labels_valid = _validate_labels( + portfolio.method_labels, + _PORTFOLIO_METHODS, + label, + "portfolio-construction method labels", + errors, + ) + if portfolio_labels_valid and set(portfolio.method_labels) != _PORTFOLIO_METHODS: + _add_error( + errors, f"{label}: all source portfolio considerations are required." + ) + causal_valid = _validate_string_tuple( + portfolio.causal_exposures, label, "causal exposures", errors + ) + neutral_valid = _validate_string_tuple( + portfolio.controlled_unintended_exposures, + label, + "controlled unintended exposures", + errors, + allow_empty=True, + ) + causal_exposures = set(portfolio.causal_exposures) if causal_valid else set() + neutral_exposures = ( + set(portfolio.controlled_unintended_exposures) if neutral_valid else set() + ) + if causal_valid and not causal_exposures.issubset(graph_nodes): + _add_error(errors, f"{label}: causal exposures must be graph variables.") + if causal_valid and any( + safe_outcome not in _reachable(graph_adjacency, exposure) + for exposure in causal_exposures + ): + _add_error( + errors, + f"{label}: every causal exposure must have a directed path to the outcome.", + ) + if neutral_valid and not neutral_exposures.issubset(graph_nodes): + _add_error( + errors, + f"{label}: controlled unintended exposures must be graph variables.", + ) + safe_treatment = adjustment.treatment if _nonblank(adjustment.treatment) else "" + if safe_treatment not in causal_exposures: + _add_error( + errors, f"{label}: the target causal factor must drive position sizing." + ) + if causal_exposures.intersection(neutral_exposures): + _add_error( + errors, f"{label}: causal and neutralized exposures must be disjoint." + ) + declared_colliders = _safe_label_set(adjustment.colliders) + if not declared_colliders.issubset(neutral_exposures): + _add_error(errors, f"{label}: declared colliders must be neutralized.") + for value, subject in ( + (portfolio.cost_model_id, "cost model identifier"), + (portfolio.constraint_set_id, "constraint-set identifier"), + (portfolio.economic_rationale, "economic rationale"), + ): + if not _nonblank(value): + _add_error(errors, f"{label}: {subject} must be nonblank.") + _validate_string_tuple( + portfolio.fragility_scenarios, + label, + "causal-fragility scenarios", + errors, + ) + if not _finite_real(portfolio.transfer_coefficient) or not ( + -1.0 <= float(portfolio.transfer_coefficient) <= 1.0 + ): + _add_error( + errors, + f"{label}: transfer coefficient must be finite and between minus one and one.", + ) + + backtest = report.backtest + label = "Stage 6" + backtest_labels_valid = _validate_labels( + backtest.method_labels, + _BACKTEST_METHODS, + label, + "backtest method labels", + errors, + ) + if backtest.trial_family_id != report.trial_family_id: + _add_error(errors, f"{label}: trial-family identifier must match the report.") + if backtest.declared_trial_ids != report.declared_trial_ids: + _add_error(errors, f"{label}: declared trials must match the report exactly.") + non_monte_carlo = ( + set(backtest.method_labels) - {"MONTE_CARLO"} + if backtest_labels_valid + else set() + ) + if non_monte_carlo: + if not isinstance(backtest.validation, ValidationEvidence): + _add_error( + errors, f"{label}: temporal backtests require validation evidence." + ) + else: + _validate_fold_evidence( + backtest.validation, + label, + errors, + required_refit_components=_REFIT_COMPONENTS, + ) + if selection.strong_time_dependence and ( + not _finite_real(backtest.validation.embargo) + or float(backtest.validation.embargo) <= 0.0 + ): + _add_error( + errors, + f"{label}: strong time dependence requires a backtest embargo.", + ) + if not non_monte_carlo.issubset( + _safe_label_set(backtest.validation.method_labels) + ): + _add_error( + errors, + f"{label}: backtest and validation method labels must agree.", + ) + elif backtest.validation is not None: + _add_error( + errors, + f"{label}: Monte Carlo-only evidence cannot carry temporal folds.", + ) + if backtest_labels_valid and "MONTE_CARLO" in backtest.method_labels: + if not _nonblank(backtest.monte_carlo_dgp): + _add_error(errors, f"{label}: Monte Carlo requires an explicit DGP.") + elif backtest.monte_carlo_dgp is not None: + _add_error( + errors, f"{label}: a Monte Carlo DGP requires the Monte Carlo method." + ) + + multiple_testing = report.multiple_testing_adjustments + label = "Stage 7" + methods_valid = _validate_labels( + multiple_testing.method_labels, + _MULTIPLE_TESTING_METHODS, + label, + "multiple-testing method labels", + errors, + ) + if multiple_testing.trial_family_id != report.trial_family_id: + _add_error(errors, f"{label}: trial-family identifier must match the report.") + if multiple_testing.declared_trial_ids != report.declared_trial_ids: + _add_error(errors, f"{label}: declared trials must match the report exactly.") + + partitions = multiple_testing.family_partitions + covered_trials: set[str] = set() + partition_ids: set[str] = set() + if not isinstance(partitions, tuple) or not partitions: + _add_error(errors, f"{label}: family partitions must be a nonempty tuple.") + else: + for partition in partitions: + if ( + not isinstance(partition, tuple) + or len(partition) != 2 + or not _nonblank(partition[0]) + ): + _add_error(errors, f"{label}: a family partition is invalid.") + continue + if partition[0] in partition_ids: + _add_error( + errors, f"{label}: family partition identifiers must be unique." + ) + partition_ids.add(partition[0]) + if not _validate_string_tuple( + partition[1], label, "partition trial identifiers", errors + ): + continue + for trial_id in partition[1]: + if trial_id in covered_trials: + _add_error(errors, f"{label}: family partitions must be disjoint.") + covered_trials.add(trial_id) + if trials_valid and covered_trials != set(report.declared_trial_ids): + _add_error( + errors, + f"{label}: family partitions must cover declared trials exactly.", + ) + + if not _finite_real(multiple_testing.alpha) or not ( + 0.0 < float(multiple_testing.alpha) < 1.0 + ): + _add_error( + errors, f"{label}: alpha must be a finite number between zero and one." + ) + if not isinstance(multiple_testing.backtests_independent, bool): + _add_error(errors, f"{label}: backtest-dependence flag must be boolean.") + elif multiple_testing.backtests_independent: + _add_error(errors, f"{label}: backtest dependence must be represented.") + + methods = set(multiple_testing.method_labels) if methods_valid else set() + if methods.intersection(_P_VALUE_METHODS): + if not _nonblank(multiple_testing.p_value_estimator): + _add_error(errors, f"{label}: p-value estimator must be declared.") + if not _nonblank(multiple_testing.time_dependence_model): + _add_error(errors, f"{label}: time dependence treatment must be declared.") + if multiple_testing.p_value_inputs_assume_independence is not False: + _add_error( + errors, + f"{label}: p-value inputs must explicitly reject independence assumptions.", + ) + elif any( + value is not None + for value in ( + multiple_testing.p_value_estimator, + multiple_testing.time_dependence_model, + multiple_testing.p_value_inputs_assume_independence, + ) + ): + _add_error( + errors, + f"{label}: p-value inputs require a p-value correction.", + ) + if "DEFLATED_SHARPE_RATIO" in methods: + trial_count = ( + len(report.declared_trial_ids) + if isinstance(report.declared_trial_ids, tuple) + else 0 + ) + if ( + not _finite_real(multiple_testing.sharpe_variance) + or float(multiple_testing.sharpe_variance) <= 0.0 + ): + _add_error(errors, f"{label}: positive Sharpe-ratio variance is required.") + if not _finite_real(multiple_testing.effective_trials) or not ( + 1.0 <= float(multiple_testing.effective_trials) < trial_count + ): + _add_error(errors, f"{label}: effective trials must be below total trials.") + if ( + isinstance(multiple_testing.sample_length, bool) + or not isinstance(multiple_testing.sample_length, Integral) + or int(multiple_testing.sample_length) <= 1 + ): + _add_error(errors, f"{label}: sample length must be an integer above one.") + if not _finite_real(multiple_testing.skewness): + _add_error(errors, f"{label}: skewness must be finite.") + if ( + not _finite_real(multiple_testing.kurtosis) + or float(multiple_testing.kurtosis) < 1.0 + ): + _add_error( + errors, f"{label}: Pearson kurtosis must be finite and at least one." + ) + elif _finite_real(multiple_testing.skewness) and abs( + float(multiple_testing.skewness) + ) > math.sqrt(float(multiple_testing.kurtosis) - 1.0): + _add_error( + errors, + f"{label}: skewness and Pearson kurtosis must satisfy the moment inequality.", + ) + if not _nonblank(multiple_testing.selection_bias_evidence_id): + _add_error(errors, f"{label}: selection-bias evidence is required.") + elif any( + value is not None + for value in ( + multiple_testing.sharpe_variance, + multiple_testing.effective_trials, + multiple_testing.sample_length, + multiple_testing.skewness, + multiple_testing.kurtosis, + multiple_testing.selection_bias_evidence_id, + ) + ): + _add_error( + errors, + f"{label}: Sharpe-correction inputs require the deflated Sharpe ratio.", + ) + + if errors: + raise ValueError("Causal factor protocol is invalid. " + " ".join(errors)) + return report diff --git a/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py new file mode 100644 index 0000000..8f929ae --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py @@ -0,0 +1,261 @@ +"""Specification diagnostics from the 2023 causal-factor experiments. + +This module independently implements the three linear-Gaussian structural +experiments in Sections 7.1-7.3 and their Appendix procedures. Population +functions expose exact analytical regression targets; experiment functions +generate seeded samples and fit both specifications with NumPy only. +""" + +from __future__ import annotations + +import math +from numbers import Integral +from typing import NamedTuple, Tuple + +import numpy as np + +__all__ = [ + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "fork_population_diagnostics", + "fork_specification_experiment", +] + + +class PopulationRegressionDiagnostics(NamedTuple): + """Exact population coefficients and explained-variance fraction.""" + + coefficient_names: Tuple[str, ...] + coefficients: Tuple[float, ...] + r_squared: float + + +class RegressionDiagnostics(NamedTuple): + """Classical least-squares diagnostics for one generated sample.""" + + coefficient_names: Tuple[str, ...] + coefficients: Tuple[float, ...] + standard_errors: Tuple[float, ...] + t_statistics: Tuple[float, ...] + r_squared: float + adjusted_r_squared: float + residual_variance: float + n_observations: int + + +class SpecificationPopulationResult(NamedTuple): + """Exact population targets for unconditioned and conditioned models.""" + + structure: str + conditioning_variable_role: str + unconditioned: PopulationRegressionDiagnostics + conditioned: PopulationRegressionDiagnostics + + +class SpecificationExperimentResult(NamedTuple): + """Sample diagnostics for unconditioned and conditioned regressions.""" + + structure: str + conditioning_variable_role: str + unconditioned: RegressionDiagnostics + conditioned: RegressionDiagnostics + + +_MAX_SAMPLE_SIZE = 1_000_000 + + +def _as_sample_size(value: Integral) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral): + raise ValueError("sample_size must be an integer") + result = int(value) + if result <= 3: + raise ValueError("sample_size must be greater than 3") + if result > _MAX_SAMPLE_SIZE: + raise ValueError(f"sample_size must not exceed {_MAX_SAMPLE_SIZE}") + return result + + +def _as_seed(value: Integral) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral): + raise ValueError("seed must be an integer") + result = int(value) + if result < 0 or result > 2**32 - 1: + raise ValueError("seed must be between 0 and 2**32 - 1") + return result + + +def _population_regression( + names: Tuple[str, ...], coefficients: Tuple[float, ...], r_squared: float +) -> PopulationRegressionDiagnostics: + return PopulationRegressionDiagnostics( + coefficient_names=names, + coefficients=coefficients, + r_squared=r_squared, + ) + + +def _ols_diagnostics( + outcome: np.ndarray, + regressors: Tuple[Tuple[str, np.ndarray], ...], +) -> RegressionDiagnostics: + n_observations = int(outcome.size) + design = np.column_stack( + (np.ones(n_observations, dtype=float), *(values for _, values in regressors)) + ) + n_parameters = int(design.shape[1]) + if n_observations <= n_parameters: + raise ValueError("the regression requires positive residual degrees of freedom") + if np.linalg.matrix_rank(design) != n_parameters: + raise ValueError("the generated regression design is rank deficient") + + coefficients, _, _, _ = np.linalg.lstsq(design, outcome, rcond=None) + fitted = design @ coefficients + residuals = outcome - fitted + residual_sum_squares = float(residuals @ residuals) + centered = outcome - float(np.mean(outcome)) + total_sum_squares = float(centered @ centered) + if not math.isfinite(total_sum_squares) or total_sum_squares <= 0.0: + raise ValueError("the generated outcome must have positive finite variance") + + residual_degrees_of_freedom = n_observations - n_parameters + residual_variance = residual_sum_squares / residual_degrees_of_freedom + gram_inverse = np.linalg.inv(design.T @ design) + coefficient_variances = np.diag(gram_inverse) * residual_variance + if np.any(coefficient_variances < -1.0e-12): + raise ValueError("the coefficient variance calculation is invalid") + standard_errors = np.sqrt(np.maximum(coefficient_variances, 0.0)) + t_statistics = np.divide( + coefficients, + standard_errors, + out=np.full(coefficients.shape, np.nan, dtype=float), + where=standard_errors > 0.0, + ) + if not np.all(np.isfinite(coefficients)) or not np.all( + np.isfinite(standard_errors) + ): + raise ValueError("the regression diagnostics must be finite") + + r_squared = 1.0 - residual_sum_squares / total_sum_squares + if r_squared < 0.0 and r_squared >= -1.0e-12: + r_squared = 0.0 + if r_squared > 1.0 and r_squared <= 1.0 + 1.0e-12: + r_squared = 1.0 + adjusted_r_squared = 1.0 - (1.0 - r_squared) * ( + (n_observations - 1) / residual_degrees_of_freedom + ) + names = ("intercept", *(name for name, _ in regressors)) + return RegressionDiagnostics( + coefficient_names=names, + coefficients=tuple(float(value) for value in coefficients), + standard_errors=tuple(float(value) for value in standard_errors), + t_statistics=tuple(float(value) for value in t_statistics), + r_squared=float(r_squared), + adjusted_r_squared=float(adjusted_r_squared), + residual_variance=float(residual_variance), + n_observations=n_observations, + ) + + +def fork_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 20-24 (the confounder fork).""" + + return SpecificationPopulationResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=_population_regression(("intercept", "X"), (0.0, 0.5), 0.25), + conditioned=_population_regression( + ("intercept", "X", "Z"), (0.0, 0.0, 1.0), 0.5 + ), + ) + + +def collider_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 27-33 (the collider).""" + + return SpecificationPopulationResult( + structure="collider", + conditioning_variable_role="collider", + unconditioned=_population_regression(("intercept", "X"), (0.0, 0.0), 0.0), + conditioned=_population_regression( + ("intercept", "X", "Z"), (0.0, -0.5, 0.5), 0.5 + ), + ) + + +def confounded_mediator_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 37-42 (the confounded mediator).""" + + return SpecificationPopulationResult( + structure="confounded_mediator", + conditioning_variable_role="confounded_mediator", + unconditioned=_population_regression(("intercept", "X"), (0.0, 1.0), 1.0 / 7.0), + conditioned=_population_regression( + ("intercept", "X", "Z"), + (0.0, -0.5, 1.5), + 11.0 / 14.0, + ), + ) + + +def fork_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded fork experiment from Section 7.1 and the Appendix.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + z = random_state.standard_normal(n_observations) + x = z + random_state.standard_normal(n_observations) + y = z + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) + + +def collider_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded collider experiment from Section 7.2 and the Appendix.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + x = random_state.standard_normal(n_observations) + y = random_state.standard_normal(n_observations) + z = x + y + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="collider", + conditioning_variable_role="collider", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) + + +def confounded_mediator_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded confounded-mediator experiment from Section 7.3.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + x = random_state.standard_normal(n_observations) + w = random_state.standard_normal(n_observations) + z = x + w + random_state.standard_normal(n_observations) + y = z + w + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="confounded_mediator", + conditioning_variable_role="confounded_mediator", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/treatment_effects.py b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py new file mode 100644 index 0000000..3c6d64a --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py @@ -0,0 +1,320 @@ +"""Treatment-effect estimands from *Causal Factor Investing* (2023). + +The functions are independent implementations of Equations 2-9 and the +linear instrumental-variable ratio in Section 4.3.2.4. They operate on +already identified interventional means, conditional means, probabilities, +or covariances. They do not infer a causal graph from data and do not claim +that the supplied adjustment variables or instrument are valid. +""" + +from __future__ import annotations + +import math +from numbers import Real +from typing import NamedTuple, Sequence + +import numpy as np + +__all__ = [ + "DifferenceInDifferencesEstimate", + "TreatmentEffectDecomposition", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "difference_in_differences", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "randomized_mean_difference", + "treatment_effect_decomposition", +] + + +class TreatmentEffectDecomposition(NamedTuple): + """Observed mean difference decomposed into ATT and selection bias.""" + + observed_difference: float + average_treatment_effect_on_treated: float + sample_selection_bias: float + + +class DifferenceInDifferencesEstimate(NamedTuple): + """Two-group, two-period difference-in-differences decomposition.""" + + treated_change: float + control_change: float + estimate: float + + +_PROBABILITY_TOLERANCE = 1.0e-12 + + +def _as_finite_real(name: str, value: Real) -> float: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real): + raise ValueError(f"{name} must be a real scalar") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must be a representable real scalar") from error + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _as_finite_vector(name: str, values: Sequence[Real]) -> np.ndarray: + try: + raw = np.asarray(values) + except (TypeError, ValueError) as error: + raise ValueError( + f"{name} must be a one-dimensional numeric sequence" + ) from error + if raw.ndim != 1 or raw.size == 0 or raw.dtype.kind not in "iuf": + raise ValueError(f"{name} must be a nonempty one-dimensional numeric sequence") + try: + result = np.array(raw, dtype=float, copy=True) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must contain representable real values") from error + if not np.all(np.isfinite(result)): + raise ValueError(f"{name} must contain only finite values") + return result + + +def _as_probability_vector(name: str, values: Sequence[Real]) -> np.ndarray: + result = _as_finite_vector(name, values) + if np.any(result < 0.0) or np.any(result > 1.0): + raise ValueError(f"{name} must contain probabilities in [0, 1]") + if not math.isclose( + float(np.sum(result)), + 1.0, + rel_tol=_PROBABILITY_TOLERANCE, + abs_tol=_PROBABILITY_TOLERANCE, + ): + raise ValueError(f"{name} must sum to one") + return result + + +def average_treatment_effect( + treated_interventional_mean: Real, + control_interventional_mean: Real, +) -> float: + """Return ``E[Y|do(X=x1)] - E[Y|do(X=x0)]`` (Equation 2).""" + + treated = _as_finite_real( + "treated_interventional_mean", treated_interventional_mean + ) + control = _as_finite_real( + "control_interventional_mean", control_interventional_mean + ) + return _as_finite_real("average_treatment_effect", treated - control) + + +def treatment_effect_decomposition( + observed_treated_mean: Real, + observed_control_mean: Real, + counterfactual_control_mean_for_treated: Real, +) -> TreatmentEffectDecomposition: + """Decompose the observed difference into ATT and selection bias. + + This is Equation 3 under consistency: the treated group's observed mean + is its treated potential-outcome mean. + """ + + treated = _as_finite_real("observed_treated_mean", observed_treated_mean) + control = _as_finite_real("observed_control_mean", observed_control_mean) + counterfactual = _as_finite_real( + "counterfactual_control_mean_for_treated", + counterfactual_control_mean_for_treated, + ) + observed_difference = _as_finite_real("observed_difference", treated - control) + effect_on_treated = _as_finite_real( + "average_treatment_effect_on_treated", treated - counterfactual + ) + selection_bias = _as_finite_real("sample_selection_bias", counterfactual - control) + return TreatmentEffectDecomposition( + observed_difference=observed_difference, + average_treatment_effect_on_treated=effect_on_treated, + sample_selection_bias=selection_bias, + ) + + +def randomized_mean_difference( + treated_outcomes: Sequence[Real], + control_outcomes: Sequence[Real], +) -> float: + """Return the randomized-study difference in sample means. + + Under the random-assignment identities in Equations 4-6, this is the + finite-sample estimator of the average treatment effect. + """ + + treated = _as_finite_vector("treated_outcomes", treated_outcomes) + control = _as_finite_vector("control_outcomes", control_outcomes) + return _as_finite_real( + "randomized_mean_difference", + float(np.mean(treated) - np.mean(control)), + ) + + +def difference_in_differences( + treated_before: Real, + treated_after: Real, + control_before: Real, + control_after: Real, +) -> DifferenceInDifferencesEstimate: + """Return the two-group, two-period difference-in-differences estimate.""" + + treated_change = _as_finite_real( + "treated_change", + _as_finite_real("treated_after", treated_after) + - _as_finite_real("treated_before", treated_before), + ) + control_change = _as_finite_real( + "control_change", + _as_finite_real("control_after", control_after) + - _as_finite_real("control_before", control_before), + ) + estimate = _as_finite_real( + "difference_in_differences", treated_change - control_change + ) + return DifferenceInDifferencesEstimate( + treated_change=treated_change, + control_change=control_change, + estimate=estimate, + ) + + +def backdoor_adjusted_expectation( + conditional_outcome_means: Sequence[Real], + adjustment_probabilities: Sequence[Real], +) -> float: + """Standardize conditional outcome means over an adjustment distribution. + + For an indicator outcome, this is the discrete probability adjustment in + Equation 8. For a general outcome, it is the expectation form used by + Equation 7. + """ + + means = _as_finite_vector("conditional_outcome_means", conditional_outcome_means) + probabilities = _as_probability_vector( + "adjustment_probabilities", adjustment_probabilities + ) + if means.shape != probabilities.shape: + raise ValueError( + "conditional_outcome_means and adjustment_probabilities must have " + "the same length" + ) + return _as_finite_real( + "backdoor_adjusted_expectation", float(np.dot(means, probabilities)) + ) + + +def backdoor_adjusted_average_treatment_effect( + treated_conditional_means: Sequence[Real], + control_conditional_means: Sequence[Real], + adjustment_probabilities: Sequence[Real], +) -> float: + """Return the standardized average treatment effect in Equation 7.""" + + treated = _as_finite_vector("treated_conditional_means", treated_conditional_means) + control = _as_finite_vector("control_conditional_means", control_conditional_means) + probabilities = _as_probability_vector( + "adjustment_probabilities", adjustment_probabilities + ) + if treated.shape != control.shape or treated.shape != probabilities.shape: + raise ValueError( + "treated_conditional_means, control_conditional_means, and " + "adjustment_probabilities must have the same length" + ) + with np.errstate(over="ignore", invalid="ignore"): + contrasts = treated - control + result = float(np.dot(contrasts, probabilities)) + return _as_finite_real("backdoor_adjusted_average_treatment_effect", result) + + +def frontdoor_adjusted_probability( + mediator_probabilities_given_treatment: Sequence[Real], + outcome_probabilities_given_mediator_and_treatment: Sequence[Sequence[Real]], + treatment_probabilities: Sequence[Real], +) -> float: + """Return the discrete front-door adjustment in Equation 9. + + Rows of ``outcome_probabilities_given_mediator_and_treatment`` correspond + to mediator states; columns correspond to treatment states. Each entry is + the conditional probability of one fixed outcome event. + """ + + mediator_probabilities = _as_probability_vector( + "mediator_probabilities_given_treatment", + mediator_probabilities_given_treatment, + ) + treatment_weights = _as_probability_vector( + "treatment_probabilities", treatment_probabilities + ) + try: + raw = np.asarray(outcome_probabilities_given_mediator_and_treatment) + except (TypeError, ValueError) as error: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must be a " + "two-dimensional numeric array" + ) from error + if raw.ndim != 2 or raw.dtype.kind not in "iuf": + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must be a " + "two-dimensional numeric array" + ) + try: + conditional = np.array(raw, dtype=float, copy=True) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "representable real values" + ) from error + if not np.all(np.isfinite(conditional)): + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "only finite values" + ) + if np.any(conditional < 0.0) or np.any(conditional > 1.0): + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "probabilities in [0, 1]" + ) + expected_shape = (mediator_probabilities.size, treatment_weights.size) + if conditional.shape != expected_shape: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must have shape " + "(number of mediator states, number of treatment states)" + ) + probability = float( + np.dot(mediator_probabilities, np.dot(conditional, treatment_weights)) + ) + probability = _as_finite_real("frontdoor_adjusted_probability", probability) + if probability < 0.0 and probability >= -_PROBABILITY_TOLERANCE: + return 0.0 + if probability > 1.0 and probability <= 1.0 + _PROBABILITY_TOLERANCE: + return 1.0 + return probability + + +def linear_instrumental_variable_effect( + outcome_instrument_covariance: Real, + treatment_instrument_covariance: Real, +) -> float: + """Return the linear IV covariance ratio from Section 4.3.2.4. + + The caller must establish the graphical and substantive IV premises. A + zero treatment-instrument covariance violates the ratio's relevance + requirement and is rejected. + """ + + numerator = _as_finite_real( + "outcome_instrument_covariance", outcome_instrument_covariance + ) + denominator = _as_finite_real( + "treatment_instrument_covariance", treatment_instrument_covariance + ) + if denominator == 0.0: + raise ValueError("treatment_instrument_covariance must be nonzero") + result = numerator / denominator + if not math.isfinite(result): + raise ValueError("the instrumental-variable effect must be finite") + return result diff --git a/src/RiskLabAI/cluster/__init__.py b/src/RiskLabAI/cluster/__init__.py new file mode 100644 index 0000000..e6bf92e --- /dev/null +++ b/src/RiskLabAI/cluster/__init__.py @@ -0,0 +1,27 @@ +""" +RiskLabAI Clustering Module + +This module provides functions for financial portfolio clustering, +including implementations of the Optimized Nested Clustering (ONC) +algorithm described by de Prado. +""" + +from .clustering import ( + cluster_k_means_base, + cluster_k_means_top, + covariance_to_correlation, + make_new_outputs, + random_block_correlation, + random_block_covariance, + random_covariance_sub, +) + +__all__ = [ + "covariance_to_correlation", + "cluster_k_means_base", + "make_new_outputs", + "cluster_k_means_top", + "random_covariance_sub", + "random_block_covariance", + "random_block_correlation", +] diff --git a/src/RiskLabAI/cluster/clustering.py b/src/RiskLabAI/cluster/clustering.py new file mode 100644 index 0000000..6fac684 --- /dev/null +++ b/src/RiskLabAI/cluster/clustering.py @@ -0,0 +1,462 @@ +""" +Implements clustering algorithms, including the Optimized Nested Clustering (ONC) +method from Marcos Lopez de Prado. + +Reference: + De Prado, M. (2020) Advances in financial machine learning. John Wiley & Sons. +""" + +from typing import Optional + +import numpy as np +import pandas as pd +from scipy.linalg import block_diag +from sklearn.cluster import KMeans +from sklearn.metrics import silhouette_samples +from sklearn.utils import check_random_state + + +def covariance_to_correlation(covariance: np.ndarray) -> np.ndarray: + r""" + Derive the correlation matrix from a covariance matrix. + + .. math:: + \text{corr}_{ij} = \frac{\text{cov}_{ij}} + {\sqrt{\text{cov}_{ii} \text{cov}_{jj}}} + + Reference: + Snippet 2.3, Page 27. + + Parameters + ---------- + covariance : np.ndarray + A square covariance matrix. + + Returns + ------- + np.ndarray + The corresponding correlation matrix. + """ + # Single source of truth: RiskLabAI.data.denoise.cov_to_corr implements the + # same conversion (Snippet 2.3) with added zero-std and diagonal safeguards. + # Output is identical to floating-point precision for valid covariance + # matrices. Imported locally to avoid any import cycle at module load. + from RiskLabAI.data.denoise.denoising import cov_to_corr + + return cov_to_corr(covariance) + + +def cluster_k_means_base( + correlation: pd.DataFrame, + max_clusters: int = 10, + iterations: int = 10, + random_state: Optional[int] = None, +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Perform the base K-Means clustering step. + + This function iterates through different numbers of clusters (from 2 to + `max_clusters`) and multiple initializations (`iterations`), selecting the + clustering result that yields the highest average silhouette score + t-statistic (mean / std). + + Reference: + Snippet 4.1, Page 56. + + Parameters + ---------- + correlation : pd.DataFrame + The correlation matrix to be clustered. + max_clusters : int, default=10 + The maximum number of clusters to try. + iterations : int, default=10 + Number of K-Means initializations for each cluster count. + random_state : int, optional + Random state for K-Means reproducibility. + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_sorted: The correlation matrix, reordered by cluster. + - clusters: A dictionary mapping cluster ID to a list of item names. + - silhouette_scores: A Series of silhouette scores for each item. + """ + # Calculate distance matrix + distance = ((1 - correlation.fillna(0)) / 2.0) ** 0.5 + + best_kmeans = None + best_silhouette_scores = None + best_score = -np.inf + + rng = check_random_state(random_state) + + for _ in range(iterations): + for n_clusters in range(2, max_clusters + 1): + # Use a different random_state for each K-Means fit + iter_seed = rng.randint(0, np.iinfo(np.int32).max) + + kmeans = KMeans( + n_clusters=n_clusters, + n_init=1, # We handle iterations externally + random_state=iter_seed, + ) + kmeans.fit(distance) + + silhouette_scores_ = silhouette_samples(distance, kmeans.labels_) + + # Use silhouette score t-statistic (mean/std) as the quality metric + stat_mean = silhouette_scores_.mean() + stat_std = silhouette_scores_.std() + + if stat_std == 0: + # Avoid division by zero if all silhouette scores are identical + score = np.sign(stat_mean) * np.inf + else: + score = stat_mean / stat_std + + if score > best_score: + best_score = score + best_kmeans = kmeans + best_silhouette_scores = silhouette_scores_ + + if best_kmeans is None: + raise ValueError("Clustering failed to find a valid solution.") + + # Sort items by cluster label + index_sorted = np.argsort(best_kmeans.labels_) + correlation_sorted = correlation.iloc[index_sorted, index_sorted] + + # Create cluster dictionary + clusters = { + i: correlation.columns[np.where(best_kmeans.labels_ == i)[0]].tolist() + for i in np.unique(best_kmeans.labels_) + } + + silhouette_series = pd.Series(best_silhouette_scores, index=distance.index) + + return correlation_sorted, clusters, silhouette_series + + +def make_new_outputs( + correlation: pd.DataFrame, + clusters_1: dict[int, list[str]], + clusters_2: dict[int, list[str]], +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Merge two disjoint sets of clusters and re-calculate metrics. + + Reference: + Snippet 4.2, Page 58. + + Parameters + ---------- + correlation : pd.DataFrame + The *original* correlation matrix. + clusters_1 : Dict[int, List[str]] + The first set of clusters (e.g., the "good" clusters). + clusters_2 : Dict[int, List[str]] + The second set of clusters (e.g., the re-clustered "bad" ones). + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_new: The correlation matrix, reordered by new clusters. + - clusters_new: The merged dictionary of all clusters. + - silhouette_new: New silhouette scores based on the merged clustering. + """ + clusters_new = {} + for i in clusters_1: + clusters_new[len(clusters_new)] = list(clusters_1[i]) + for i in clusters_2: + clusters_new[len(clusters_new)] = list(clusters_2[i]) + + # Get the new full, sorted index + index_new = [ + item for cluster_items in clusters_new.values() for item in cluster_items + ] + correlation_new = correlation.loc[index_new, index_new] + + # Calculate new silhouette scores based on the *original* distance + distance = ((1 - correlation.fillna(0)) / 2.0) ** 0.5 + labels_kmeans = np.zeros(len(distance.columns)) + + # Create the label array for silhouette_samples + for i, items in clusters_new.items(): + item_indices = [distance.index.get_loc(k) for k in items] + labels_kmeans[item_indices] = i + + silhouette_new = pd.Series( + silhouette_samples(distance, labels_kmeans), index=distance.index + ) + return correlation_new, clusters_new, silhouette_new + + +def cluster_k_means_top( + correlation: pd.DataFrame, + max_clusters: Optional[int] = None, + iterations: int = 10, + random_state: Optional[int] = None, +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Perform Optimized Nested Clustering (ONC). + + This method recursively re-clusters unstable clusters (those with a + silhouette t-statistic below the average). + + Reference: + Snippet 4.2, Page 58. + + Parameters + ---------- + correlation : pd.DataFrame + The correlation matrix to be clustered. + max_clusters : int, optional + Maximum number of clusters. Defaults to `N-1`. + iterations : int, default=10 + Number of K-Means initializations. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_sorted: The final sorted correlation matrix. + - clusters: The final dictionary of clusters. + - silhouette_scores: The final silhouette scores for each item. + """ + n_cols = correlation.shape[1] + if max_clusters is None: + max_clusters = n_cols - 1 + + max_clusters = min(max_clusters, n_cols - 1) + if max_clusters < 2: + return ( + correlation, + {0: correlation.columns.tolist()}, + pd.Series(dtype="float64"), + ) + + # 1. Run base clustering + corr_sorted, clusters, silhouette = cluster_k_means_base( + correlation, + max_clusters=max_clusters, + iterations=iterations, + random_state=random_state, + ) + + # 2. Calculate t-statistic for each cluster + cluster_t_stats = { + i: silhouette[clusters[i]].mean() / silhouette[clusters[i]].std() + for i in clusters + if silhouette[clusters[i]].std() > 0 + } + + if not cluster_t_stats: + return corr_sorted, clusters, silhouette # No valid clusters found + + t_stat_mean = np.mean(list(cluster_t_stats.values())) + + # 3. Identify clusters to re-cluster + redo_clusters = [i for i, t_stat in cluster_t_stats.items() if t_stat < t_stat_mean] + + if len(redo_clusters) <= 1: + # Base case: All clusters are stable, or only one is unstable + return corr_sorted, clusters, silhouette + else: + # 4. Recurse on unstable clusters + keys_redo = [item for i in redo_clusters for item in clusters[i]] + corr_temp = correlation.loc[keys_redo, keys_redo] + + # Keep track of mean t-stat for comparison + t_stat_mean_redo = np.mean([cluster_t_stats[i] for i in redo_clusters]) + + # Calculate remaining clusters for recursive call + n_clusters_good = len(clusters) - len(redo_clusters) + remained_n_clusters = max_clusters - n_clusters_good + + # Recursive call + corr_sorted_2, clusters_2, silh_2 = cluster_k_means_top( + corr_temp, + max_clusters=min(remained_n_clusters, corr_temp.shape[1] - 1), + iterations=iterations, + random_state=random_state, + ) + + # 5. Merge results + clusters_1 = {i: clusters[i] for i in clusters if i not in redo_clusters} + corr_new, clusters_new, silh_new = make_new_outputs( + correlation, clusters_1, clusters_2 + ) + + # 6. Decide whether to keep the re-clustered result + new_t_stats = [ + silh_new[clusters_new[i]].mean() / silh_new[clusters_new[i]].std() + for i in clusters_new + if silh_new[clusters_new[i]].std() > 0 + ] + + if not new_t_stats: + return corr_sorted, clusters, silhouette # Re-clustering failed + + new_t_stat_mean = np.mean(new_t_stats) + + if new_t_stat_mean <= t_stat_mean_redo: + # Re-clustering did not improve, return original + return corr_sorted, clusters, silhouette + else: + # Re-clustering improved, return new results + return corr_new, clusters_new, silh_new + + +def random_covariance_sub( + n_observations: int, + n_columns: int, + sigma: float, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate a random covariance matrix for a single block. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_observations : int + Number of observations to simulate. + n_columns : int + Number of columns (assets) in this block. + sigma : float + Sigma for the idiosyncratic noise. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + np.ndarray + The (n_columns x n_columns) covariance matrix. + """ + rng = check_random_state(random_state) + + if n_columns == 1: + return np.ones((1, 1)) + + # Common factor + data = rng.normal(size=(n_observations, 1)) + data = np.repeat(data, n_columns, axis=1) + + # Idiosyncratic noise + data += rng.normal(scale=sigma, size=data.shape) + + covariance = np.cov(data, rowvar=False) + return covariance + + +def random_block_covariance( + n_columns: int, + n_blocks: int, + block_size_min: int = 1, + sigma: float = 1.0, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate a random block-diagonal covariance matrix. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_columns : int + Total number of columns (assets). + n_blocks : int + Number of blocks to create. + block_size_min : int, default=1 + The minimum size for each block. + sigma : float, default=1.0 + Sigma for idiosyncratic noise. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + np.ndarray + The (n_columns x n_columns) block-diagonal covariance matrix. + """ + rng = check_random_state(random_state) + + # Generate block sizes + parts = rng.choice( + range(1, n_columns - (block_size_min - 1) * n_blocks), + n_blocks - 1, + replace=False, + ) + parts.sort() + parts = np.append(parts, n_columns - (block_size_min - 1) * n_blocks) + parts = np.append(parts[0], np.diff(parts)) - 1 + block_size_min + + cov_list = [] + for col_size in parts: + # Number of observations must be > number of columns + n_obs = int(max(col_size * (col_size + 1) / 2.0, 100)) + + this_covariance = random_covariance_sub( + n_obs, col_size, sigma, random_state=rng + ) + cov_list.append(this_covariance) + + return block_diag(*cov_list) + + +def random_block_correlation( + n_columns: int, + n_blocks: int, + random_state: Optional[int] = None, + block_size_min: int = 1, +) -> pd.DataFrame: + """ + Generate a random block-diagonal correlation matrix with noise. + + This adds a "market" component (a single block covariance) + to the block-diagonal structure. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_columns : int + Total number of columns (assets). + n_blocks : int + Number of blocks to create. + random_state : int, optional + Random state for reproducibility. + block_size_min : int, default=1 + The minimum size for each block. + + Returns + ------- + pd.DataFrame + The (n_columns x n_columns) correlation matrix. + """ + rng = check_random_state(random_state) + + # Block-diagonal structure + covariance1 = random_block_covariance( + n_columns, + n_blocks, + block_size_min=block_size_min, + sigma=0.5, + random_state=rng, + ) + + # Market component (noise) + covariance2 = random_block_covariance( + n_columns, 1, block_size_min=n_columns, sigma=1.0, random_state=rng + ) + + covariance = covariance1 + covariance2 + correlation = covariance_to_correlation(covariance) + correlation = pd.DataFrame(correlation) + + return correlation diff --git a/src/RiskLabAI/controller/__init__.py b/src/RiskLabAI/controller/__init__.py new file mode 100644 index 0000000..11fb6fd --- /dev/null +++ b/src/RiskLabAI/controller/__init__.py @@ -0,0 +1,2 @@ +from .bars_initializer import * +from .data_structure_controller import * diff --git a/src/RiskLabAI/controller/bars_initializer.py b/src/RiskLabAI/controller/bars_initializer.py new file mode 100644 index 0000000..cfd2996 --- /dev/null +++ b/src/RiskLabAI/controller/bars_initializer.py @@ -0,0 +1,299 @@ +""" +Controller class to act as a factory for initializing +various bar types (Standard, Time, Imbalance, Run). +""" + +from typing import Any, Callable, Optional, Union + +import pandas as pd + +from RiskLabAI.data.structures.abstract_bars import AbstractBars + +# Import bar types +from RiskLabAI.data.structures.imbalance_bars import ( + ExpectedImbalanceBars, + FixedImbalanceBars, +) +from RiskLabAI.data.structures.run_bars import ExpectedRunBars, FixedRunBars +from RiskLabAI.data.structures.standard_bars import StandardBars +from RiskLabAI.data.structures.time_bars import TimeBars + +# Import constants +from RiskLabAI.utils.constants import ( + CUMULATIVE_DOLLAR, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, +) + + +class BarsInitializerController: + """ + Controller for initializing various types of bars. + + This class acts as a factory, providing static methods to + construct different bar objects with sensible default parameters. + """ + + def __init__(self): + """ + Initializes the controller and maps method names to methods. + """ + self.method_name_to_method: dict[str, Callable[..., AbstractBars]] = { + "expected_dollar_imbalance_bars": self.initialize_expected_dollar_imbalance_bars, + "expected_volume_imbalance_bars": self.initialize_expected_volume_imbalance_bars, + "expected_tick_imbalance_bars": self.initialize_expected_tick_imbalance_bars, + "fixed_dollar_imbalance_bars": self.initialize_fixed_dollar_imbalance_bars, + "fixed_volume_imbalance_bars": self.initialize_fixed_volume_imbalance_bars, + "fixed_tick_imbalance_bars": self.initialize_fixed_tick_imbalance_bars, + "expected_dollar_run_bars": self.initialize_expected_dollar_run_bars, + "expected_volume_run_bars": self.initialize_expected_volume_run_bars, + "expected_tick_run_bars": self.initialize_expected_tick_run_bars, + "fixed_dollar_run_bars": self.initialize_fixed_dollar_run_bars, + "fixed_volume_run_bars": self.initialize_fixed_volume_run_bars, + "fixed_tick_run_bars": self.initialize_fixed_tick_run_bars, + "dollar_standard_bars": self.initialize_dollar_standard_bars, + "volume_standard_bars": self.initialize_volume_standard_bars, + "tick_standard_bars": self.initialize_tick_standard_bars, + "time_bars": self.initialize_time_bars, + } + + @staticmethod + def initialize_expected_dollar_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + window_size_for_expected_imbalance_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, # Accept extra kwargs but don't use them + ) -> ExpectedImbalanceBars: + """ + Initialize expected dollar imbalance bars. + """ + return ExpectedImbalanceBars( + bar_type="dollar_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_volume_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedImbalanceBars: + """Initialize expected volume imbalance bars.""" + return ExpectedImbalanceBars( + bar_type="volume_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_tick_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedImbalanceBars: + """Initialize expected tick imbalance bars.""" + return ExpectedImbalanceBars( + bar_type="tick_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_dollar_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed dollar imbalance bars.""" + return FixedImbalanceBars( + bar_type="dollar_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_volume_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed volume imbalance bars.""" + return FixedImbalanceBars( + bar_type="volume_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_tick_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed tick imbalance bars.""" + return FixedImbalanceBars( + bar_type="tick_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_dollar_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected dollar run bars.""" + return ExpectedRunBars( + bar_type="dollar_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_volume_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected volume run bars.""" + return ExpectedRunBars( + bar_type="volume_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_tick_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected tick run bars.""" + return ExpectedRunBars( + bar_type="tick_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_dollar_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed dollar run bars.""" + return FixedRunBars( + bar_type="dollar_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_volume_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed volume run bars.""" + return FixedRunBars( + bar_type="volume_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_tick_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed tick run bars.""" + return FixedRunBars( + bar_type="tick_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_dollar_standard_bars( + threshold: Union[float, pd.Series] = 70000000, + **kwargs: Any, + ) -> StandardBars: + """Initialize dollar standard bars.""" + return StandardBars(bar_type=CUMULATIVE_DOLLAR, threshold=threshold) + + @staticmethod + def initialize_volume_standard_bars( + threshold: Union[float, pd.Series] = 30000, + **kwargs: Any, + ) -> StandardBars: + """Initialize volume standard bars.""" + return StandardBars(bar_type=CUMULATIVE_VOLUME, threshold=threshold) + + @staticmethod + def initialize_tick_standard_bars( + threshold: Union[float, pd.Series] = 6000, + **kwargs: Any, + ) -> StandardBars: + """Initialize tick standard bars.""" + return StandardBars(bar_type=CUMULATIVE_TICKS, threshold=threshold) + + @staticmethod + def initialize_time_bars( + resolution_type: str = "D", + resolution_units: int = 1, + **kwargs: Any, + ) -> TimeBars: + """Initialize time bars.""" + return TimeBars( + resolution_type=resolution_type, resolution_units=resolution_units + ) diff --git a/src/RiskLabAI/controller/data_structure_controller.py b/src/RiskLabAI/controller/data_structure_controller.py new file mode 100644 index 0000000..5b23082 --- /dev/null +++ b/src/RiskLabAI/controller/data_structure_controller.py @@ -0,0 +1,188 @@ +""" +Main controller for processing tick data into bars. + +This class reads data in batches from CSVs or DataFrames and +uses the BarsInitializerController to construct bars based on +a specified method. +""" + +import logging +from collections.abc import Generator +from typing import Any, Optional, Union + +import pandas as pd + +from RiskLabAI.controller.bars_initializer import BarsInitializerController +from RiskLabAI.data.structures.abstract_bars import AbstractBars +from RiskLabAI.utils.constants import ( + CLOSE_PRICE, + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_DOLLAR, + CUMULATIVE_SELL_VOLUME, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, + DATE_TIME, + HIGH_PRICE, + LOW_PRICE, + OPEN_PRICE, + THRESHOLD, + TICK_NUMBER, +) + +logger = logging.getLogger(__name__) + +# Define the bar column schema +BAR_COLUMNS = [ + DATE_TIME, + TICK_NUMBER, + OPEN_PRICE, + HIGH_PRICE, + LOW_PRICE, + CLOSE_PRICE, + CUMULATIVE_VOLUME, + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_SELL_VOLUME, + CUMULATIVE_TICKS, + CUMULATIVE_DOLLAR, + THRESHOLD, +] + + +class Controller: + """ + Controller for initializing and processing bars from data sources. + """ + + def __init__(self) -> None: + """ + Initializes the Controller and its internal BarsInitializerController. + """ + self.bars_initializer = BarsInitializerController() + + def handle_input_command( + self, + method_name: str, + method_arguments: dict[str, Any], + input_data: Union[str, pd.DataFrame], + output_path: Optional[str] = None, + batch_size: int = 1_000_000, + ) -> pd.DataFrame: + """ + Handles the command to initialize a bar generator and process data. + (Parameters same as original) + """ + # 1. Initialize the bar generator + try: + initializer_method = self.bars_initializer.method_name_to_method[ + method_name + ] + except KeyError: + valid_methods = list(self.bars_initializer.method_name_to_method.keys()) + logger.error( + "Bar method '%s' not found. Valid methods are: %s", + method_name, + valid_methods, + ) + return pd.DataFrame(columns=BAR_COLUMNS) + + try: + bar_generator: AbstractBars = initializer_method(**method_arguments) + except TypeError as e: + logger.error( + "Error initializing bar method '%s' with arguments %s: %s", + method_name, + method_arguments, + e, + ) + return pd.DataFrame(columns=BAR_COLUMNS) + + # 2. Get the correct batch generator + if isinstance(input_data, str): + data_generator = self.read_batches_from_string(input_data, batch_size) + elif isinstance(input_data, pd.DataFrame): + data_generator = self.read_batches_from_dataframe(input_data, batch_size) + else: + raise TypeError("input_data must be a string (path) or pd.DataFrame") + + all_bars: list[list[Any]] = [] + + # 3. Process data in batches + logger.info("Processing data in batches...") + try: + for data_batch in data_generator: + # We assume data is [datetime, price, volume] + # .values returns a NumPy array, which is an efficient iterable + bars = bar_generator.construct_bars_from_data(data=data_batch.values) + all_bars.extend(bars) + except Exception as e: + logger.warning( + "Error during bar construction: %s. Returning DataFrame with " + "bars constructed so far.", + e, + ) + # Continue to return whatever was processed + + logger.info("Done. Constructed %d bars.", len(all_bars)) + + # 4. Create final DataFrame + bars_df = pd.DataFrame(all_bars, columns=BAR_COLUMNS) + + if output_path: + logger.info("Saving bars to %s...", output_path) + try: + bars_df.to_csv(output_path, index=False) + logger.info("Save complete.") + except Exception as e: + logger.error("Error saving file to %s: %s", output_path, e) + + return bars_df + + @staticmethod + def read_batches_from_string( + input_path: str, batch_size: int + ) -> Generator[pd.DataFrame, None, None]: + """ + Reads data in batches from a CSV file efficiently. + + Parameters + ---------- + input_path : str + File path to read from. + batch_size : int + Size of each batch. + + Yields + ------- + Generator[pd.DataFrame, None, None] + A generator yielding batches of data. + """ + try: + # Use a generator to read the file in chunks + # This is more memory-efficient and avoids reading the file twice. + yield from pd.read_csv(input_path, chunksize=batch_size, parse_dates=[0]) + except FileNotFoundError: + logger.error("File not found at %s", input_path) + return + except pd.errors.ParserError as e: + logger.error("Error parsing CSV file %s: %s", input_path, e) + return + except Exception as e: + logger.error( + "An unexpected error occurred while reading %s: %s", input_path, e + ) + return + + @staticmethod + def read_batches_from_dataframe( + input_data: pd.DataFrame, batch_size: int + ) -> Generator[pd.DataFrame, None, None]: + """ + Reads data in batches from a DataFrame. + """ + n_rows = input_data.shape[0] + if n_rows == 0: + return + + for start_row in range(0, n_rows, batch_size): + end_row = min(start_row + batch_size, n_rows) + yield input_data.iloc[start_row:end_row] diff --git a/src/RiskLabAI/core/__init__.py b/src/RiskLabAI/core/__init__.py new file mode 100644 index 0000000..06fd69c --- /dev/null +++ b/src/RiskLabAI/core/__init__.py @@ -0,0 +1,67 @@ +""" +RiskLabAI.core — the extension layer. + +This package holds the small set of abstractions that make RiskLabAI *modular, +extendable and maintainable*: a reusable component :class:`Registry`, the base +interfaces every model family targets, and a built-in catalogue of the +components the library already ships. + +It is the recommended entry point for adding a **new** model (for example, an +extension or alternative to a method from López de Prado's books): implement the +relevant base interface, register it in the matching registry, and it becomes +discoverable and constructible by name — without editing any central file. See +``EXTENDING.md`` at the repository root for a step-by-step guide. + +Importing this package is cheap: every component is registered lazily, so heavy +dependencies are only imported when a component is actually created. + +Quick tour +---------- +>>> from RiskLabAI.core import list_components, CROSS_VALIDATORS +>>> sorted(list_components()) # doctest: +SKIP +['bars', 'bet_sizers', 'cross_validators', ...] +>>> "purgedkfold" in CROSS_VALIDATORS +True +>>> cv = CROSS_VALIDATORS.create("purgedkfold", n_splits=5) # doctest: +SKIP +""" + +from __future__ import annotations + +from ._builtins import ( + BARS, + BET_SIZERS, + CROSS_VALIDATORS, + FEATURE_IMPORTANCE, + LABELERS, + PORTFOLIO_OPTIMIZERS, + REGISTRIES, + get_registry, + list_components, +) +from .base import ( + BaseBetSizer, + BaseLabeler, + BasePortfolioOptimizer, + Estimator, +) +from .registry import Registry + +__all__ = [ + # Registry machinery + "Registry", + "REGISTRIES", + "get_registry", + "list_components", + # Per-family registries + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + # Base contracts + "Estimator", + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer", +] diff --git a/src/RiskLabAI/core/_builtins.py b/src/RiskLabAI/core/_builtins.py new file mode 100644 index 0000000..7cb533a --- /dev/null +++ b/src/RiskLabAI/core/_builtins.py @@ -0,0 +1,195 @@ +""" +The built-in component catalogue. + +Defines one :class:`~RiskLabAI.core.registry.Registry` per model family and +populates it with the components the library already ships. Registration is +**lazy** (by import path), so importing this module — and therefore +``RiskLabAI.core`` — does not import pandas, numba, scikit-learn or torch. The +heavy import only happens when a component is actually created. + +Families currently exposed only as free functions (labeling, bet sizing, +portfolio optimization) get an empty registry here: it is the extension point +where new class-based models register themselves (see ``EXTENDING.md``). + +The registries are deliberately a *superset* of the existing hand-written +factories (``CrossValidatorFactory``, ``FeatureImportanceFactory``, +``BarsInitializerController``), which remain unchanged for backward +compatibility. ``test/core/test_builtin_parity.py`` asserts the registries stay +in sync with those factories so the two cannot silently drift apart. +""" + +from __future__ import annotations + +from .registry import Registry + +__all__ = [ + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + "REGISTRIES", + "get_registry", + "list_components", +] + +# --------------------------------------------------------------------------- # +# Bar structures. The extension point for a new bar type is a new AbstractBars +# subclass; register it here (or via @BARS.register in your own code). +# --------------------------------------------------------------------------- # +BARS = Registry("bars") +BARS.register_lazy( + "standard_bars", + "RiskLabAI.data.structures.standard_bars:StandardBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "time_bars", + "RiskLabAI.data.structures.time_bars:TimeBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "expected_imbalance_bars", + "RiskLabAI.data.structures.imbalance_bars:ExpectedImbalanceBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "fixed_imbalance_bars", + "RiskLabAI.data.structures.imbalance_bars:FixedImbalanceBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "expected_run_bars", + "RiskLabAI.data.structures.run_bars:ExpectedRunBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "fixed_run_bars", + "RiskLabAI.data.structures.run_bars:FixedRunBars", + metadata={"family": "bars", "afml_chapter": 2}, +) + +# --------------------------------------------------------------------------- # +# Cross-validators. Keys mirror CrossValidatorFactory.VALIDATORS exactly. +# --------------------------------------------------------------------------- # +CROSS_VALIDATORS = Registry("cross_validators") +CROSS_VALIDATORS.register_lazy( + "kfold", + "RiskLabAI.backtest.validation.kfold:KFold", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "walkforward", + "RiskLabAI.backtest.validation.walk_forward:WalkForward", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "purgedkfold", + "RiskLabAI.backtest.validation.purged_kfold:PurgedKFold", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "combinatorialpurged", + "RiskLabAI.backtest.validation.combinatorial_purged:CombinatorialPurged", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "baggedcombinatorialpurged", + "RiskLabAI.backtest.validation.bagged_combinatorial_purged:BaggedCombinatorialPurged", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "adaptivecombinatorialpurged", + "RiskLabAI.backtest.validation.adaptive_combinatorial_purged:AdaptiveCombinatorialPurged", + metadata={"family": "cross_validator"}, +) + +# --------------------------------------------------------------------------- # +# Feature-importance strategies. Keys mirror FeatureImportanceFactory exactly. +# --------------------------------------------------------------------------- # +FEATURE_IMPORTANCE = Registry("feature_importance") +FEATURE_IMPORTANCE.register_lazy( + "MDI", + "RiskLabAI.features.feature_importance.feature_importance_mdi:FeatureImportanceMDI", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "ClusteredMDI", + "RiskLabAI.features.feature_importance.clustered_feature_importance_mdi:ClusteredFeatureImportanceMDI", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "MDA", + "RiskLabAI.features.feature_importance.feature_importance_mda:FeatureImportanceMDA", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "ClusteredMDA", + "RiskLabAI.features.feature_importance.clustered_feature_importance_mda:ClusteredFeatureImportanceMDA", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "SFI", + "RiskLabAI.features.feature_importance.feature_importance_sfi:FeatureImportanceSFI", + metadata={"family": "feature_importance"}, +) + +# --------------------------------------------------------------------------- # +# Free-function families: empty registries, ready for new class-based models. +# These correspond to the BaseLabeler / BaseBetSizer / BasePortfolioOptimizer +# contracts in RiskLabAI.core.base. +# --------------------------------------------------------------------------- # +LABELERS = Registry("labelers") +BET_SIZERS = Registry("bet_sizers") +PORTFOLIO_OPTIMIZERS = Registry("portfolio_optimizers") + +# --------------------------------------------------------------------------- # +# Family name -> registry, for discovery and a unified catalogue. +# --------------------------------------------------------------------------- # +REGISTRIES: dict[str, Registry] = { + "bars": BARS, + "cross_validators": CROSS_VALIDATORS, + "feature_importance": FEATURE_IMPORTANCE, + "labelers": LABELERS, + "bet_sizers": BET_SIZERS, + "portfolio_optimizers": PORTFOLIO_OPTIMIZERS, +} + + +def get_registry(family: str) -> Registry: + """ + Return the registry for a model family. + + Parameters + ---------- + family : str + One of the keys of :data:`REGISTRIES` (e.g. ``"bars"``, + ``"cross_validators"``). + + Raises + ------ + KeyError + If the family name is unknown (the message lists valid families). + """ + try: + return REGISTRIES[family] + except KeyError: + valid = ", ".join(repr(k) for k in sorted(REGISTRIES)) + raise KeyError( + f"{family!r} is not a known model family. Valid families: {valid}." + ) from None + + +def list_components() -> dict[str, list[str]]: + """ + Return a catalogue mapping each family name to its available component keys. + + Useful for discovery and for documentation that should not drift:: + + >>> from RiskLabAI.core import list_components + >>> catalogue = list_components() + >>> catalogue["cross_validators"] # doctest: +SKIP + ['adaptivecombinatorialpurged', 'baggedcombinatorialpurged', ...] + """ + return {family: reg.available() for family, reg in REGISTRIES.items()} diff --git a/src/RiskLabAI/core/base.py b/src/RiskLabAI/core/base.py new file mode 100644 index 0000000..6e0c709 --- /dev/null +++ b/src/RiskLabAI/core/base.py @@ -0,0 +1,193 @@ +""" +Base interfaces (contracts) for RiskLabAI model families. + +This module gives every model family a single, documented contract that new +implementations can target. Two kinds of contract live here: + +1. **Canonical interfaces that already exist** elsewhere in the library are + re-exported from here so there is one obvious place to look: + :class:`AbstractBars`, :class:`CrossValidator`, and + :class:`FeatureImportanceStrategy`. These are re-exported *lazily* (PEP 562) + so that ``import RiskLabAI.core`` does not eagerly pull in pandas/numba and + the rest of the data/backtest/features sub-packages. + +2. **New, optional abstract base classes** for families that are currently + exposed only as free functions (labeling, bet sizing, portfolio + optimization). They are purely additive: existing functions keep working + unchanged. A *new* model can subclass one of these to advertise a uniform + interface (and to be registered and discovered through + :mod:`RiskLabAI.core.registry`). + +Plus a structural :class:`Estimator` protocol describing the scikit-learn-style +``fit``/``predict`` objects the cross-validation and feature-importance code +already expects. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Protocol, runtime_checkable + +import pandas as pd + +__all__ = [ + # Structural protocol + "Estimator", + # New optional contracts for free-function families + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer", + # Re-exported canonical interfaces (lazy) + "AbstractBars", + "BarBuilder", + "CrossValidator", + "FeatureImportanceStrategy", +] + + +@runtime_checkable +class Estimator(Protocol): + """ + Structural type for a scikit-learn-style estimator. + + Anything with ``fit`` and ``predict`` satisfies this protocol; no + inheritance is required (it is ``runtime_checkable``, so + ``isinstance(model, Estimator)`` works for duck-typed models). This is the + object the cross-validation and feature-importance machinery consumes. + """ + + def fit(self, X: Any, y: Any = None, **kwargs: Any) -> Any: ... + + def predict(self, X: Any) -> Any: ... + + +class BaseLabeler(ABC): + """ + Optional contract for a labeling method (e.g. triple-barrier, trend-scanning). + + The library's existing labelers are free functions and remain so. New + class-based labelers may subclass this to expose a uniform ``label`` entry + point and be registered in the ``labelers`` registry. See + ``EXTENDING.md`` for a worked example. + """ + + @abstractmethod + def label( + self, + prices: pd.Series, + events: pd.DataFrame | None = None, + **kwargs: Any, + ) -> pd.DataFrame: + """ + Produce labels for the given price series. + + Parameters + ---------- + prices : pd.Series + Price (or log-price) series indexed by timestamp. + events : pd.DataFrame, optional + Event definitions (e.g. event start times, vertical barriers, + target sizes) where applicable. + **kwargs + Method-specific options. + + Returns + ------- + pd.DataFrame + Labels, conventionally indexed by event time. + """ + raise NotImplementedError + + +class BaseBetSizer(ABC): + """ + Optional contract for a bet-sizing method. + + Maps model outputs (e.g. predicted probabilities) to a position size in + ``[-1, 1]``. New class-based bet sizers may subclass this and register in + the ``bet_sizers`` registry. + """ + + @abstractmethod + def bet_size(self, probabilities: pd.Series, **kwargs: Any) -> pd.Series: + """ + Convert prediction probabilities into signed bet sizes. + + Parameters + ---------- + probabilities : pd.Series + Predicted probabilities (or scores) indexed by timestamp. + **kwargs + Method-specific options. + + Returns + ------- + pd.Series + Signed bet sizes, conventionally in ``[-1, 1]``. + """ + raise NotImplementedError + + +class BasePortfolioOptimizer(ABC): + """ + Optional contract for a portfolio-construction method (e.g. HRP, NCO). + + New class-based optimizers may subclass this and register in the + ``portfolio_optimizers`` registry. + """ + + @abstractmethod + def weights(self, returns: pd.DataFrame, **kwargs: Any) -> pd.Series: + """ + Compute portfolio weights. + + Parameters + ---------- + returns : pd.DataFrame + Asset returns (rows = observations, columns = assets). Some + optimizers may accept a covariance matrix instead; document the + expectation in the concrete subclass. + **kwargs + Method-specific options. + + Returns + ------- + pd.Series + Portfolio weights indexed by asset. + """ + raise NotImplementedError + + +# --------------------------------------------------------------------------- # +# Lazy re-exports of the canonical interfaces that already live in the library. +# Importing them here eagerly would pull in the heavy data/backtest/features +# sub-package __init__ chains, defeating the lazy-import design. PEP 562 +# __getattr__ defers that cost until the name is actually accessed. +# --------------------------------------------------------------------------- # +_LAZY_REEXPORTS = { + "AbstractBars": "RiskLabAI.data.structures.abstract_bars:AbstractBars", + "BarBuilder": "RiskLabAI.data.structures.abstract_bars:AbstractBars", + "CrossValidator": ( + "RiskLabAI.backtest.validation.cross_validator_interface:CrossValidator" + ), + "FeatureImportanceStrategy": ( + "RiskLabAI.features.feature_importance.feature_importance_strategy" + ":FeatureImportanceStrategy" + ), +} + + +def __getattr__(name: str) -> Any: + target = _LAZY_REEXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + import importlib + + module_path, _, attribute = target.partition(":") + obj = getattr(importlib.import_module(module_path), attribute) + globals()[name] = obj # cache for next access + return obj + + +def __dir__() -> list: + return sorted(__all__) diff --git a/src/RiskLabAI/core/registry.py b/src/RiskLabAI/core/registry.py new file mode 100644 index 0000000..ab47928 --- /dev/null +++ b/src/RiskLabAI/core/registry.py @@ -0,0 +1,402 @@ +""" +A small, dependency-free component registry. + +Several sub-packages in RiskLabAI already implement the same pattern by hand: +a string key is mapped to a class, and a factory instantiates the right class +from that key (``controller/bars_initializer.py``, +``backtest/validation/cross_validator_factory.py``, +``features/feature_importance/feature_importance_factory.py``). Each does it +slightly differently (different casing rules, different error behaviour, a +hand-written ``dict`` in three places). + +``Registry`` factors that pattern into one reusable, well-tested object so that: + +* every model family discovers and instantiates its components the same way; +* **new** models (e.g. an extension proposed in a RiskLabAI methods paper) can be + added with a single ``@registry.register("my_model")`` decorator, without + editing a central ``dict`` — the open/closed principle in practice; +* lookups fail loudly with a helpful message listing the valid keys, instead of + silently returning ``None`` or an empty result. + +The registry is intentionally tiny and imports nothing heavy, so importing it +(and the :mod:`RiskLabAI.core` package) stays cheap. Components may be registered +*lazily* by a ``"module.path:attribute"`` string, so registering the built-in +catalogue does not import pandas/numba/torch until a component is actually +created. +""" + +from __future__ import annotations + +import importlib +import inspect +import logging +from collections.abc import Iterator +from typing import ( + Any, + Callable, +) + +logger = logging.getLogger(__name__) + +__all__ = ["Registry"] + +# A factory is anything callable that returns a component instance: usually a +# class, but a plain function works too. +Factory = Callable[..., Any] + + +class _Entry: + """Internal record for one registered component (eager or lazy).""" + + __slots__ = ("key", "_obj", "_lazy_target", "metadata") + + def __init__( + self, + key: str, + obj: Factory | None = None, + lazy_target: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + self.key = key + self._obj = obj + self._lazy_target = lazy_target + self.metadata: dict[str, Any] = dict(metadata or {}) + + @property + def is_lazy(self) -> bool: + """True if the component has not been imported/resolved yet.""" + return self._obj is None + + def resolve(self) -> Factory: + """Return the factory, importing it on first use if registered lazily.""" + if self._obj is None: + if not self._lazy_target: + raise RuntimeError( + f"Registry entry {self.key!r} has neither an object nor a " + f"lazy import target." + ) + module_path, sep, attribute = self._lazy_target.partition(":") + if not sep: + raise ValueError( + f"Lazy target {self._lazy_target!r} for {self.key!r} must be " + f"of the form 'package.module:attribute'." + ) + module = importlib.import_module(module_path) + self._obj = getattr(module, attribute) + return self._obj + + +class Registry: + """ + A name -> component factory registry with case-insensitive lookup. + + Parameters + ---------- + name : str + Human-readable name of the registry (used in error messages), + e.g. ``"bars"`` or ``"cross_validators"``. + base : type, optional + An optional expected base class / interface for registered components. + It is **not** enforced (so existing free functions and duck-typed + components keep working), but it is recorded for documentation and can + be inspected via :attr:`base`. + + Examples + -------- + >>> from RiskLabAI.core.registry import Registry + >>> animals = Registry("animals") + >>> @animals.register("dog", aliases=("doggo",)) + ... class Dog: + ... def speak(self) -> str: + ... return "woof" + >>> animals.available() + ['dog'] + >>> animals.create("DOG").speak() # case-insensitive + 'woof' + >>> animals.create("doggo").speak() # alias + 'woof' + """ + + def __init__(self, name: str, *, base: type | None = None) -> None: + self.name = name + self.base = base + # canonical key -> entry + self._entries: dict[str, _Entry] = {} + # lower-cased key OR alias -> canonical key + self._index: dict[str, str] = {} + + # ------------------------------------------------------------------ # + # Registration + # ------------------------------------------------------------------ # + def register( + self, + key: Any = None, + obj: Factory | None = None, + *, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> Any: + """ + Register a component. Usable as a decorator or as a direct call. + + Decorator forms:: + + @reg.register("my_model") + class MyModel: ... + + @reg.register # key inferred from class name + class MyModel: ... + + Direct form:: + + reg.register("my_model", MyModel) + + Parameters + ---------- + key : str or callable, optional + The lookup key. When used as a bare decorator (``@reg.register``) + this is the class/function being decorated and the key is taken from + its ``__name__``. + obj : callable, optional + The factory (class or function). If given, this is a direct + registration and ``obj`` is returned unchanged. + aliases : tuple of str, optional + Additional names that resolve to the same component. + metadata : dict, optional + Arbitrary metadata stored alongside the entry (e.g. a description, + an AFML page reference, the family it belongs to). + override : bool, default False + Allow replacing an existing key. Otherwise a duplicate raises + :class:`KeyError`, which protects against accidental shadowing. + + Returns + ------- + The decorated object (decorator forms) or ``obj`` (direct form). + """ + # Bare decorator: @reg.register (key is the class/function itself) + if obj is None and callable(key) and not isinstance(key, str): + target = key + self._register( + target.__name__, + target, + aliases=aliases, + metadata=metadata, + override=override, + ) + return target + + # Direct call: reg.register("name", obj) + if obj is not None: + self._register( + key, + obj, + aliases=aliases, + metadata=metadata, + override=override, + ) + return obj + + # Decorator factory: @reg.register("name") + def decorator(target: Factory) -> Factory: + resolved_key = key if isinstance(key, str) else target.__name__ + self._register( + resolved_key, + target, + aliases=aliases, + metadata=metadata, + override=override, + ) + return target + + return decorator + + def register_lazy( + self, + key: str, + target: str, + *, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> None: + """ + Register a component by import path, deferring the import to first use. + + Parameters + ---------- + key : str + The lookup key. + target : str + Import path of the form ``"package.module:attribute"``. + aliases, metadata, override + See :meth:`register`. + """ + self._register( + key, + None, + lazy_target=target, + aliases=aliases, + metadata=metadata, + override=override, + ) + + def _register( + self, + key: str, + obj: Factory | None, + *, + lazy_target: str | None = None, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> None: + if not isinstance(key, str) or not key: + raise TypeError(f"Registry key must be a non-empty string, got {key!r}.") + lower = key.lower() + if lower in self._index and not override: + existing = self._index[lower] + raise KeyError( + f"{self.name!r} registry already has a component under " + f"{key!r} (canonical key {existing!r}). " + f"Pass override=True to replace it." + ) + # On override, drop any stale aliases pointing at the old canonical key. + if override and lower in self._index: + old_canonical = self._index[lower] + self._entries.pop(old_canonical, None) + self._index = {k: v for k, v in self._index.items() if v != old_canonical} + + self._entries[key] = _Entry( + key, + obj=obj, + lazy_target=lazy_target, + metadata=metadata, + ) + self._index[lower] = key + for alias in aliases: + if not isinstance(alias, str) or not alias: + raise TypeError(f"Alias must be a non-empty string, got {alias!r}.") + self._index[alias.lower()] = key + logger.debug("Registered %r in %r registry.", key, self.name) + + def unregister(self, key: str) -> None: + """Remove a component and all of its aliases. Raises if absent.""" + entry = self._lookup(key) + canonical = entry.key + self._entries.pop(canonical, None) + self._index = {k: v for k, v in self._index.items() if v != canonical} + + # ------------------------------------------------------------------ # + # Lookup & construction + # ------------------------------------------------------------------ # + def get(self, key: str) -> Factory: + """Return the registered factory (class/callable) for ``key``.""" + return self._lookup(key).resolve() + + def create( + self, + key: str, + *args: Any, + filter_unknown_kwargs: bool = False, + **kwargs: Any, + ) -> Any: + """ + Instantiate the component registered under ``key``. + + Parameters + ---------- + key : str + The lookup key (case-insensitive; aliases accepted). + *args, **kwargs + Forwarded to the component's constructor. + filter_unknown_kwargs : bool, default False + If True, silently drop keyword arguments the constructor does not + accept (unless it accepts ``**kwargs``). This mirrors the behaviour + of the existing cross-validator / feature-importance factories, + which let callers pass a shared bag of options. Off by default so + that typos surface as clear ``TypeError``s. + """ + factory = self.get(key) + if filter_unknown_kwargs: + kwargs = _filter_kwargs(factory, kwargs) + return factory(*args, **kwargs) + + def metadata(self, key: str) -> dict[str, Any]: + """Return the metadata dict registered alongside ``key``.""" + return dict(self._lookup(key).metadata) + + def is_lazy(self, key: str) -> bool: + """True if ``key`` is registered lazily and not yet imported.""" + return self._lookup(key).is_lazy + + def _lookup(self, key: str) -> _Entry: + try: + canonical = self._index[key.lower()] + except AttributeError: + raise TypeError( + f"Registry key must be a string, got {type(key).__name__}." + ) from None + except KeyError: + raise KeyError(self._not_found_message(key)) from None + return self._entries[canonical] + + def _not_found_message(self, key: str) -> str: + available = self.available() + listing = ", ".join(repr(k) for k in available) if available else "(none)" + return ( + f"{key!r} is not registered in the {self.name!r} registry. " + f"Available components: {listing}." + ) + + # ------------------------------------------------------------------ # + # Introspection / mapping protocol + # ------------------------------------------------------------------ # + def available(self) -> list[str]: + """Sorted list of canonical keys.""" + return sorted(self._entries.keys()) + + def keys(self) -> list[str]: + """Alias for :meth:`available` (mapping-like access).""" + return self.available() + + def aliases(self) -> dict[str, str]: + """Mapping of alias -> canonical key (excludes canonical keys themselves).""" + return { + alias: canonical + for alias, canonical in self._index.items() + if alias != canonical.lower() + } + + def __contains__(self, key: object) -> bool: + return isinstance(key, str) and key.lower() in self._index + + def __getitem__(self, key: str) -> Factory: + return self.get(key) + + def __iter__(self) -> Iterator[str]: + return iter(self.available()) + + def __len__(self) -> int: + return len(self._entries) + + def __repr__(self) -> str: # pragma: no cover - cosmetic + return f"" + + +def _filter_kwargs(factory: Factory, kwargs: dict[str, Any]) -> dict[str, Any]: + """ + Drop keyword arguments the factory's signature does not accept. + + If the signature contains ``**kwargs`` (VAR_KEYWORD), everything is kept. + """ + target = factory.__init__ if inspect.isclass(factory) else factory + try: + signature = inspect.signature(target) + except (TypeError, ValueError): # pragma: no cover - builtins without sig + return kwargs + params = signature.parameters.values() + if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params): + return kwargs + accepted = {p.name for p in params} + return {k: v for k, v in kwargs.items() if k in accepted} diff --git a/src/RiskLabAI/data/__init__.py b/src/RiskLabAI/data/__init__.py new file mode 100644 index 0000000..07cd3e3 --- /dev/null +++ b/src/RiskLabAI/data/__init__.py @@ -0,0 +1,34 @@ +""" +RiskLabAI Data Module + +This package contains all modules related to financial data processing, +including: +- Bar data structures (standard, time, imbalance, run) +- Labeling (triple-barrier, trend-scanning) +- Fractional differentiation +- Covariance matrix denoising +- Distance metrics +- Sample weighting +- Synthetic data generation +""" + +# Import sub-packages +from . import ( + denoise, + differentiation, + distance, + labeling, + structures, + synthetic_data, + weights, +) + +__all__ = [ + "denoise", + "differentiation", + "distance", + "labeling", + "structures", + "synthetic_data", + "weights", +] diff --git a/src/RiskLabAI/data/denoise/__init__.py b/src/RiskLabAI/data/denoise/__init__.py new file mode 100644 index 0000000..3506d53 --- /dev/null +++ b/src/RiskLabAI/data/denoise/__init__.py @@ -0,0 +1,40 @@ +""" +RiskLabAI Data Denoising Module + +Implements noise reduction techniques for covariance and correlation +matrices, based on Random Matrix Theory (RMT) and the +Marcenko-Pastur distribution. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 2. +""" + +from .denoising import ( + corr_to_cov, + cov_to_corr, + denoise_cov, + denoised_corr, + denoised_corr2, + find_max_eval, + fit_kde, + marcenko_pastur_pdf, + optimal_portfolio, + pca, +) +from .nercome import nercome_denoised_covariance + +__all__ = [ + "marcenko_pastur_pdf", + "fit_kde", + "find_max_eval", + "pca", + "denoised_corr", + "denoised_corr2", + "cov_to_corr", + "corr_to_cov", + "denoise_cov", + "optimal_portfolio", + # NERCOME (Appraisal 24 admit) + "nercome_denoised_covariance", +] diff --git a/src/RiskLabAI/data/denoise/denoising.py b/src/RiskLabAI/data/denoise/denoising.py new file mode 100644 index 0000000..61646e8 --- /dev/null +++ b/src/RiskLabAI/data/denoise/denoising.py @@ -0,0 +1,474 @@ +""" +Implements covariance matrix denoising using Random Matrix Theory (RMT). + +This module provides functions to: +1. Fit the Marcenko-Pastur distribution to the eigenvalues of a + correlation matrix. +2. Identify and remove eigenvalues associated with noise. +3. Reconstruct a "denoised" correlation and covariance matrix. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 2. +""" + +from numbers import Integral, Real +from typing import Optional + +import numpy as np +import pandas as pd +from scipy.optimize import minimize +from sklearn.neighbors import KernelDensity + +# --- FIX 5: Removed unused imports for LedoitWolf and block_diag --- + + +def marcenko_pastur_pdf(variance: float, q: float, num_points: int = 1000) -> pd.Series: + r""" + Compute the Marcenko-Pastur (MP) probability density function. + + This function defines the theoretical distribution of eigenvalues + for a random covariance matrix. + + .. math:: + f(\lambda) = \frac{q}{2\pi\sigma^2\lambda} \sqrt{(\lambda_{max} - \lambda) + (\lambda - \lambda_{min})} + + Parameters + ---------- + variance : float + Variance of the observations (\(\sigma^2\)). + q : float + Ratio T/N, where T is observations and N is features. + num_points : int, default=1000 + Number of points in the PDF. + + Returns + ------- + pd.Series + The Marcenko-Pastur PDF, indexed by eigenvalues (\(\lambda\)). + """ + lambda_min = variance * (1 - (1.0 / q) ** 0.5) ** 2 + lambda_max = variance * (1 + (1.0 / q) ** 0.5) ** 2 + + # --- FIX 1: Add epsilon to prevent division by zero if lambda_min=0 (when q=1) --- + e_min = max(lambda_min, 1e-10) + eigenvalues = np.linspace(e_min, lambda_max, num_points) + + pdf = (q / (2 * np.pi * variance * eigenvalues)) * ( + (lambda_max - eigenvalues) * (eigenvalues - lambda_min) + ) ** 0.5 + + # Set PDF to 0 where eigenvalues are outside the valid range (e.g., due to numerical precision) + pdf[np.isnan(pdf)] = 0 + + return pd.Series(pdf.flatten(), index=eigenvalues.flatten()) + + +def fit_kde( + observations: np.ndarray, + bandwidth: float = 0.01, + kernel: str = "gaussian", +) -> KernelDensity: + """ + Fit a Kernel Density Estimator (KDE) to observations. + + Parameters + ---------- + observations : np.ndarray + The observed data (e.g., eigenvalues). + bandwidth : float, default=0.01 + The bandwidth for the kernel. + kernel : str, default="gaussian" + The kernel to use. + + Returns + ------- + KernelDensity + A fitted `sklearn.neighbors.KernelDensity` object. + """ + observations = observations.reshape(-1, 1) + kde = KernelDensity(kernel=kernel, bandwidth=bandwidth).fit(observations) + return kde + + +def _mp_pdf_fit_error( + variance: float, q: float, eigenvalues: np.ndarray, bandwidth: float +) -> float: # <-- FIX 2: Added bandwidth + r""" + Error function for fitting the MP PDF to observed eigenvalues. + + Calculates the sum of squared errors between the theoretical + MP PDF and the empirical PDF (from KDE). + + Parameters + ---------- + variance : float + The \(\sigma^2\) parameter to test. + q : float + The T/N ratio. + eigenvalues : np.ndarray + The observed eigenvalues. + bandwidth : float + The KDE bandwidth. + + Returns + ------- + float + The sum of squared errors. + """ + # Ensure eigenvalues is 1D for PDF generation + if eigenvalues.ndim == 2: + eigenvalues = np.diag(eigenvalues) + + theoretical_pdf = marcenko_pastur_pdf(variance, q, num_points=eigenvalues.shape[0]) + + # Fit empirical PDF + # --- FIX 2: Pass bandwidth to fit_kde --- + kde = fit_kde(eigenvalues, bandwidth=bandwidth) + empirical_pdf = np.exp( + kde.score_samples(theoretical_pdf.index.values.reshape(-1, 1)) + ) + + # Calculate SSE + sse = np.sum((empirical_pdf - theoretical_pdf.values) ** 2) + return sse + + +def find_max_eval( + eigenvalues: np.ndarray, q: float, bandwidth: float +) -> tuple[float, float]: + r""" + Find the maximum theoretical eigenvalue (\(\lambda_{max}\)) + by fitting the Marcenko-Pastur distribution. + + Parameters + ---------- + eigenvalues : np.ndarray + The diagonal matrix (or 1D vector) of observed eigenvalues. + q : float + The T/N ratio. + bandwidth : float + The KDE bandwidth. + + Returns + ------- + Tuple[float, float] + - lambda_max: The maximum theoretical eigenvalue. + - variance: The fitted variance (\(\sigma^2\)). + """ + # --- FIX 3: Ensure we have a 1D array for fitting --- + if eigenvalues.ndim == 2: + eigenvalues_1d = np.diag(eigenvalues) + else: + eigenvalues_1d = eigenvalues + + # Minimize the SSE to find the best-fit variance + # --- FIX 2: Pass bandwidth to the objective function --- + def objective_func(*args): + return _mp_pdf_fit_error(args[0], q, eigenvalues_1d, bandwidth) + + optimizer_result = minimize( + objective_func, + x0=np.array([0.5]), # Initial variance guess + bounds=((1e-5, 1 - 1e-5),), + ) + + if optimizer_result.success: + variance = optimizer_result.x[0] + else: + variance = 1.0 # Fallback + + # Calculate lambda_max based on the fitted variance + lambda_max = variance * (1 + (1.0 / q) ** 0.5) ** 2 + return lambda_max, variance + + +def denoised_corr( + eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int +) -> np.ndarray: + """ + Reconstruct the correlation matrix using only the eigenvalues + associated with signal (i.e., > lambda_max). + + Note: Assumes eigenvalues are sorted in descending order. + + Parameters + ---------- + eigenvalues : np.ndarray + The diagonal matrix of *all* eigenvalues (sorted descending). + eigenvectors : np.ndarray + The matrix of eigenvectors (corresponding to descending eigenvalues). + num_facts : int + The number of factors (signal eigenvalues) to keep. + + Returns + ------- + np.ndarray + The denoised correlation matrix. + """ + # 1. Get the eigenvalues and eigenvectors for signal + # --- FIX 3: This logic is now correct as eigenvalues are descending --- + eigenvalues_1d = np.diag(eigenvalues) + eigenvalues_signal = np.diag(eigenvalues_1d[:num_facts]) + + eigenvectors_signal = eigenvectors[:, :num_facts] + + # 2. Reconstruct the signal-only correlation matrix + corr1 = eigenvectors_signal @ eigenvalues_signal @ eigenvectors_signal.T + + # 3. Get the eigenvalues for noise and average them + if num_facts < eigenvalues.shape[0]: + # --- FIX 3: Correctly averages the smaller (noise) eigenvalues --- + avg_noise_eigenvalue = eigenvalues_1d[num_facts:].mean() + eigenvectors_noise = eigenvectors[:, num_facts:] + + # 4. Reconstruct the noise-only correlation matrix + corr2 = ( + eigenvectors_noise + @ (np.diag([avg_noise_eigenvalue] * (eigenvalues.shape[0] - num_facts))) + @ eigenvectors_noise.T + ) + + # 5. Add them back together + corr1 = corr1 + corr2 + + # 6. Rescale to be a valid correlation matrix + diag_inv_sqrt = 1.0 / np.sqrt(np.diag(corr1)) + corr1 = np.diag(diag_inv_sqrt) @ corr1 @ np.diag(diag_inv_sqrt) + np.fill_diagonal(corr1, 1.0) # Clean up numerical errors + return corr1 + + +def denoised_corr2( + eigenvalues: np.ndarray, + eigenvectors: np.ndarray, + num_factors: int, + alpha: float = 0.0, +) -> np.ndarray: + r"""Denoise a correlation matrix by targeted eigenvector shrinkage. + + The leading ``num_factors`` eigenpairs are retained as signal. The + remaining eigenpairs are treated as noise, and their off-diagonal + contribution is multiplied by ``alpha`` while their diagonal contribution + is preserved. Thus ``alpha=0`` applies full targeted shrinkage and + ``alpha=1`` reconstructs the unshrunk matrix. + + This is the targeted-shrinkage construction in López de Prado (2020), + *Machine Learning for Asset Managers*, Section 2.5.2, Code Snippet 2.6. + + Parameters + ---------- + eigenvalues : np.ndarray + Square diagonal matrix of eigenvalues, ordered consistently with + ``eigenvectors`` and conventionally from largest to smallest. + eigenvectors : np.ndarray + Square matrix whose columns are the corresponding eigenvectors. + num_factors : int + Number of leading eigenpairs classified as signal. Values from zero + through the matrix dimension are allowed. + alpha : float, default=0.0 + Noise shrinkage intensity in the closed interval ``[0, 1]``. + + Returns + ------- + np.ndarray + The targeted-shrinkage matrix. + + Raises + ------ + TypeError + If the arrays are not real numeric arrays, ``num_factors`` is not an + integer, or ``alpha`` is not real. + ValueError + If the arrays are empty or incompatible, the eigenvalue matrix is not + diagonal, an input is non-finite, or a parameter lies outside its + permitted range. + """ + eigenvalues_array = np.asarray(eigenvalues) + eigenvectors_array = np.asarray(eigenvectors) + + if ( + eigenvalues_array.ndim != 2 + or eigenvalues_array.shape[0] != eigenvalues_array.shape[1] + ): + raise ValueError("eigenvalues must be a non-empty square diagonal matrix") + if eigenvalues_array.shape[0] == 0: + raise ValueError("eigenvalues must be a non-empty square diagonal matrix") + if eigenvectors_array.shape != eigenvalues_array.shape: + raise ValueError("eigenvectors must be square and match eigenvalues") + + for name, array in ( + ("eigenvalues", eigenvalues_array), + ("eigenvectors", eigenvectors_array), + ): + if not np.issubdtype(array.dtype, np.number) or np.iscomplexobj(array): + raise TypeError(f"{name} must be a real numeric array") + if not np.all(np.isfinite(array)): + raise ValueError(f"{name} must contain only finite values") + + diagonal = np.diag(np.diag(eigenvalues_array)) + if not np.allclose(eigenvalues_array, diagonal, rtol=0.0, atol=1e-12): + raise ValueError("eigenvalues must be a diagonal matrix") + + if isinstance(num_factors, (bool, np.bool_)) or not isinstance( + num_factors, Integral + ): + raise TypeError("num_factors must be an integer") + num_factors = int(num_factors) + dimension = eigenvalues_array.shape[0] + if not 0 <= num_factors <= dimension: + raise ValueError("num_factors must be between zero and the matrix dimension") + + if isinstance(alpha, (bool, np.bool_)) or not isinstance(alpha, Real): + raise TypeError("alpha must be a real number") + alpha = float(alpha) + if not np.isfinite(alpha) or not 0.0 <= alpha <= 1.0: + raise ValueError("alpha must be finite and between zero and one") + + signal_values = eigenvalues_array[:num_factors, :num_factors] + signal_vectors = eigenvectors_array[:, :num_factors] + noise_values = eigenvalues_array[num_factors:, num_factors:] + noise_vectors = eigenvectors_array[:, num_factors:] + + signal = signal_vectors @ signal_values @ signal_vectors.T + noise = noise_vectors @ noise_values @ noise_vectors.T + return signal + alpha * noise + (1.0 - alpha) * np.diag(np.diag(noise)) + + +# --- Utility Functions --- + + +def pca(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """ + Computes the principal component analysis of a Hermitian matrix. + Ensures eigenvalues are sorted descending. + + :param matrix: Hermitian matrix (e.g., correlation matrix) + :type matrix: np.ndarray + :return: (eigenvalues_vector, eigenvectors_matrix) + :rtype: Tuple[np.ndarray, np.ndarray] + """ + eigenvalues, eigenvectors = np.linalg.eigh(matrix) + indices = eigenvalues.argsort()[::-1] # Sort descending + eigenvalues = eigenvalues[indices] + eigenvectors = eigenvectors[:, indices] + return eigenvalues, eigenvectors + + +def cov_to_corr(cov: np.ndarray) -> np.ndarray: + """Convert covariance matrix to correlation matrix.""" + std = np.sqrt(np.diag(cov)) + # Handle division by zero if any std is 0 + std[std == 0] = 1.0 + corr = cov / np.outer(std, std) + corr[corr < -1] = -1.0 # Handle numerical errors + corr[corr > 1] = 1.0 + np.fill_diagonal(corr, 1.0) # Ensure diagonal is 1 + return corr + + +def corr_to_cov(corr: np.ndarray, std: np.ndarray) -> np.ndarray: + """Convert correlation matrix to covariance matrix.""" + return corr * np.outer(std, std) + + +def denoise_cov(cov0: np.ndarray, q: float, bandwidth: float = 0.01) -> np.ndarray: + """ + De-noises a covariance matrix. + + Parameters + ---------- + cov0 : np.ndarray + The original (noisy) covariance matrix. + q : float + The T/N ratio. + bandwidth : float, default=0.01 + The KDE bandwidth. + + Returns + ------- + np.ndarray + The de-noised covariance matrix. + """ + + corr0 = cov_to_corr(cov0) + + # --- FIX 3: Use pca helper to get DESCENDING eigenvalues/vectors --- + eigenvalues, eigenvectors = pca(corr0) + eigenvalues_diag = np.diag(eigenvalues) # 2D diag matrix (desc) + + # Find the noise cutoff + # --- FIX 2: Pass bandwidth down to find_max_eval --- + emax0, var0 = find_max_eval(eigenvalues_diag, q, bandwidth) + + # --- FIX 3: Correctly find num factors as count of evals > emax0 --- + n_facts0 = np.sum(eigenvalues > emax0) + + # Denoise the correlation matrix + corr1 = denoised_corr(eigenvalues_diag, eigenvectors, n_facts0) + + # Convert back to covariance + cov1 = corr_to_cov(corr1, np.diag(cov0) ** 0.5) + return cov1 + + +def optimal_portfolio(cov: np.ndarray, mu: Optional[np.ndarray] = None) -> np.ndarray: + """ + Compute the optimal (e.g., minimum variance) portfolio weights. + + (Note: This is duplicated in `optimization/nco.py`) + + Parameters + ---------- + cov : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. If None, computes GMV portfolio. + + Returns + ------- + np.ndarray + The optimal portfolio weights. + """ + inv_cov = np.linalg.inv(cov) + ones = np.ones(shape=(inv_cov.shape[0], 1)) + + if mu is None: + mu = ones + + w = inv_cov @ mu + w /= ones.T @ w + return w.flatten() + + +def optimal_portfolio_denoised( + cov: np.ndarray, + q: float, + mu: Optional[np.ndarray] = None, + bandwidth: float = 0.01, +) -> np.ndarray: + """ + Compute the optimal portfolio weights from a denoised covariance matrix. + + Parameters + ---------- + cov : np.ndarray + The *original* (noisy) covariance matrix. + q : float + The T/N ratio. + mu : np.ndarray, optional + Vector of expected returns. + bandwidth : float, default=0.01 + The KDE bandwidth. + + Returns + ------- + np.ndarray + The optimal, denoised portfolio weights. + """ + # --- FIX 4: Pass bandwidth to denoise_cov --- + cov_denoised = denoise_cov(cov, q, bandwidth) + + # Compute optimal portfolio on the denoised matrix + return optimal_portfolio(cov_denoised, mu) diff --git a/src/RiskLabAI/data/denoise/nercome.py b/src/RiskLabAI/data/denoise/nercome.py new file mode 100644 index 0000000..9899e68 --- /dev/null +++ b/src/RiskLabAI/data/denoise/nercome.py @@ -0,0 +1,133 @@ +r""" +NERCOME: nonparametric eigenvalue-regularized covariance estimation (Lam 2016). + +de Prado denoises the covariance by Marcenko-Pastur eigenvalue clipping (`denoising.denoise_cov`), which +assumes a clean noise bulk separated from the signal eigenvalues by a gap. When the eigenvalue spectrum +has no clean gap (a slowly-decaying bulk) or is non-stationary, that assumption breaks and clipping +degenerates toward the raw sample covariance. NERCOME (Lam 2016) instead regularizes the eigenvalues by +sample-splitting: it estimates the eigenvectors on one split of the data and the oracle eigenvalues by +projecting the held-out split's sample covariance onto those eigenvectors, averaging over many random +splits. This recovers the covariance more accurately and with better conditioning on no-gap / +non-stationary spectra, and converges to clipping where the gap assumption holds. + +Admitted in Appraisal 24 (CONTRIBUTIONS_LEDGER 2026-06-27) - portfolio's first admitted extension. Regime +tag, verbatim from the verdict: + + prefer NERCOME over MP clipping for covariance estimation when the eigenvalue spectrum has no clean + gap or is non-stationary (better accuracy and conditioning, and lower OOS volatility via NCO / + min-variance); it converges to clipping on clean-gap stationary spectra. It costs more turnover and + concentration than clipping, gives no risk-adjusted-return edge (none does - 1/N stands), and HRP + does not benefit (use it through NCO / min-variance). + +Held-out confirmed (appraisals/24_results, HELDOUT.md): on the sealed no-gap x non-stationary x T=30 +corner NERCOME has distinguishably lower covariance error (relative Frobenius 0.563 vs MP clipping 0.684) +and better conditioning (29.6 vs 46.3), with lower NCO realized risk (1.315 vs 1.553); HRP is insensitive. +Unlike `denoise_cov`, which cleans a covariance matrix, NERCOME is a data-driven (sample-splitting) +estimator, so it takes the return matrix. Evidence and caveats: appraisals/24_verdict.md. + +References +---------- +Lam, C. (2016) Nonparametric eigenvalue-regularized precision or covariance matrix estimator. + The Annals of Statistics, 44(3), 928-953. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 2 (the MP-clipping baseline). +""" + +from __future__ import annotations + +import numpy as np + +from .denoising import corr_to_cov, cov_to_corr + + +def _ensure_positive_definite(matrix: np.ndarray, eps: float = 1e-10) -> np.ndarray: + """Project a symmetric matrix to the nearest PD one by flooring its eigenvalues.""" + matrix = (matrix + matrix.T) / 2.0 + values, vectors = np.linalg.eigh(matrix) + floor = ( + max(eps, eps * float(values.max())) if values.size and values.max() > 0 else eps + ) + values = np.clip(values, floor, None) + out = (vectors * values) @ vectors.T + return (out + out.T) / 2.0 + + +def nercome_denoised_covariance( + returns: np.ndarray, + n_splits: int = 50, + split_fraction: float = 2.0 / 3.0, + random_state: int | None = None, +) -> np.ndarray: + r""" + NERCOME denoised covariance (Lam 2016), estimated by sample-splitting from a return matrix. + + For each of ``n_splits`` random row permutations the ``T`` observations are split into two halves; the + eigenvectors ``P`` come from the first half's sample covariance, and the regularized eigenvalues are + the oracle projection ``d_i = p_i' S2 p_i`` of the second half's sample covariance ``S2`` onto those + eigenvectors. The estimator ``P diag(d) P'`` is averaged over the splits (each term is PSD, so the + average is PSD). The returns are standardized to unit sample variance, NERCOME-cleaned in correlation + space, then scaled back to a covariance by the sample column standard deviations. + + Prefer NERCOME over MP eigenvalue clipping when the eigenvalue spectrum has no clean gap or is + non-stationary (better accuracy and conditioning, and lower out-of-sample volatility via NCO / + min-variance); it converges to clipping on clean-gap stationary spectra, costs more turnover and + concentration, gives no risk-adjusted-return edge, and HRP does not benefit (use it through NCO / + min-variance). See the module docstring for the full verbatim regime tag and appraisals/24_verdict.md. + Unlike :func:`denoising.denoise_cov`, this takes the return matrix, not a covariance. + + Parameters + ---------- + returns : np.ndarray + The ``T x p`` matrix of asset returns (rows = observations, columns = assets). + n_splits : int, default 50 + Number of random sample-splits averaged. + split_fraction : float, default 2/3 + Fraction ``m / T`` of rows used for the eigenvector (first) half (Lam's ~2T/3 default). + random_state : int, optional + Seed for the split permutations. + + Returns + ------- + np.ndarray + The ``p x p`` NERCOME denoised covariance (symmetric positive-definite). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> returns = rng.standard_normal((120, 8)) + >>> cov = nercome_denoised_covariance(returns, n_splits=20, random_state=0) + >>> cov.shape == (8, 8) and np.allclose(cov, cov.T) + True + """ + returns = np.asarray(returns, dtype=float) + if returns.ndim != 2: + raise ValueError("returns must be a 2-D (T x p) array") + n, p = returns.shape + if n < 4: + raise ValueError("NERCOME needs at least 4 observations") + std = returns.std(axis=0, ddof=1) + std = np.where(std <= 0, 1.0, std) + z = returns / std + + m = max(p + 1, int(round(split_fraction * n))) + m = min(m, n - 2) + rng = np.random.default_rng(random_state) + accumulated = np.zeros((p, p)) + used = 0 + for _ in range(n_splits): + order = rng.permutation(n) + first, second = order[:m], order[m:] + if len(second) < 2: + continue + cov_first = np.cov(z[first], rowvar=False) + cov_second = np.cov(z[second], rowvar=False) + _, vectors = np.linalg.eigh(cov_first) + oracle = np.einsum( + "ij,jk,ik->i", vectors.T, cov_second, vectors.T + ) # p_i' S2 p_i + oracle = np.clip(oracle, 0.0, None) + accumulated += (vectors * oracle) @ vectors.T + used += 1 + estimate = accumulated / max(used, 1) + correlation = cov_to_corr(_ensure_positive_definite(estimate)) + return corr_to_cov(correlation, std) diff --git a/src/RiskLabAI/data/differentiation/__init__.py b/src/RiskLabAI/data/differentiation/__init__.py new file mode 100644 index 0000000..877c5a7 --- /dev/null +++ b/src/RiskLabAI/data/differentiation/__init__.py @@ -0,0 +1,7 @@ +from .adaptive_differentiation import ( + adaptive_differencing_order, + adaptive_fractional_difference, + rescaled_range_hurst, + wavelet_variance_hurst, +) +from .differentiation import * diff --git a/src/RiskLabAI/data/differentiation/adaptive_differentiation.py b/src/RiskLabAI/data/differentiation/adaptive_differentiation.py new file mode 100644 index 0000000..6eeaa2c --- /dev/null +++ b/src/RiskLabAI/data/differentiation/adaptive_differentiation.py @@ -0,0 +1,268 @@ +r""" +Adaptive Fractional Differencing (AFD), a bias-corrected differencing-order estimate. + +de Prado's fixed-width fractional differentiation (`differentiation.py`) finds the minimum order d that +makes a series stationary via an ADF search. On finite samples that min-d-via-ADF is biased and +under-differences: the under-powered ADF test passes while the series still carries spurious residual +long-memory weights, so the recovered order is too small. AFD instead estimates the differencing order +directly from the data's long-memory structure, with a finite-sample bias correction, and so recovers +the genuine order far more accurately when memory is strong and the sample is finite. + +Admitted in Appraisal 13 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer AFD over fixed-width FFD when the differencing order itself must be right - strong long memory + and finite samples, where min-d-via-ADF under-differences; on weak memory the gap narrows. The + implemented AFD is a tractable clean-room approximation of the published wavelet-Hurst + ridge + + CV-truncation method (no public code). + +The real-data predictive-lift confirmation is a tracked obligation (``REAL_DATA_FOLLOWUPS.md``): the +order accuracy is established against ground truth, but the downstream predictive value of the more +accurate order is not yet decisive. Evidence and caveats: appraisals/13_verdict.md (and 13b if logged). + +This is a tractable clean-room approximation of the published AFD (IEEE Access 2025; no public code): +the differencing order is anchored at ``0.5 + d_hat`` where ``d_hat`` is a finite-sample-bias-corrected +Hurst estimate of the increments (a ridge-style equal-weight blend of a wavelet-variance and an R/S +Hurst estimate, which lowers the variance/bias of either alone), the fixed-width-FFD truncation is +chosen by cross-validation over weight thresholds, and the order is raised to the ADF stationarity +boundary if not yet stationary. ``pywt`` (PyWavelets) is an optional dependency: when present AFD uses +the wavelet + R/S Hurst blend, otherwise the R/S estimate alone (the blend is the validated default). + +References +---------- +Adaptive Fractional Differencing (2025) IEEE Access. (Clean-room approximation; no public source.) +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 5 (the FFD baseline). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from statsmodels.tsa.stattools import adfuller + +from .differentiation import fractional_difference_fixed_single + +try: # optional wavelet dependency (enables the wavelet-variance Hurst component of the blend) + import pywt + + _HAS_PYWT = True +except ( + ImportError +): # pragma: no cover - exercised only in environments without PyWavelets + _HAS_PYWT = False + +DEFAULT_DELTA_GRID = [round(i * 0.05, 3) for i in range(0, 25)] # 0.00 .. 1.20 +DEFAULT_CV_THRESHOLDS = (1e-3, 1e-4, 1e-5) +DEFAULT_ADF_SIGNIFICANCE = 0.05 +DEFAULT_ADF_MAXLAG = 1 + + +def wavelet_variance_hurst( + series: np.ndarray, wavelet: str = "db2", skip_fine: int = 1 +) -> float: + r""" + Wavelet-variance Hurst estimate: :math:`\log_2 \mathrm{Var}(\text{detail}_j) \approx (2H-1)\,j`. + + Returns ``nan`` if ``pywt`` is not installed or the series is too short to form enough octaves. + """ + if not _HAS_PYWT: + return float("nan") + x = np.asarray(series, dtype=float) + x = x - x.mean() + n = x.size + dec_len = pywt.Wavelet(wavelet).dec_len + max_level = pywt.dwt_max_level(n, dec_len) + level = max(3, min(max_level, int(np.log2(n)) - 2)) + if level < 3: + return float("nan") + coefficients = pywt.wavedec(x, wavelet, level=level) + details = coefficients[1:] # coarsest .. finest + octaves = np.arange(level, 0, -1) + variances = np.array([np.mean(d**2) for d in details]) + mask = octaves >= (1 + skip_fine) + j = octaves[mask].astype(float) + v = np.where(variances[mask] <= 0, 1e-12, variances[mask]) + if j.size < 2: + return float("nan") + slope = np.polyfit(j, np.log2(v), 1)[0] + return (slope + 1.0) / 2.0 + + +def rescaled_range_hurst(series: np.ndarray) -> float: + """Classical rescaled-range (R/S) Hurst estimate (the log-log slope of R/S against window size).""" + x = np.asarray(series, dtype=float) + n = x.size + if n < 32: + return float("nan") + sizes = np.unique( + np.floor(np.logspace(np.log10(8), np.log10(n // 2), 8)).astype(int) + ) + rs_means, used = [], [] + for s in sizes: + k = n // s + if k < 1: + continue + values = [] + for i in range(k): + segment = x[i * s : (i + 1) * s] + deviations = np.cumsum(segment - segment.mean()) + spread = deviations.max() - deviations.min() + scale = segment.std() + if scale > 0: + values.append(spread / scale) + if values: + rs_means.append(np.mean(values)) + used.append(s) + if len(used) < 2: + return float("nan") + return float(np.polyfit(np.log(used), np.log(rs_means), 1)[0]) + + +def adaptive_differencing_order(increments: np.ndarray) -> float: + r""" + Bias-corrected memory order :math:`\hat{d}` of a series of increments. + + A ridge-style equal-weight blend of the wavelet-variance and R/S Hurst estimates, + :math:`\hat{d} = \bar{H} - 0.5`, clipped to ``[0, 0.99]``. Falls back to the R/S estimate alone if + ``pywt`` is unavailable. Clean-room approximation of the published wavelet-Hurst + ridge procedure. + """ + h_wavelet = wavelet_variance_hurst(increments) + h_rs = rescaled_range_hurst(increments) + estimates = [h for h in (h_wavelet, h_rs) if np.isfinite(h)] + if not estimates: + return float("nan") + h_blend = float(np.mean(estimates)) + return float(np.clip(h_blend - 0.5, 0.0, 0.99)) + + +def _adf_pvalue(values: np.ndarray, maxlag: int) -> float: + v = np.asarray(values, dtype=float) + v = v[~np.isnan(v)] + if v.size < 20 or np.allclose(v.std(), 0.0): + return 1.0 + try: + return float(adfuller(v, maxlag=maxlag, regression="c", autolag=None)[1]) + except Exception: # pragma: no cover + return 1.0 + + +def _memory_retained(differenced: pd.Series, level: pd.Series) -> float: + common = differenced.dropna() + if len(common) < 5: + return float("nan") + level_aligned = level.reindex(common.index) + ok = (~common.isna()) & (~level_aligned.isna()) + if ok.sum() < 5 or common[ok].std() == 0 or level_aligned[ok].std() == 0: + return float("nan") + return float( + abs(np.corrcoef(common[ok].to_numpy(), level_aligned[ok].to_numpy())[0, 1]) + ) + + +def adaptive_fractional_difference( + series: pd.Series, + delta_grid: list | None = None, + cv_thresholds: tuple = DEFAULT_CV_THRESHOLDS, + adf_significance: float = DEFAULT_ADF_SIGNIFICANCE, + adf_maxlag: int = DEFAULT_ADF_MAXLAG, +) -> dict: + r""" + Adaptive Fractional Differencing of a (price/level) series. + + Estimates the memory order from the series increments with a bias-corrected Hurst blend + (:func:`adaptive_differencing_order`), anchors the differencing order at that estimate plus one + (the integration order of a price), chooses the fixed-width-FFD weight-truncation threshold by + cross-validation (the threshold that retains the most memory subject to ADF stationarity), and + raises the order along ``delta_grid`` only as far as the ADF stationarity boundary if needed. + + Prefer AFD over fixed-width FFD when the differencing order itself must be right (strong long memory, + finite samples), where de Prado's min-d-via-ADF under-differences; on weak memory the gap narrows. + See the module docstring for the full regime tag, the clean-room-approximation note, the tracked + real-data follow-up, and appraisals/13_verdict.md. + + Parameters + ---------- + series : pd.Series + The price / level series to difference. + delta_grid : list, optional + Ascending grid of candidate orders. Defaults to 0.00 .. 1.20 in steps of 0.05. + cv_thresholds : tuple, default (1e-3, 1e-4, 1e-5) + The FFD weight-truncation thresholds searched by cross-validation. + adf_significance : float, default 0.05 + The ADF p-value below which the series is judged stationary. + adf_maxlag : int, default 1 + The ADF lag (matches the repo's fixed-width FFD search). + + Returns + ------- + dict + ``order`` (the differencing order applied), ``d_hat`` (the bias-corrected memory order of the + increments), ``adf_pvalue``, ``memory_retained`` (abs correlation of the differenced series with + the level), ``threshold`` (the chosen truncation), and ``series`` (the differenced pd.Series). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> price = pd.Series(np.cumsum(rng.standard_normal(500))) # an integrated series + >>> result = adaptive_fractional_difference(price) + >>> 0.0 <= result["order"] <= 2.0 and result["adf_pvalue"] <= 1.0 + True + """ + if delta_grid is None: + delta_grid = DEFAULT_DELTA_GRID + level = series.dropna() + increments = level.diff().dropna().to_numpy() + d_hat_increment = adaptive_differencing_order(increments) + if not np.isfinite(d_hat_increment): + return { + "order": float("nan"), + "d_hat": float("nan"), + "adf_pvalue": 1.0, + "memory_retained": float("nan"), + "threshold": float("nan"), + "series": pd.Series(dtype=float), + } + order_anchor = float(np.clip(0.5 + d_hat_increment, 0.0, max(delta_grid))) + + # CV truncation: the threshold with max retained memory among stationary diffs at the anchor order. + best_threshold, best = cv_thresholds[1], None + for threshold in cv_thresholds: + diff = fractional_difference_fixed_single( + series, order_anchor, threshold + ).dropna() + if len(diff) < 20: + continue + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) + memory = _memory_retained(diff, series) + candidate = ( + pvalue < adf_significance, + memory if np.isfinite(memory) else -1.0, + threshold, + pvalue, + diff, + ) + if best is None or (candidate[0], candidate[1]) > (best[0], best[1]): + best = candidate + if best is not None: + best_threshold = best[2] + + # Anchor at the order estimate; raise it upward only to reach the stationarity boundary. + order = order_anchor + diff = fractional_difference_fixed_single(series, order, best_threshold).dropna() + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) if len(diff) >= 20 else 1.0 + grid_up = [g for g in delta_grid if g > order_anchor] + while pvalue >= adf_significance and grid_up: + order = grid_up.pop(0) + diff = fractional_difference_fixed_single( + series, order, best_threshold + ).dropna() + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) if len(diff) >= 20 else 1.0 + + return { + "order": float(order), + "d_hat": float(d_hat_increment), + "adf_pvalue": float(pvalue), + "memory_retained": _memory_retained(diff, series), + "threshold": float(best_threshold), + "series": diff, + } diff --git a/src/RiskLabAI/data/differentiation/differentiation.py b/src/RiskLabAI/data/differentiation/differentiation.py new file mode 100644 index 0000000..8a7a049 --- /dev/null +++ b/src/RiskLabAI/data/differentiation/differentiation.py @@ -0,0 +1,399 @@ +""" +Implements fractional differentiation of time series, including both +expanding-window (standard) and fixed-width-window (FFD) methods. + +Includes functions to find the minimum differentiation factor 'd' that +results in a stationary series. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 5. +""" + +from typing import TYPE_CHECKING, Optional + +import numpy as np +import pandas as pd +from statsmodels.tsa.stattools import adfuller + +if TYPE_CHECKING: + import matplotlib.pyplot as plt # type hints only; optional at runtime + + +def calculate_weights_std(degree: float, size: int) -> np.ndarray: + """ + Compute weights for standard (expanding window) fractional differentiation. + + Reference: + Snippet 5.2, Page 82. + + Parameters + ---------- + degree : float + The degree of differentiation (d). + size : int + The number of weights to generate (e.g., the series length). + + Returns + ------- + np.ndarray + An array of weights, ordered for dot product (w_0 at the end). + """ + weights = [1.0] + for k in range(1, size): + weight = -weights[-1] / k * (degree - k + 1) + weights.append(weight) + + # Reverse for dot product: [w_k, ..., w_1, w_0] + return np.array(weights[::-1]).reshape(-1, 1) + + +def calculate_weights_ffd(degree: float, threshold: float = 1e-5) -> np.ndarray: + """ + Compute weights for Fixed-Width Window Fractional Differentiation (FFD). + + The weights are generated until they fall below the `threshold`. + + Reference: + Snippet 5.3, Page 83. + + Parameters + ---------- + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + The minimum absolute weight to include. + + Returns + ------- + np.ndarray + An array of weights, ordered for dot product (w_0 at the end). + """ + weights = [1.0] + k = 1 + while True: + weight = -weights[-1] / k * (degree - k + 1) + if abs(weight) < threshold: + break + weights.append(weight) + k += 1 + + # Reverse for dot product: [w_k, ..., w_1, w_0] + return np.array(weights[::-1]).reshape(-1, 1) + + +def fractional_difference_std( + series: pd.DataFrame, degree: float, threshold: float = 0.01 +) -> pd.DataFrame: + """ + Compute the standard (expanding window) fractionally differentiated series. + + This method uses all available past data for each calculation. + + Reference: + Snippet 5.2, Page 82. + + Parameters + ---------- + series : pd.DataFrame + DataFrame of time series (one column per series). + degree : float + The degree of differentiation (d). + threshold : float, default=0.01 + The weight-loss threshold to determine the minimum number + of observations required (warm-up period). + + Returns + ------- + pd.DataFrame + DataFrame of fractionally differentiated series. + """ + # 1. Compute weights for the full series + weights = calculate_weights_std(degree, series.shape[0]) + + # 2. Determine warm-up period + weights_cumsum_abs = np.cumsum(np.abs(weights)) + weights_cumsum_abs /= weights_cumsum_abs[-1] + skip = np.searchsorted(weights_cumsum_abs, threshold) + + result_df = pd.DataFrame(index=series.index, columns=series.columns, dtype=float) + + # The expanding-window value result[t] = sum_j w_j * x[t - j] is exactly the + # first len(x) terms of a causal convolution of the series with the weights + # in natural order [w_0, w_1, ...]. Using np.convolve replaces the O(n^2) + # per-row growing-window dot product with a single O(n*k) pass per column, + # producing identical values (verified in test/test_performance.py). + weights_natural = weights[::-1].flatten() + + for name in series.columns: + # Use .ffill() - fillna(method=) is deprecated + series_ffill = series[[name]].ffill().dropna() + n_obs = series_ffill.shape[0] + if series_ffill.empty or n_obs < skip: + continue + + series_np = series_ffill.to_numpy().reshape(-1) + convolved = np.convolve(series_np, weights_natural[:n_obs])[:n_obs] + result_df.loc[series_ffill.index[skip:], name] = convolved[skip:] + + return result_df.dropna(how="all") + + +def fractional_difference_fixed( + series: pd.DataFrame, degree: float, threshold: float = 1e-5 +) -> pd.DataFrame: + """ + Compute the Fixed-Width Window (FFD) fractionally differentiated series. + + This method iterates over columns and applies a fast, convolved + FFD calculation to each one. + + Reference: + Snippet 5.3, Page 83. + + Parameters + ---------- + series : pd.DataFrame + DataFrame of time series (one column per series). + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + Threshold to determine the fixed window width. + + Returns + ------- + pd.DataFrame + DataFrame of fractionally differentiated series. + """ + result_df = pd.DataFrame(index=series.index) + + for name in series.columns: + result_df[name] = fractional_difference_fixed_single( + series[name], degree, threshold + ) + + return result_df.dropna(how="all") + + +def fractional_difference_fixed_single( + series: pd.Series, degree: float, threshold: float = 1e-5 +) -> pd.Series: + """ + Compute the FFD series for a single `pd.Series` using np.convolve. + + Parameters + ---------- + series : pd.Series + Time series. + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + Threshold to determine the fixed window width. + + Returns + ------- + pd.Series + The fractionally differentiated series. + """ + + # 1. Compute weights + # Reverse weights: calculate_weights_ffd returns [w_k, ..., w_0] + weights = calculate_weights_ffd(degree, threshold).flatten()[::-1] + width = len(weights) + + # 2. Prepare data (drop leading NaNs) + series_ffill = series.ffill().dropna() + + if series_ffill.empty or series_ffill.shape[0] < width: + # Not enough data to convolve + return pd.Series(index=series.index, dtype="float64") + + series_np = series_ffill.to_numpy() + + # 3. Apply convolution + # 'valid' mode computes the dot product only where the + # window fully overlaps the series. + convolved_vals = np.convolve(series_np, weights, mode="valid") + + # 4. Create result series on the valid index + # The result aligns with the *end* of the window + result_index = series_ffill.index[width - 1 :] + valid_results = pd.Series(convolved_vals, index=result_index) + + # 5. Reindex to original full index + # This correctly places NaNs at the start (warm-up) + # and in any gaps that were in the original series. + return valid_results.reindex(series.index) + + +def plot_weights( + degree_range: tuple[float, float], + number_degrees: int, + size: int, + ax: Optional["plt.Axes"] = None, +) -> "plt.Axes": + """ + Plot the weights of fractionally differentiated series for various degrees. + + Parameters + ---------- + degree_range : Tuple[float, float] + (min_degree, max_degree) to plot. + number_degrees : int + Number of 'd' values to plot in the range. + size : int + Length of the time series (number of weights). + ax : plt.Axes, optional + Matplotlib axes to plot on. If None, a new figure/axes is created. + + Returns + ------- + plt.Axes + The axes object with the plot. + """ + import matplotlib.pyplot as plt # optional dependency: RiskLabAI[plot] + + if ax is None: + fig, ax = plt.subplots(figsize=(10, 6)) + + weights_df = pd.DataFrame() + for degree in np.linspace(degree_range[0], degree_range[1], number_degrees): + degree = round(degree, 2) + weights = calculate_weights_std(degree, size) + weights_df[degree] = pd.Series(weights.flatten(), index=range(size - 1, -1, -1)) + + weights_df.plot(ax=ax) + ax.set_xlabel("Lag") + ax.set_ylabel("Weight") + ax.set_title("Fractional Differentiation Weights") + ax.legend(title="Degree (d)") + return ax + + +def find_optimal_ffd_simple( + input_series: pd.DataFrame, p_value_threshold: float = 0.05 +) -> pd.DataFrame: + """ + Find the minimum 'd' that passes the ADF test, for a range of d. + + This function tests `d` in 11 steps from 0 to 1. + + Reference: + Snippet 5.4, Page 85. + + Parameters + ---------- + input_series : pd.DataFrame + DataFrame containing a 'close' column. + p_value_threshold : float, default=0.05 + The ADF test p-value to pass. + + Returns + ------- + pd.DataFrame + DataFrame of ADF test results for each 'd'. + """ + results_list = [] + + # Resample to daily to ensure consistent lags + series_daily = np.log(input_series[["close"]]).resample("1D").last().dropna() + + for d in np.linspace(0, 1, 11): + differentiated = fractional_difference_fixed( + series_daily, d, threshold=0.01 + ).dropna() + + if differentiated.empty: + continue + + corr = np.corrcoef( + series_daily.loc[differentiated.index, "close"], differentiated["close"] + )[0, 1] + + try: + adf_result = adfuller( + differentiated["close"], maxlag=1, regression="c", autolag=None + ) + + results_list.append( + { + "d": d, + "adfStat": adf_result[0], + "pVal": adf_result[1], + "lags": adf_result[2], + "nObs": adf_result[3], + "95% conf": adf_result[4]["5%"], + "corr": corr, + } + ) + except Exception: + # Handle cases where ADF test fails (e.g., insufficient data) + continue + + if not results_list: + return pd.DataFrame() + + return pd.DataFrame(results_list).set_index("d") + + +def fractionally_differentiated_log_price( + input_series: pd.Series, + threshold: float = 1e-5, + step: float = 0.01, + p_value_threshold: float = 0.05, +) -> pd.Series: + """ + Find the minimum 'd' that makes a log-price series stationary. + + This function iteratively increases 'd' by 'step' until the ADF test + p-value falls below `p_value_threshold`. + + Parameters + ---------- + input_series : pd.Series + Time series of prices. + threshold : float, default=1e-5 + The weight threshold for FFD. + step : float, default=0.01 + The increment for testing 'd'. + p_value_threshold : float, default=0.05 + The significance level for the ADF test. + + Returns + ------- + pd.Series + The fractionally differentiated series with the minimum + stationary-passing 'd'. + """ + log_price = np.log(input_series) + degree = 0.0 + p_value = 1.0 + + differentiated_series = None + + while p_value > p_value_threshold: + degree += step + if degree > 2.0: # Safety break + raise ValueError("Failed to find stationary 'd' < 2.0") + + differentiated = fractional_difference_fixed_single( + log_price, degree, threshold=threshold + ).dropna() + + if differentiated.empty: + continue # Not enough data for this 'd' + + try: + adf_test = adfuller(differentiated, maxlag=1, regression="c", autolag=None) + p_value = adf_test[1] + except Exception: + p_value = 1.0 # Failed test, keep going + + if differentiated_series is None: + differentiated_series = differentiated # Store first valid series + + # Return the last computed series that passed + if differentiated_series is None: + raise ValueError("Could not generate any differentiated series.") + + return differentiated diff --git a/src/RiskLabAI/data/distance/__init__.py b/src/RiskLabAI/data/distance/__init__.py new file mode 100644 index 0000000..f2260bc --- /dev/null +++ b/src/RiskLabAI/data/distance/__init__.py @@ -0,0 +1,2 @@ +from .codependence import distance_correlation, ksg_mutual_information +from .distance_metric import * diff --git a/src/RiskLabAI/data/distance/codependence.py b/src/RiskLabAI/data/distance/codependence.py new file mode 100644 index 0000000..a9fdb48 --- /dev/null +++ b/src/RiskLabAI/data/distance/codependence.py @@ -0,0 +1,188 @@ +r""" +Nonparametric codependence kernels: the KSG mutual-information estimator and distance correlation. + +de Prado's codependence kernel (`distance_metric.calculate_mutual_information`) estimates mutual +information from a binned (histogram) joint distribution, whose bias and variance grow as the sample +shrinks relative to the bin count and which depends on an arbitrary bin choice. These two estimators +are binning-free alternatives for the codependence-kernel role of the variation-of-information metric +and ONC clustering: + +- the Kraskov-Stogbauer-Grassberger (KSG) k-nearest-neighbour mutual-information estimator, which uses + adaptive kNN distances rather than a fixed grid, so it is far less biased on short, noisy, or + nonlinear / heavy-tailed samples; and +- distance correlation, a tuning-free dependence index in [0, 1] that is zero only at independence and + detects nonlinear dependence with no estimation parameter. + +Admitted in Appraisal 11 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tags, verbatim from the verdict: + + KSG: prefer KSG over binned MI/VI on short, noisy, or nonlinear/heavy-tailed samples - it is far + less biased there and is essentially unbiased on linear dependence; on large samples with simple + near-linear dependence the binned estimator is at least as good, so KSG converges rather than + dominating everywhere. Use raw KSG; the surrogate de-bias adds nothing. + + Distance correlation: a tuning-free nonlinear dependence screen / codependence kernel - preferred + when a parameter-free nonlinear screen or maximally stable clustering is wanted (best real-data + cluster stability). It is a dependence index, not a metric on partitions like VI (keep VI/KSG for + the metric role). + +Evidence and caveats: appraisals/11_verdict.md. + +References +---------- +Kraskov, A., Stogbauer, H. and Grassberger, P. (2004) Estimating mutual information. + Physical Review E, 69(6), 066138. +Szekely, G. J., Rizzo, M. L. and Bakirov, N. K. (2007) Measuring and testing dependence by + correlation of distances. Annals of Statistics, 35(6), 2769-2794. +""" + +from __future__ import annotations + +import numpy as np +from scipy.spatial import cKDTree +from scipy.special import digamma + + +def _jitter(values: np.ndarray, rng: np.random.Generator) -> np.ndarray: + """Break exact ties with negligible data-scaled noise so kNN distances are well defined.""" + values = np.asarray(values, dtype=float).ravel() + scale = np.std(values) + if scale == 0.0: + scale = 1.0 + return values + rng.standard_normal(values.shape) * scale * 1e-10 + + +def ksg_mutual_information( + x: np.ndarray, + y: np.ndarray, + k: int = 4, + random_state: int | None = 0, +) -> float: + r""" + Kraskov-Stogbauer-Grassberger mutual information (algorithm 1), in nats. + + Uses the Chebyshev (max) norm in the joint space; for each point the distance to its k-th joint + neighbour sets the marginal search radius, and the marginal neighbour counts ``n_x``, ``n_y`` enter + the digamma estimator + + .. math:: + \hat{I}(X;Y) = \psi(k) + \psi(N) - \langle \psi(n_x+1) + \psi(n_y+1) \rangle . + + Being binning-free it is far less biased than histogram MI on short / nonlinear / heavy-tailed + samples, and essentially unbiased on linear dependence; on large near-linear samples binned MI is at + least as good, so prefer KSG in the former regime. It can return a slightly negative value for + (near-)independent data, which is a characterized property of the estimator, not an error. See the + module docstring for the full regime tag and appraisals/11_verdict.md. + + Parameters + ---------- + x, y : np.ndarray + The two 1-D samples (equal length). + k : int, default 4 + Number of nearest neighbours. + random_state : int, optional + Seed for the tie-breaking jitter (deterministic by default). + + Returns + ------- + float + The estimated mutual information in nats. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> z = rng.standard_normal(2000) + >>> x = z + 0.1 * rng.standard_normal(2000) + >>> y = z + 0.1 * rng.standard_normal(2000) + >>> ksg_mutual_information(x, y) > 0.5 # strong dependence + True + """ + rng = np.random.default_rng(random_state) + x = _jitter(x, rng) + y = _jitter(y, rng) + n = x.size + if n < 2: + return 0.0 + k = min(k, n - 1) + z = np.column_stack([x, y]) + joint = cKDTree(z) + distances, _ = joint.query(z, k=k + 1, p=np.inf) # k+1 to exclude self + eps = distances[:, -1] + tree_x = cKDTree(x.reshape(-1, 1)) + tree_y = cKDTree(y.reshape(-1, 1)) + radius = eps * (1.0 - 1e-10) + n_x = ( + np.asarray( + tree_x.query_ball_point( + x.reshape(-1, 1), radius, p=np.inf, return_length=True + ) + ) + - 1 + ) + n_y = ( + np.asarray( + tree_y.query_ball_point( + y.reshape(-1, 1), radius, p=np.inf, return_length=True + ) + ) + - 1 + ) + n_x = np.maximum(n_x, 0) + n_y = np.maximum(n_y, 0) + mi = digamma(k) + digamma(n) - np.mean(digamma(n_x + 1) + digamma(n_y + 1)) + return float(mi) + + +def _double_center(distance_matrix: np.ndarray) -> np.ndarray: + row = distance_matrix.mean(axis=1, keepdims=True) + col = distance_matrix.mean(axis=0, keepdims=True) + return distance_matrix - row - col + distance_matrix.mean() + + +def distance_correlation(x: np.ndarray, y: np.ndarray) -> float: + r""" + Distance correlation (Szekely-Rizzo-Bakirov 2007), a dependence index in [0, 1]. + + Computed from the double-centered pairwise-distance matrices: with + :math:`A = \mathrm{centered}(|x_i - x_j|)` and :math:`B = \mathrm{centered}(|y_i - y_j|)`, + + .. math:: + \mathrm{dCor} = \sqrt{\frac{\overline{A \cdot B}}{\sqrt{\overline{A \cdot A}\,\overline{B \cdot B}}}}. + + It is zero only at population independence and detects nonlinear dependence with no estimation + parameter, so it is a tuning-free nonlinear dependence screen / codependence kernel. It is a + dependence index, not a metric on partitions like the variation of information (keep VI / KSG for the + metric role). See the module docstring for the full regime tag and appraisals/11_verdict.md. + + Parameters + ---------- + x, y : np.ndarray + The two 1-D samples (equal length). + + Returns + ------- + float + The distance correlation in [0, 1]. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = rng.standard_normal(500) + >>> y = x ** 2 + 0.1 * rng.standard_normal(500) # nonlinear dependence + >>> distance_correlation(x, y) > 0.3 + True + """ + x = np.asarray(x, float).ravel() + y = np.asarray(y, float).ravel() + a = np.abs(x[:, None] - x[None, :]) + b = np.abs(y[:, None] - y[None, :]) + centered_a = _double_center(a) + centered_b = _double_center(b) + dcov2 = (centered_a * centered_b).mean() + dvar_x = (centered_a * centered_a).mean() + dvar_y = (centered_b * centered_b).mean() + denom = np.sqrt(dvar_x * dvar_y) + if denom <= 0: + return 0.0 + return float(np.sqrt(max(dcov2, 0.0) / denom)) diff --git a/src/RiskLabAI/data/distance/distance_metric.py b/src/RiskLabAI/data/distance/distance_metric.py new file mode 100644 index 0000000..0549235 --- /dev/null +++ b/src/RiskLabAI/data/distance/distance_metric.py @@ -0,0 +1,304 @@ +""" +Implements information-theoretic and other distance metrics for +financial data analysis. + +Includes: +- Variation of Information (VI) +- Mutual Information (MI) +- Optimal number of bins calculation +- Angular distance + +Reference: + De Prado, M. (2020) Advances in financial machine learning. + John Wiley & Sons, Chapter 3. +""" + +from typing import Optional + +import numpy as np +import scipy.stats as ss +from sklearn.metrics import mutual_info_score + + +def calculate_variation_of_information( + x: np.ndarray, y: np.ndarray, bins: int, norm: bool = False +) -> float: + """ + Calculate the Variation of Information (VI). + + VI(X, Y) = H(X) + H(Y) - 2 * I(X, Y) + + Reference: + Snippet 3.2, Page 44. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + bins : int + Number of bins for the 2D histogram. + norm : bool, default=False + If True, normalize VI to [0, 1] by dividing by + the joint entropy H(X, Y). + + Returns + ------- + float + The Variation of Information. + """ + histogram_xy = np.histogram2d(x, y, bins)[0] + mutual_information = mutual_info_score(None, None, contingency=histogram_xy) + + marginal_x = ss.entropy(histogram_xy.sum(axis=1)) + marginal_y = ss.entropy(histogram_xy.sum(axis=0)) + + variation_xy = marginal_x + marginal_y - 2 * mutual_information + + if norm: + joint_xy = marginal_x + marginal_y - mutual_information + if joint_xy == 0: + return 0.0 # Avoid division by zero if entropies are 0 + variation_xy /= joint_xy + + return variation_xy + + +def calculate_number_of_bins( + num_observations: int, correlation: Optional[float] = None +) -> int: + """ + Calculate the optimal number of bins for discretization. + + Uses a formula optimized for either univariate (correlation=None) + or bivariate (correlation is provided) cases. + + Reference: + Snippet 3.3, Page 46. + + Parameters + ---------- + num_observations : int + Number of observations (N). + correlation : float, optional + Correlation between the two series. If None, + uses the univariate formula. + + Returns + ------- + int + The optimal number of bins. + """ + if correlation is None: + # Univariate formula + z = ( + 8 + + 324 * num_observations + + 12 * (36 * num_observations + 729 * num_observations**2) ** 0.5 + ) ** (1 / 3.0) + bins = round(z / 6.0 + 2.0 / (3 * z) + 1.0 / 3) + else: + if np.isclose(correlation, 1.0) or np.isclose(correlation, -1.0): + # Handle perfect correlation case by setting to almost 1 + correlation = np.sign(correlation) * (1.0 - 1e-10) + + if (1.0 - correlation**2) == 0: + # Handle numerical instability if correlation is still 1 + return calculate_number_of_bins(num_observations, correlation=None) + + # Bivariate formula + bins = round( + (2**-0.5) + * (1 + (1 + 24 * num_observations / (1.0 - correlation**2)) ** 0.5) ** 0.5 + ) + return int(bins) + + +def calculate_variation_of_information_extended( + x: np.ndarray, y: np.ndarray, norm: bool = False +) -> float: + """ + Calculate Variation of Information with an optimal number of bins. + + This function first calculates the optimal number of bins + using `calculate_number_of_bins` before computing VI. + + Reference: + Snippet 3.3, Page 46. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + norm : bool, default=False + If True, normalize the VI. + + Returns + ------- + float + The Variation of Information. + """ + correlation = np.corrcoef(x, y)[0, 1] + num_bins = calculate_number_of_bins(x.shape[0], correlation=correlation) + + return calculate_variation_of_information(x, y, num_bins, norm) + + +def calculate_mutual_information( + x: np.ndarray, y: np.ndarray, norm: bool = False +) -> float: + """ + Calculate Mutual Information (MI) with an optimal number of bins. + + MI(X, Y) = H(X) + H(Y) - H(X, Y) + Normalized MI = MI(X, Y) / min(H(X), H(Y)) + + Reference: + Snippet 3.4, Page 48. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + norm : bool, default=False + If True, normalize the MI to [0, 1]. + + Returns + ------- + float + The Mutual Information. + """ + correlation = np.corrcoef(x, y)[0, 1] + num_bins = calculate_number_of_bins(x.shape[0], correlation=correlation) + + histogram_xy = np.histogram2d(x, y, num_bins)[0] + mutual_information = mutual_info_score(None, None, contingency=histogram_xy) + + if norm: + marginal_x = ss.entropy(histogram_xy.sum(axis=1)) + marginal_y = ss.entropy(histogram_xy.sum(axis=0)) + + min_entropy = min(marginal_x, marginal_y) + if min_entropy == 0: + return 0.0 # Avoid division by zero + + mutual_information /= min_entropy + + return mutual_information + + +def calculate_distance(dependence: np.ndarray, metric: str = "angular") -> np.ndarray: + r""" + Calculate a distance matrix from a dependence matrix (e.g., correlation). + + Metrics: + - 'angular': \( D_A(i, j) = \sqrt{0.5 (1 - \rho_{ij})} \) + - 'absolute_angular': \( D_{AA}(i, j) = \sqrt{0.5 (1 - |\rho_{ij}|)} \) + + Parameters + ---------- + dependence : np.ndarray + The dependence (correlation) matrix. + metric : str, default="angular" + The metric to use: "angular" or "absolute_angular". + + Returns + ------- + np.ndarray + The resulting distance matrix. + """ + # Clip to handle potential floating point errors + dependence = np.clip(dependence, -1.0, 1.0) + + if metric == "angular": + distance = ((1 - dependence).round(6) / 2.0) ** 0.5 + elif metric == "absolute_angular": + distance = ((1 - np.abs(dependence)).round(6) / 2.0) ** 0.5 + else: + raise ValueError(f"Unknown metric: {metric}") + return distance + + +def calculate_kullback_leibler_divergence(p: np.ndarray, q: np.ndarray) -> float: + """ + Calculate Kullback-Leibler (KL) divergence. + + D_KL(P || Q) = -sum[ p_i * log(q_i / p_i) ] + = sum[ p_i * log(p_i / q_i) ] + + Parameters + ---------- + p : np.ndarray + The "true" probability distribution. + q : np.ndarray + The "approximating" probability distribution. + + Returns + ------- + float + The KL divergence. + """ + # Ensure probabilities sum to 1 + p = p / np.sum(p) + q = q / np.sum(q) + + # Filter for terms where p > 0 and q > 0 + # Where p_i = 0, the term is 0. + # Where q_i = 0 (and p_i > 0), the term is +inf. + mask = (p > 0) & (q > 0) + p_filtered = p[mask] + q_filtered = q[mask] + + if len(p_filtered) == 0: + return 0.0 # No overlapping support + + # Check if any p_i > 0 corresponds to q_i = 0 + if np.any(p[q == 0] > 0): + return np.inf + + divergence = -np.sum(p_filtered * np.log(q_filtered / p_filtered)) + return divergence + + +def calculate_cross_entropy(p: np.ndarray, q: np.ndarray) -> float: + """ + Calculate the cross-entropy between two distributions. + + H(P, Q) = -sum[ p_i * log(q_i) ] + + Parameters + ---------- + p : np.ndarray + The "true" probability distribution. + q : np.ndarray + The "approximating" probability distribution. + + Returns + ------- + float + The cross-entropy. + """ + p = p / np.sum(p) + q = q / np.sum(q) + + # Filter for terms where p > 0 and q > 0 + # Where p_i = 0, the term is 0. + # Where q_i = 0 (and p_i > 0), the term is +inf. + mask = (p > 0) & (q > 0) + p_filtered = p[mask] + q_filtered = q[mask] + + if len(p_filtered) == 0: + return 0.0 + + # Check if any p_i > 0 corresponds to q_i = 0 + if np.any(p[q == 0] > 0): + return np.inf + + entropy = -np.sum(p_filtered * np.log(q_filtered)) + return entropy diff --git a/src/RiskLabAI/data/labeling/__init__.py b/src/RiskLabAI/data/labeling/__init__.py new file mode 100644 index 0000000..fb5a3e8 --- /dev/null +++ b/src/RiskLabAI/data/labeling/__init__.py @@ -0,0 +1,49 @@ +""" +RiskLabAI Data Labeling Module + +Implements the core logic for financial time series labeling, including: +- The Triple-Barrier Method +- Meta-Labeling +- Trend-Scanning Labels +- CUSUM Filtering + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapters 3 & 4. +""" + +from .financial_labels import ( + calculate_t_value_linear_regression, + find_trend_using_trend_scanning, +) +from .labeling import ( + cusum_filter_events_dynamic_threshold, + daily_volatility_with_log_returns, + expand_call, + lin_parts, + meta_events, + meta_labeling, + process_jobs, + report_progress, + symmetric_cusum_filter, + triple_barrier, + vertical_barrier, +) + +__all__ = [ + # from labeling.py + "cusum_filter_events_dynamic_threshold", + "symmetric_cusum_filter", + "daily_volatility_with_log_returns", + "vertical_barrier", + "triple_barrier", + "meta_events", + "meta_labeling", + "lin_parts", + "process_jobs", + "expand_call", + "report_progress", + # from financial_labels.py + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning", +] diff --git a/src/RiskLabAI/data/labeling/financial_labels.py b/src/RiskLabAI/data/labeling/financial_labels.py new file mode 100644 index 0000000..295d349 --- /dev/null +++ b/src/RiskLabAI/data/labeling/financial_labels.py @@ -0,0 +1,137 @@ +""" +Implements the Trend-Scanning Labeling method. + +This method labels events based on the t-value of the linear regression +slope over various forward-looking windows, selecting the window +with the most significant trend. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +import numpy as np +import pandas as pd +from scipy import stats + + +def calculate_t_value_linear_regression(prices: pd.Series) -> float: + """ + Calculate the t-value of the slope of a linear regression. + + Parameters + ---------- + prices : pd.Series + Time series of prices. + + Returns + ------- + float + The t-value of the regression slope. Returns np.nan if + regression fails (e.g., constant series or < 2 data points). + """ + # --- SUGGESTION: Explicitly handle insufficient data --- + if prices.shape[0] < 2: + return np.nan + + x = np.arange(prices.shape[0]) + try: + ols = stats.linregress(x, prices.values) + except ValueError: + return np.nan + + if ols.stderr == 0: + # Perfect trend: t -> +/- inf. Constant series: 0/0 -> undefined (NaN), + # consistent with the documented contract. + return np.sign(ols.slope) * np.inf if ols.slope != 0 else np.nan + + return ols.slope / ols.stderr + + +def find_trend_using_trend_scanning( + molecule: pd.Index, close: pd.Series, span: tuple[int, int] +) -> pd.DataFrame: + """ + Implement the trend-scanning method. + + For each event in `molecule`, this function scans forward over + various window lengths (defined by `span`). It computes the t-value + of the OLS slope for each window and identifies the window + that maximizes `|t_value|`. + + Reference: + Snippet 4.1, Page 69. + + Parameters + ---------- + molecule : pd.Index + The timestamps of the events to be labeled. + close : pd.Series + Time series of close prices. + span : Tuple[int, int] + A tuple of (min_span, max_span) defining the range of + window lengths to scan. `min_span` must be >= 2. + `max_span` is exclusive, as in `range()`. + + Returns + ------- + pd.DataFrame + DataFrame indexed by event, with columns: + - 'End Time': The timestamp of the *end* of the vertical barrier + (i.e., end of the max span). + - 't-Value': The t-value of the most significant trend found. + - 'Trend': The sign of the trend (-1, 0, or 1). + """ + outputs = pd.DataFrame(index=molecule, columns=["End Time", "t-Value", "Trend"]) + + # --- SUGGESTION: Add robustness check for span --- + # Ensure min_span < max_span and min_span is at least 2 for OLS. + if span[0] >= span[1] or span[0] < 2: + outputs["Trend"] = pd.to_numeric(outputs["Trend"], downcast="signed") + return outputs.dropna(subset=["Trend"]) + + spans = range(*span) + # Use span[1] - 1 directly. It's safer than max(spans) + # which fails on an empty range. + max_span_val = span[1] - 1 + + for index in molecule: + t_values = pd.Series(dtype="float64") + + try: + location = close.index.get_loc(index) + except KeyError: + continue # Event timestamp not in close index + + # Ensure we don't scan past the end of the series + if location + max_span_val >= close.shape[0]: + continue + + # Get timestamp for the vertical barrier (end of max span) + vertical_barrier_time = close.index[location + max_span_val] + + for span_val in spans: + # End of this specific window + tail_time = close.index[location + span_val - 1] + window_prices = close.loc[index:tail_time] + + t_values.loc[tail_time] = calculate_t_value_linear_regression(window_prices) + + if t_values.empty: + continue + + # Find the window end that maximized |t_value| + # Use idxmax on the absolute values, but get the original t-value + best_t_value_idx = t_values.replace([-np.inf, np.inf, np.nan], 0).abs().idxmax() + best_t_value = t_values[best_t_value_idx] + + outputs.loc[index] = [ + vertical_barrier_time, + best_t_value, + np.sign(best_t_value), + ] + + outputs["End Time"] = pd.to_datetime(outputs["End Time"]) + outputs["Trend"] = pd.to_numeric(outputs["Trend"], downcast="signed") + + return outputs.dropna(subset=["Trend"]) diff --git a/src/RiskLabAI/data/labeling/labeling.py b/src/RiskLabAI/data/labeling/labeling.py new file mode 100644 index 0000000..7a12491 --- /dev/null +++ b/src/RiskLabAI/data/labeling/labeling.py @@ -0,0 +1,453 @@ +""" +Implements the core financial labeling functions, including: +- CUSUM filters for event sampling. +- Volatility estimation. +- The Triple-Barrier Method (vertical and horizontal barriers). +- Meta-labeling. +- Re-exports of the parallel-execution helpers from ``RiskLabAI.hpc``. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapters 3 & 4. +""" + +from concurrent.futures import ProcessPoolExecutor +from typing import Optional + +import numpy as np +import pandas as pd + +# Parallel-processing helpers have a single source of truth in RiskLabAI.hpc. +# They were previously duplicated in this module; they are re-exported here +# under their historical names for backward compatibility. `lin_parts` maps to +# hpc's `linear_partitions` (identical for valid inputs, with an added guard). +from RiskLabAI.hpc import ( # noqa: F401 (re-exported for backward compat) + expand_call, + process_jobs, + report_progress, +) +from RiskLabAI.hpc import linear_partitions as lin_parts # noqa: F401 + + +def cusum_filter_events_dynamic_threshold( + prices: pd.Series, threshold: pd.Series +) -> pd.DatetimeIndex: + """ + Detect events using the Symmetric CUSUM filter with a dynamic threshold. + + This filter identifies timestamps where the cumulative sum of price + changes exceeds a dynamic, time-varying threshold. + + Reference: + Snippet 3.2, Page 48 (modified for dynamic threshold). + + Parameters + ---------- + prices : pd.Series + A pandas Series of prices. + threshold : pd.Series + A pandas Series containing the threshold values for each timestamp. + Must be aligned with the `prices` index. + + Returns + ------- + pd.DatetimeIndex + Timestamps of the detected events (when a barrier was touched). + """ + time_events = [] + shift_positive, shift_negative = 0.0, 0.0 + price_delta = prices.diff().dropna() + + # Align price changes with thresholds + price_delta, thresholds = price_delta.align(threshold, join="inner", copy=False) + + for (index, value), thresh_val in zip(price_delta.items(), thresholds.values): + shift_positive = max(0.0, shift_positive + value) + shift_negative = min(0.0, shift_negative + value) + + if shift_negative < -thresh_val: + shift_negative = 0.0 # Reset only this counter + time_events.append(index) + elif shift_positive > thresh_val: + shift_positive = 0.0 # Reset only this counter + time_events.append(index) + + return pd.DatetimeIndex(time_events) + + +def symmetric_cusum_filter(prices: pd.Series, threshold: float) -> pd.DatetimeIndex: + """ + Detect events using the Symmetric CUSUM filter with a fixed threshold. + + Reference: + Snippet 3.2, Page 48. + + Parameters + ---------- + prices : pd.Series + A pandas Series of prices. + threshold : float + The fixed threshold value. + + Returns + ------- + pd.DatetimeIndex + Timestamps of the detected events (when a barrier was touched). + """ + time_events = [] + shift_positive, shift_negative = 0.0, 0.0 + price_delta = prices.diff().dropna() + + for index, value in price_delta.items(): + shift_positive = max(0.0, shift_positive + value) + shift_negative = min(0.0, shift_negative + value) + + if shift_negative < -threshold: + shift_negative = 0.0 + time_events.append(index) + elif shift_positive > threshold: + shift_positive = 0.0 + time_events.append(index) + + return pd.DatetimeIndex(time_events) + + +def daily_volatility_with_log_returns(close: pd.Series, span: int = 100) -> pd.Series: + """ + Calculate daily volatility using log returns. + + This method computes the EWMA standard deviation of daily log returns. + + Reference: + Snippet 3.1, Page 44. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + span : int, default=100 + The span parameter for the EWMA. + + Returns + ------- + pd.Series + Series of daily volatility estimates. + """ + # Find timestamps of 1 day prior + df = close.index.searchsorted(close.index - pd.Timedelta(days=1)) + df = df[df > 0] + df = pd.Series( + close.index[df - 1], + index=close.index[close.shape[0] - df.shape[0] :], + ) + + # Calculate log returns + returns = np.log(close.loc[df.index] / close.loc[df.values].values) + stds = returns.ewm(span=span).std().rename("std") + return stds + + +def vertical_barrier( + close: pd.Series, time_events: pd.DatetimeIndex, number_days: int +) -> pd.Series: + """ + Create a vertical barrier for the triple-barrier method. + + This function finds the timestamp that occurs `number_days` after + each event in `time_events`. + + Reference: + Snippet 3.4, Page 50. + + Parameters + ---------- + close : pd.Series + Time series of close prices (used for the index). + time_events : pd.DatetimeIndex + The timestamps of the events (e.g., from CUSUM filter). + number_days : int + The number of days to look forward for the vertical barrier. + + Returns + ------- + pd.Series + A Series where the index is the event timestamp and the + value is the vertical barrier timestamp. + """ + barrier_dates = time_events + pd.Timedelta(days=number_days) + + # Find the integer location of the barrier dates in the price index + timestamp_array = close.index.searchsorted(barrier_dates) + + # Filter out any indices that are out of bounds + valid_indices = timestamp_array < close.shape[0] + timestamp_array = timestamp_array[valid_indices] + + barrier_series = pd.Series( + close.index[timestamp_array], + index=time_events[valid_indices], + ) + return barrier_series + + +def triple_barrier( + close: pd.Series, + events: pd.DataFrame, + ptsl: list[float], + molecule: list[pd.Timestamp], +) -> pd.DataFrame: + """ + Apply the triple-barrier method for a subset of events. + + This is the core worker function for `meta_events`. It finds the + first touch time of the upper, lower, or vertical barrier. + + Reference: + Snippet 3.3, Page 50. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + events : pd.DataFrame + DataFrame with event info. Must contain 'End Time' (vertical + barrier), 'Base Width' (volatility), and 'Side' (if applicable). + ptsl : List[float] + A list of two floats: [profit_taking_multiplier, stop_loss_multiplier]. + molecule : List[pd.Timestamp] + The subset of event timestamps this worker is responsible for. + + Returns + ------- + pd.DataFrame + A DataFrame with columns ['stop_loss', 'profit_taking'] + indexed by the event timestamp, indicating the first + touch time for each horizontal barrier. + """ + # Filter for events this worker owns + events_filtered = events.loc[molecule] + output = pd.DataFrame(index=events_filtered.index) + output["End Time"] = events_filtered["End Time"] # Use original end time + + # 1. Set horizontal barriers + if ptsl[0] > 0: + profit_taking = ptsl[0] * events_filtered["Base Width"] + else: + profit_taking = pd.Series(np.inf, index=events_filtered.index) + + if ptsl[1] > 0: + stop_loss = -ptsl[1] * events_filtered["Base Width"] + else: + stop_loss = pd.Series(-np.inf, index=events_filtered.index) + + # Get side if it exists, otherwise default to 1 (long) + side = events_filtered.get("Side", pd.Series(1.0, index=events_filtered.index)) + + # 2. Find first touch time. + # Positional numpy indexing replaces the per-event pandas label slicing and + # scalar `.loc` writes. For each event we slice the close-price array between + # the event start and its vertical barrier, compute side-adjusted log + # returns, and take the first index where each horizontal barrier is + # crossed. This yields the same first-touch timestamps as the previous + # implementation (verified in test/test_performance.py) but is ~10-70x + # faster on realistic sizes. + close_index = close.index + close_values = close.to_numpy(dtype=float) + close_index_values = close_index.values + + vertical_filled = events_filtered["End Time"].fillna(close_index[-1]) + start_positions = close_index.get_indexer(events_filtered.index) + end_positions = ( + close_index.searchsorted(vertical_filled.to_numpy(), side="right") - 1 + ) + + stop_loss_values = stop_loss.to_numpy() + profit_taking_values = profit_taking.to_numpy() + side_values = side.to_numpy(dtype=float) + + n_events = len(events_filtered) + stop_loss_touch = np.full(n_events, np.datetime64("NaT"), dtype="datetime64[ns]") + profit_taking_touch = np.full( + n_events, np.datetime64("NaT"), dtype="datetime64[ns]" + ) + + for i in range(n_events): + start, end = start_positions[i], end_positions[i] + segment = close_values[start : end + 1] + path_returns = np.log(segment / segment[0]) * side_values[i] + + below = path_returns < stop_loss_values[i] + if below.any(): + stop_loss_touch[i] = close_index_values[start + np.argmax(below)] + + above = path_returns > profit_taking_values[i] + if above.any(): + profit_taking_touch[i] = close_index_values[start + np.argmax(above)] + + # First barrier touched = earliest of {vertical, stop-loss, profit-taking}, + # ignoring NaT (the same semantics as the previous output.min(axis=1)). + candidates = pd.DataFrame( + np.vstack( + [ + events_filtered["End Time"].to_numpy().astype("datetime64[ns]"), + stop_loss_touch, + profit_taking_touch, + ] + ).T, + index=events_filtered.index, + ) + output["End Time"] = candidates.min(axis=1) + return output[["End Time"]] + + +def meta_events( + close: pd.Series, + time_events: pd.DatetimeIndex, + ptsl: list[float], + target: pd.Series, + return_min: float, + num_threads: int, + vertical_barrier_times: Optional[pd.Series] = None, + side: Optional[pd.Series] = None, +) -> pd.DataFrame: + """ + Generate events for meta-labeling using the triple-barrier method. + + This function sets up and executes the triple-barrier search in parallel. + + Reference: + Snippet 3.6, Page 51. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + time_events : pd.DatetimeIndex + Timestamps of the events (e.g., from CUSUM filter). + ptsl : List[float] + [profit_taking_multiplier, stop_loss_multiplier]. + target : pd.Series + Series of volatility (or 'Base Width') for setting barrier width. + return_min : float + The minimum target volatility required to run a search. + num_threads : int + Number of parallel threads to use. + vertical_barrier_times : pd.Series, optional + A Series of vertical barrier timestamps. If None, barriers are not used. + side : pd.Series, optional + A Series of position sides (1 or -1). If None, assumes long-only. + + Returns + ------- + pd.DataFrame + The events DataFrame with columns: + - 'End Time': The timestamp of the *first* barrier touch. + - 'Base Width': The volatility used for the barriers. + - 'Side': (Optional) The side of the bet. + """ + # 1. Filter events by volatility + target = target.reindex(time_events) + target = target[target > return_min] + if target.empty: + return pd.DataFrame(columns=["End Time", "Base Width", "Side"]) + + # 2. Set up vertical barrier + if vertical_barrier_times is None: + vertical_barrier_times = pd.Series(pd.NaT, index=target.index) + else: + vertical_barrier_times = vertical_barrier_times.reindex(target.index) + + # 3. Set up sides + if side is None: + side_series = pd.Series(1.0, index=target.index) + ptsl_final = [ptsl[0], ptsl[0]] # Symmetric barriers + else: + side_series = side.reindex(target.index) + ptsl_final = ptsl[:2] # Asymmetric barriers + + # 4. Create base events DataFrame + events = pd.concat( + { + "End Time": vertical_barrier_times, + "Base Width": target, + "Side": side_series, + }, + axis=1, + ).dropna(subset=["Base Width"]) + + # 5. Run in parallel + molecule_subsets = np.array_split(events.index, num_threads) + + with ProcessPoolExecutor(max_workers=num_threads) as executor: + results = list( + executor.map( + triple_barrier, + [close] * num_threads, + [events] * num_threads, + [ptsl_final] * num_threads, + molecule_subsets, + ) + ) + + # Combine results and update End Time + first_touch_times = pd.concat(results, axis=0)["End Time"] + events["End Time"] = first_touch_times.reindex(events.index) + + if side is None: + events = events.drop("Side", axis=1) + + return events + + +def meta_labeling(events: pd.DataFrame, close: pd.Series) -> pd.DataFrame: + """ + Calculate returns and assign binary labels for meta-labeling. + + Reference: + Snippet 3.7, Page 52. + + Parameters + ---------- + events : pd.DataFrame + The events DataFrame from `meta_events`. + close : pd.Series + Time series of close prices. + + Returns + ------- + pd.DataFrame + DataFrame with columns: + - 'End Time': Time of first barrier touch. + - 'Return': The return of the trade. + - 'Label': The binary label (0 or 1 if 'Side' is present, -1 or 1 if not). + - 'Side': (Optional) The side of the bet. + """ + events_filtered = events.dropna(subset=["End Time"]) + + # Get all unique timestamps + all_dates = events_filtered.index.union( + events_filtered["End Time"].values + ).drop_duplicates() + + # Reindex prices to only the required dates + close_filtered = close.reindex(all_dates, method="bfill") + + out = pd.DataFrame(index=events_filtered.index) + out["End Time"] = events_filtered["End Time"] + + out["Return"] = np.log( + close_filtered.loc[events_filtered["End Time"].values].values + ) - np.log(close_filtered.loc[events_filtered.index].values) + + if "Side" in events_filtered: + out["Return"] *= events_filtered["Side"] + out["Side"] = events_filtered["Side"] + + # Assign labels + out["Label"] = np.sign(out["Return"]) + + if "Side" in events_filtered: + # Meta-labeling: 1 if profitable, 0 if not + out.loc[out["Return"] <= 0, "Label"] = 0.0 + + return out diff --git a/src/RiskLabAI/data/structures/__init__.py b/src/RiskLabAI/data/structures/__init__.py new file mode 100644 index 0000000..6ad32f4 --- /dev/null +++ b/src/RiskLabAI/data/structures/__init__.py @@ -0,0 +1,35 @@ +""" +RiskLabAI Data Structures Module +""" + +# Exports from standard_bars.py +# Exports from imbalance_bars.py +from .imbalance_bars import ExpectedImbalanceBars, FixedImbalanceBars + +# Exports from run_bars.py +from .run_bars import ExpectedRunBars, FixedRunBars +from .standard_bars import StandardBars + +# Exports from time_bars.py +from .time_bars import TimeBars + +# Hardened (anti-degeneracy) information-driven bars (Appraisal 19 robustness fix) +from .hardened_information_driven_bars import ( + HardenedExpectedImbalanceBars, + HardenedExpectedRunBars, +) + +# Note: Removed 'pca_weights' from here as it belongs in 'optimization', +# not 'data.structures'. + + +__all__ = [ + "StandardBars", + "TimeBars", + "ExpectedImbalanceBars", + "FixedImbalanceBars", + "ExpectedRunBars", + "FixedRunBars", + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars", +] diff --git a/src/RiskLabAI/data/structures/abstract_bars.py b/src/RiskLabAI/data/structures/abstract_bars.py new file mode 100644 index 0000000..d71ea25 --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_bars.py @@ -0,0 +1,252 @@ +""" +A base class for the various bar types. Includes the logic shared between +classes, to minimise duplicated code. +""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import Any, Optional, Union + +import numpy as np + +from RiskLabAI.utils.constants import ( + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_DOLLAR, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, + N_TICKS_ON_BAR_FORMATION, + PREVIOUS_TICK_RULE, +) + +# Type hint for a single tick: (datetime, price, volume) +TickData = Union[list[Any], tuple[Any, ...], np.ndarray] + + +class AbstractBars(ABC): + """ + Abstract base class for all bar types. + + Contains the base properties and methods shared between all + bar structures, such as Standard, Time, and Information-Driven bars. + """ + + def __init__(self, bar_type: str): + """ + AbstractBars constructor. + + Parameters + ---------- + bar_type : str + The type of bar, used as a key for statistics. + e.g., 'dollar', 'volume', 'tick_imbalance', etc. + """ + # Main fields + self.bar_type = bar_type + self.tick_counter = 0 # Global tick counter + + # Cached fields for the current bar being built + self.previous_tick_price: Optional[float] = None + self.open_price: Optional[float] = None + self.close_price: Optional[float] = None + self.high_price: float = -np.inf + self.low_price: float = np.inf + + self.base_statistics: dict[str, Union[int, float]] = { + PREVIOUS_TICK_RULE: 0, + CUMULATIVE_TICKS: 0, + CUMULATIVE_DOLLAR: 0, + CUMULATIVE_VOLUME: 0, + CUMULATIVE_BUY_VOLUME: 0, + N_TICKS_ON_BAR_FORMATION: 0, + } + + @abstractmethod + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs bars from input tick data. + + This method must be implemented by all concrete subtypes. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is expected to be a tuple/list of + (date_time, price, volume). + + Returns + ------- + List[List[Any]] + A list of the constructed bars, where each bar is a list + of aggregated values. + """ + pass + + def update_base_fields(self, price: float, tick_rule: int, volume: float): + """ + Update the base statistics for the current bar in progress. + + Parameters + ---------- + price : float + Price of the current tick. + tick_rule : int + Tick rule (1, -1, or 0) of the current tick. + volume : float + Volume of the current tick. + """ + dollar_value = price * volume + + if self.open_price is None: + self.open_price = price + + # Update high/low prices + self.high_price, self.low_price = self._high_and_low_price_update(price) + + # Update bar base statistics + self.base_statistics[CUMULATIVE_TICKS] += 1 + self.base_statistics[CUMULATIVE_DOLLAR] += dollar_value + self.base_statistics[CUMULATIVE_VOLUME] += volume + + if tick_rule == 1: + self.base_statistics[CUMULATIVE_BUY_VOLUME] += volume + + @abstractmethod + def _bar_construction_condition(self, threshold: float) -> bool: + """ + Check if the condition to sample a new bar has been met. + + Parameters + ---------- + threshold : float + The threshold value to check against. + + Returns + ------- + bool + True if the bar should be sampled, False otherwise. + """ + pass + + def _reset_cached_fields(self): + """ + Reset the cached fields when a new bar is sampled. + """ + self.open_price = None + self.high_price, self.low_price = -np.inf, np.inf + + self.base_statistics[CUMULATIVE_TICKS] = 0 + self.base_statistics[CUMULATIVE_DOLLAR] = 0 + self.base_statistics[CUMULATIVE_VOLUME] = 0 + self.base_statistics[CUMULATIVE_BUY_VOLUME] = 0 + + def _tick_rule(self, price: float) -> int: + """ + Compute the tick rule (sign of the price change). + + Reference: + Page 29, Advances in Financial Machine Learning. + + Parameters + ---------- + price : float + Price of the current tick. + + Returns + ------- + int + The signed tick (1, -1, or 0). + """ + if self.previous_tick_price is not None: + tick_difference = price - self.previous_tick_price + else: + tick_difference = 0.0 + + if tick_difference != 0: + signed_tick = int(np.sign(tick_difference)) + self.base_statistics[PREVIOUS_TICK_RULE] = signed_tick + else: + # Use the previous non-zero tick rule + signed_tick = int(self.base_statistics[PREVIOUS_TICK_RULE]) + + self.previous_tick_price = price + return signed_tick + + def _high_and_low_price_update(self, price: float) -> tuple[float, float]: + """ + Update the high and low prices for the current bar. + + Parameters + ---------- + price : float + Price of the current tick. + + Returns + ------- + Tuple[float, float] + (new_high_price, new_low_price) + """ + high_price = max(price, self.high_price) + low_price = min(price, self.low_price) + return high_price, low_price + + def _construct_next_bar( + self, + date_time: Any, + tick_index: int, + price: float, + high_price: float, + low_price: float, + threshold: float, + ) -> list[Any]: + """ + Format and return the newly constructed bar. + + Parameters + ---------- + date_time : Any + Timestamp for the bar. + tick_index : int + The global tick counter index. + price : float + The close price of the bar. + high_price : float + The high price of the bar. + low_price : float + The low price of the bar. + threshold : float + The threshold that triggered this bar's construction. + + Returns + ------- + List[Any] + A list containing all aggregated bar data. + """ + open_price = self.open_price if self.open_price is not None else price + close_price = price + low_price = min(low_price, open_price) + high_price = max(high_price, open_price) + + # Get cumulative statistics + cumulative_ticks = self.base_statistics[CUMULATIVE_TICKS] + cumulative_dollar = self.base_statistics[CUMULATIVE_DOLLAR] + cumulative_volume = self.base_statistics[CUMULATIVE_VOLUME] + cumulative_buy_volume = self.base_statistics[CUMULATIVE_BUY_VOLUME] + cumulative_sell_volume = cumulative_volume - cumulative_buy_volume + + next_bar = [ + date_time, + tick_index, + open_price, + high_price, + low_price, + close_price, + cumulative_volume, + cumulative_buy_volume, + cumulative_sell_volume, + cumulative_ticks, + cumulative_dollar, + threshold, + ] + + return next_bar diff --git a/src/RiskLabAI/data/structures/abstract_imbalance_bars.py b/src/RiskLabAI/data/structures/abstract_imbalance_bars.py new file mode 100644 index 0000000..7cf805a --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_imbalance_bars.py @@ -0,0 +1,178 @@ +""" +Abstract base class for Imbalance Bars (Fixed and Expected). +""" + +from abc import abstractmethod +from collections.abc import Iterable +from typing import Any, Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import TickData +from RiskLabAI.data.structures.abstract_information_driven_bars import ( + AbstractInformationDrivenBars, +) +from RiskLabAI.utils.constants import ( + CUMULATIVE_THETA, + CUMULATIVE_TICKS, + EXPECTED_IMBALANCE, + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, + PREVIOUS_TICK_IMBALANCES_LIST, +) + + +class AbstractImbalanceBars(AbstractInformationDrivenBars): + """ + Abstract class for Imbalance Bars (Fixed and Expected). + + Implements the bar sampling logic based on cumulative imbalance (theta) + exceeding a dynamic threshold E[T] * |E[b_t]|. + + Reference: + Pages 29-30, Advances in Financial Machine Learning. + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + window_size_for_expected_imbalance_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + analyse_thresholds: bool, + ): + """ + Constructor. + (Parameters same as original) + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation, + ) + + self.imbalance_bars_statistics = { + CUMULATIVE_THETA: 0.0, + EXPECTED_IMBALANCE: np.nan, + PREVIOUS_BARS_N_TICKS_LIST: [], + PREVIOUS_TICK_IMBALANCES_LIST: [], + } + + self.analyse_thresholds = [] if analyse_thresholds else None + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs imbalance bars from input tick data. + (Parameters same as original) + """ + bars_list = [] + + # Keep track of last timestamp and threshold + date_time = None + threshold = np.inf + + for tick_data in data: + self.tick_counter += 1 + + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + # Calculate imbalance + imbalance = self._imbalance_at_tick(price, tick_rule, volume) + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST].append( + imbalance + ) + self.imbalance_bars_statistics[CUMULATIVE_THETA] += imbalance + + # Warm-up E[b] if it's the first time + if np.isnan(self.imbalance_bars_statistics[EXPECTED_IMBALANCE]): + self.imbalance_bars_statistics[EXPECTED_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + + if self.analyse_thresholds is not None: + stats = { + **self.base_statistics, + **self.information_driven_bars_statistics, + **self.imbalance_bars_statistics, + "timestamp": date_time, + } + self.analyse_thresholds.append(stats) + + # Calculate threshold and check condition + expected_ticks = self.information_driven_bars_statistics[ + EXPECTED_TICKS_NUMBER + ] + expected_imbalance = self.imbalance_bars_statistics[EXPECTED_IMBALANCE] + + if np.isnan(expected_ticks) or np.isnan(expected_imbalance): + threshold = np.inf + else: + threshold = expected_ticks * np.abs(expected_imbalance) + + if self._bar_construction_condition(threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + threshold, + ) + bars_list.append(next_bar) + + # Store T for E[T] update + self.imbalance_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST].append( + self.base_statistics[CUMULATIVE_TICKS] + ) + + # Update E[T] + self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] = ( + self._expected_number_of_ticks() + ) + + # Update E[b] + self.imbalance_bars_statistics[EXPECTED_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=False, + ) + ) + + # Reset cached fields + self._reset_cached_fields() + + return bars_list + + def _bar_construction_condition(self, threshold: float) -> bool: + """Check if cumulative imbalance |theta| exceeds the threshold.""" + if np.isnan(threshold) or np.isinf(threshold): + return False + + cumulative_theta = self.imbalance_bars_statistics[CUMULATIVE_THETA] + return np.abs(cumulative_theta) >= threshold + + def _reset_cached_fields(self): + """Reset base fields and cumulative theta.""" + super()._reset_cached_fields() + self.imbalance_bars_statistics[CUMULATIVE_THETA] = 0.0 + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """Calculate E[T] when a new bar is sampled.""" + pass diff --git a/src/RiskLabAI/data/structures/abstract_information_driven_bars.py b/src/RiskLabAI/data/structures/abstract_information_driven_bars.py new file mode 100644 index 0000000..adbfd1c --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_information_driven_bars.py @@ -0,0 +1,129 @@ +""" +Abstract base class for Information-Driven Bars (Imbalance and Run bars). +""" + +from abc import abstractmethod +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import AbstractBars +from RiskLabAI.utils.constants import ( + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_TICKS_NUMBER, +) +from RiskLabAI.utils.ewma import ewma + + +class AbstractInformationDrivenBars(AbstractBars): + """ + Abstract class for Information-Driven Bars (Imbalance and Run). + + Implements the shared logic for bars that sample based on + dynamic thresholds derived from tick imbalances. + + Reference: + Pages 29-32, Advances in Financial Machine Learning. + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_run', etc. + window_size_for_expected_n_ticks_estimation : int, optional + Window size for EWMA of E[T]. (None for Fixed bars). + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + """ + super().__init__(bar_type) + self.information_driven_bars_statistics = { + EXPECTED_TICKS_NUMBER: float(initial_estimate_of_expected_n_ticks_in_bar), + EXPECTED_IMBALANCE_WINDOW: window_size_for_expected_imbalance_estimation, + } + self.window_size_for_expected_n_ticks_estimation = ( + window_size_for_expected_n_ticks_estimation + ) + + def _ewma_expected_imbalance( + self, array: list, window: int, warm_up: bool = False + ) -> float: + """ + Calculate the EWMA of the expected imbalance. + + Parameters + ---------- + array : list + List of observed imbalances. + window : int + EWMA window. + warm_up : bool, default=False + If True, wait for `E[T]` ticks before calculating. + + Returns + ------- + float + The EWMA of the expected imbalance. + """ + if warm_up: + expected_ticks = self.information_driven_bars_statistics[ + EXPECTED_TICKS_NUMBER + ] + if np.isnan(expected_ticks) or len(array) < expected_ticks: + return np.nan + + ewma_window = int(min(len(array), window)) + if ewma_window == 0: + return np.nan + + return ewma(np.array(array[-ewma_window:], dtype=float), window=ewma_window)[-1] + + def _imbalance_at_tick( + self, price: float, signed_tick: int, volume: float + ) -> float: + """ + Calculate the imbalance (theta) for the current tick. + + Parameters + ---------- + price : float + Current tick price. + signed_tick : int + Current tick rule (1, -1, 0). + volume : float + Current tick volume. + + Returns + ------- + float + The calculated imbalance. + """ + if self.bar_type in ("tick_imbalance", "tick_run"): + imbalance = float(signed_tick) + elif self.bar_type in ("dollar_imbalance", "dollar_run"): + imbalance = float(signed_tick * volume * price) + elif self.bar_type in ("volume_imbalance", "volume_run"): + imbalance = float(signed_tick * volume) + else: + raise ValueError(f"Unknown bar_type for imbalance: {self.bar_type}") + + return imbalance + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """ + Abstract method to update the expected number of ticks (E[T]). + This is implemented differently for "fixed" vs. "expected" bars. + """ + pass diff --git a/src/RiskLabAI/data/structures/abstract_run_bars.py b/src/RiskLabAI/data/structures/abstract_run_bars.py new file mode 100644 index 0000000..5320bce --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_run_bars.py @@ -0,0 +1,261 @@ +""" +Abstract base class for Run Bars (Fixed and Expected). +""" + +from abc import abstractmethod +from collections.abc import Iterable +from typing import Any, Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import TickData +from RiskLabAI.data.structures.abstract_information_driven_bars import ( + AbstractInformationDrivenBars, +) +from RiskLabAI.utils.constants import ( + BUY_TICKS_NUMBER, + CUMULATIVE_BUY_THETA, + CUMULATIVE_SELL_THETA, + CUMULATIVE_TICKS, + EXPECTED_BUY_IMBALANCE, + EXPECTED_BUY_TICKS_PROPORTION, + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_SELL_IMBALANCE, + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST, + PREVIOUS_BARS_N_TICKS_LIST, + PREVIOUS_TICK_IMBALANCES_BUY_LIST, + PREVIOUS_TICK_IMBALANCES_SELL_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class AbstractRunBars(AbstractInformationDrivenBars): + """ + Abstract class for Run Bars (Fixed and Expected). + (Docstrings same as original) + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + window_size_for_expected_imbalance_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + analyse_thresholds: bool, + ): + """ + Constructor. + (Parameters same as original) + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation, + ) + + self.run_bars_statistics = { + CUMULATIVE_BUY_THETA: 0.0, + CUMULATIVE_SELL_THETA: 0.0, + EXPECTED_BUY_IMBALANCE: np.nan, + EXPECTED_SELL_IMBALANCE: np.nan, + EXPECTED_BUY_TICKS_PROPORTION: np.nan, + BUY_TICKS_NUMBER: 0, + PREVIOUS_BARS_N_TICKS_LIST: [], + PREVIOUS_TICK_IMBALANCES_BUY_LIST: [], + PREVIOUS_TICK_IMBALANCES_SELL_LIST: [], + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST: [], + } + + self.analyse_thresholds = [] if analyse_thresholds else None + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs run bars from input tick data. + (Parameters same as original) + """ + bars_list = [] + + # Keep track of last timestamp and threshold + date_time = None + threshold = np.inf + + for tick_data in data: + self.tick_counter += 1 + + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + # Calculate imbalance + imbalance = self._imbalance_at_tick(price, tick_rule, volume) + + if imbalance > 0: + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST].append( + imbalance + ) + self.run_bars_statistics[CUMULATIVE_BUY_THETA] += imbalance + self.run_bars_statistics[BUY_TICKS_NUMBER] += 1 + elif imbalance < 0: + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST].append( + -imbalance + ) + self.run_bars_statistics[CUMULATIVE_SELL_THETA] += -imbalance + + # Warm-up E[theta_buy], E[theta_sell], and P[buy] + warm_up_stats = [ + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE], + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE], + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION], + ] + + if np.isnan(warm_up_stats).any(): + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + + # Update P[buy] + if self.base_statistics[CUMULATIVE_TICKS] > 0: + buy_ticks_num = self.run_bars_statistics[BUY_TICKS_NUMBER] + cum_ticks = self.base_statistics[CUMULATIVE_TICKS] + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] = ( + buy_ticks_num / cum_ticks + ) + + if self.analyse_thresholds is not None: + stats = { + **self.base_statistics, + **self.information_driven_bars_statistics, + **self.run_bars_statistics, + "timestamp": date_time, + } + self.analyse_thresholds.append(stats) + + # Calculate threshold and check condition + threshold = self._calculate_run_threshold() + + if self._bar_construction_condition(threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + threshold, + ) + bars_list.append(next_bar) + + # Store T and P[buy] for E[T] and E[P[buy]] updates + cum_ticks = self.base_statistics[CUMULATIVE_TICKS] + buy_ticks_num = self.run_bars_statistics[BUY_TICKS_NUMBER] + + self.run_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST].append(cum_ticks) + + # Avoid division by zero if bar has 0 ticks (should be rare) + buy_proportion = (buy_ticks_num / cum_ticks) if cum_ticks > 0 else 0 + self.run_bars_statistics[ + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST + ].append(buy_proportion) + + # Update E[T] + self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] = ( + self._expected_number_of_ticks() + ) + + # Update E[P[buy]] + window = ( + self.window_size_for_expected_n_ticks_estimation + or self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ] + ) + + prob_buy_list = self.run_bars_statistics[ + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST + ] + + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] = ewma( + np.array(prob_buy_list[-window:], dtype=float), + window, + )[-1] + + # Update E[theta_buy] and E[theta_sell] + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + ) + ) + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + ) + ) + + # Reset cached fields + self._reset_cached_fields() + + return bars_list + + def _calculate_run_threshold(self) -> float: + """Helper function to calculate the dynamic run threshold.""" + e_t = self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + e_p_buy = self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] + e_theta_buy = self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] + e_theta_sell = self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] + + if np.isnan([e_t, e_p_buy, e_theta_buy, e_theta_sell]).any(): + return np.inf + + # Threshold = E[T] * max(P[buy] * E[theta_buy], (1-P[buy]) * E[theta_sell]) + buy_threshold = e_p_buy * e_theta_buy + sell_threshold = (1 - e_p_buy) * e_theta_sell + + return e_t * max(buy_threshold, sell_threshold) + + def _bar_construction_condition(self, threshold: float) -> bool: + """Check if cumulative buy or sell run exceeds the threshold.""" + if np.isinf(threshold) or np.isnan(threshold): + return False + + max_theta = max( + self.run_bars_statistics[CUMULATIVE_BUY_THETA], + self.run_bars_statistics[CUMULATIVE_SELL_THETA], + ) + return max_theta >= threshold + + def _reset_cached_fields(self): + """Reset base fields and cumulative run counters.""" + super()._reset_cached_fields() + self.run_bars_statistics[CUMULATIVE_BUY_THETA] = 0.0 + self.run_bars_statistics[CUMULATIVE_SELL_THETA] = 0.0 + self.run_bars_statistics[BUY_TICKS_NUMBER] = 0 + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """Calculate E[T] when a new bar is sampled.""" + pass diff --git a/src/RiskLabAI/data/structures/hardened_information_driven_bars.py b/src/RiskLabAI/data/structures/hardened_information_driven_bars.py new file mode 100644 index 0000000..0862eeb --- /dev/null +++ b/src/RiskLabAI/data/structures/hardened_information_driven_bars.py @@ -0,0 +1,147 @@ +r""" +Hardened (anti-degeneracy) imbalance and run bars. + +de Prado's information-driven bars (AFML ch.2) sample a new bar when the cumulative signed imbalance +(or run) exceeds the product of the expected number of ticks E[T] and the expected absolute imbalance. +With an EWMA estimate of E[T] and the imbalance, the naive construction degenerates under some +parameterizations: the bar size can collapse toward a single tick (when the running imbalance estimate +makes the threshold tiny) or diverge toward extremely long bars (when E[T] runs away), so the bar series +is no longer usable. The hardened variant adds two guards that bound the bar size without touching the +well-behaved regime. + +Admitted in Appraisal 19 (CONTRIBUTIONS_LEDGER 2026-06-27) as a correctness / robustness fix. Scope +note, verbatim from the verdict: + + replace the naive imbalance/run-bar construction with the guarded version - it removes the + documented degeneracy (1-tick collapse / divergence) while leaving well-behaved parameterizations + unchanged. This is a robustness admission, not a performance claim. + +The guards are: (1) never close a bar before ``min_ticks`` ticks (prevents the 1-tick collapse), and +(2) force-close a bar at ``max_ticks`` ticks (prevents divergence); E[T] is additionally clamped to +``[min_ticks, max_ticks]`` via the base class's ``expected_ticks_number_bounds``. The naive +``ExpectedImbalanceBars`` / ``ExpectedRunBars`` are left unchanged and remain available; the hardened +classes only subclass them. Held-out confirmed on the sealed tick segment (appraisals/19_results, +HELDOUT.md): naive imbalance collapses (86.9% of bars <= 2 ticks, min 1) while hardened holds (0% +<= 2-tick, min 20); naive run diverges (11 bars) while hardened stays bounded; a well-behaved run +configuration is unchanged (6.2% bar-count difference). Evidence and caveats: appraisals/19_verdict.md. + +References +---------- +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 2 (information-driven bars). +""" + +from __future__ import annotations + +from RiskLabAI.utils.constants import CUMULATIVE_TICKS + +from .imbalance_bars import ExpectedImbalanceBars +from .run_bars import ExpectedRunBars + + +class HardenedExpectedImbalanceBars(ExpectedImbalanceBars): + r""" + Expected-imbalance bars with anti-degeneracy guards (Appraisal 19 robustness fix). + + Subclasses :class:`ExpectedImbalanceBars` and overrides only the bar-construction condition: a bar + cannot close before ``min_ticks`` ticks (removes the 1-tick collapse) and is force-closed at + ``max_ticks`` ticks (removes the divergence); E[T] is clamped to ``[min_ticks, max_ticks]``. Between + the guards the original de Prado condition is used unchanged, so a well-parameterized construction is + left essentially as-is. See the module docstring for the verbatim scope note and + appraisals/19_verdict.md. + + Parameters + ---------- + bar_type : str + e.g. ``'dollar_imbalance'`` / ``'tick_imbalance'`` (passed through to the base class). + min_ticks : int + Minimum number of ticks before a bar may close (the collapse guard). + max_ticks : int + Maximum number of ticks before a bar is force-closed (the divergence guard). + **kwargs + Forwarded to :class:`ExpectedImbalanceBars` (``window_size_for_expected_n_ticks_estimation``, + ``initial_estimate_of_expected_n_ticks_in_bar``, ``window_size_for_expected_imbalance_estimation``, + ``analyse_thresholds``). ``expected_ticks_number_bounds`` is set to ``(min_ticks, max_ticks)``. + + Examples + -------- + >>> bars = HardenedExpectedImbalanceBars( + ... bar_type="dollar_imbalance", + ... initial_estimate_of_expected_n_ticks_in_bar=200, + ... min_ticks=20, + ... max_ticks=5000, + ... ) + >>> data = [("2020-01-01 10:00:00", 100.0, 10.0)] * 60 + >>> out = bars.construct_bars_from_data(data) # no 1-tick collapse + >>> isinstance(out, list) + True + """ + + def __init__(self, *args, min_ticks: int, max_ticks: int, **kwargs): + if min_ticks <= 0 or max_ticks < min_ticks: + raise ValueError("require 0 < min_ticks <= max_ticks") + kwargs["expected_ticks_number_bounds"] = (min_ticks, max_ticks) + super().__init__(*args, **kwargs) + self.min_ticks = min_ticks + self.max_ticks = max_ticks + + def _bar_construction_condition(self, threshold: float) -> bool: + n_ticks = self.base_statistics[CUMULATIVE_TICKS] + if n_ticks < self.min_ticks: + return False + if n_ticks >= self.max_ticks: + return True + return super()._bar_construction_condition(threshold) + + +class HardenedExpectedRunBars(ExpectedRunBars): + r""" + Expected-run bars with anti-degeneracy guards (Appraisal 19 robustness fix). + + Subclasses :class:`ExpectedRunBars` and overrides only the bar-construction condition with the same + two guards as :class:`HardenedExpectedImbalanceBars`: no close before ``min_ticks`` ticks, force-close + at ``max_ticks`` ticks, E[T] clamped to ``[min_ticks, max_ticks]``. See the module docstring for the + verbatim scope note and appraisals/19_verdict.md. + + Parameters + ---------- + bar_type : str + e.g. ``'dollar_run'`` / ``'tick_run'``. + min_ticks : int + Minimum ticks before a bar may close (collapse guard). + max_ticks : int + Maximum ticks before a bar is force-closed (divergence guard). + **kwargs + Forwarded to :class:`ExpectedRunBars`; ``expected_ticks_number_bounds`` is set to + ``(min_ticks, max_ticks)``. + + Examples + -------- + >>> bars = HardenedExpectedRunBars( + ... bar_type="dollar_run", + ... window_size_for_expected_n_ticks_estimation=200, + ... initial_estimate_of_expected_n_ticks_in_bar=200, + ... window_size_for_expected_imbalance_estimation=10000, + ... min_ticks=20, + ... max_ticks=5000, + ... ) + >>> data = [("2020-01-01 10:00:00", 100.0, 10.0)] * 60 + >>> out = bars.construct_bars_from_data(data) + >>> isinstance(out, list) + True + """ + + def __init__(self, *args, min_ticks: int, max_ticks: int, **kwargs): + if min_ticks <= 0 or max_ticks < min_ticks: + raise ValueError("require 0 < min_ticks <= max_ticks") + kwargs["expected_ticks_number_bounds"] = (min_ticks, max_ticks) + super().__init__(*args, **kwargs) + self.min_ticks = min_ticks + self.max_ticks = max_ticks + + def _bar_construction_condition(self, threshold: float) -> bool: + n_ticks = self.base_statistics[CUMULATIVE_TICKS] + if n_ticks < self.min_ticks: + return False + if n_ticks >= self.max_ticks: + return True + return super()._bar_construction_condition(threshold) diff --git a/src/RiskLabAI/data/structures/imbalance_bars.py b/src/RiskLabAI/data/structures/imbalance_bars.py new file mode 100644 index 0000000..749dee0 --- /dev/null +++ b/src/RiskLabAI/data/structures/imbalance_bars.py @@ -0,0 +1,134 @@ +""" +Concrete implementations of Imbalance Bars: +- ExpectedImbalanceBars +- FixedImbalanceBars +""" + +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_imbalance_bars import AbstractImbalanceBars +from RiskLabAI.utils.constants import ( + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class ExpectedImbalanceBars(AbstractImbalanceBars): + """ + Concrete class for Imbalance Bars with a dynamic, EWMA-based + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_imbalance'. + window_size_for_expected_n_ticks_estimation : int + Window size for EWMA of E[T]. + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + expected_ticks_number_bounds : Tuple[float, float], optional + (Lower, Upper) bounds to clamp E[T]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds, + ) + + if expected_ticks_number_bounds is None: + self.expected_ticks_number_lower_bound = 0.0 + self.expected_ticks_number_upper_bound = np.inf + else: + self.expected_ticks_number_lower_bound = expected_ticks_number_bounds[0] + self.expected_ticks_number_upper_bound = expected_ticks_number_bounds[1] + + def _expected_number_of_ticks(self) -> float: + """ + Calculate E[T] using an EWMA of previous bar tick counts. + """ + prev_ticks_list = self.imbalance_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST] + + window = self.window_size_for_expected_n_ticks_estimation + if window is None or window <= 0: + # Fallback to simple mean if window is invalid + return np.mean(prev_ticks_list) + + if not prev_ticks_list: + # No bars yet, return initial estimate + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + + ewma_ticks = ewma( + np.array(prev_ticks_list[-window:], dtype=float), window=window + )[-1] + + return min( + max(ewma_ticks, self.expected_ticks_number_lower_bound), + self.expected_ticks_number_upper_bound, + ) + + +class FixedImbalanceBars(AbstractImbalanceBars): + """ + Concrete class for Imbalance Bars with a fixed (constant) + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + window_size_for_expected_n_ticks_estimation: Optional[int] = None, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_imbalance'. + initial_estimate_of_expected_n_ticks_in_bar : int + The *fixed* value for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + window_size_for_expected_n_ticks_estimation : int, optional + Ignored. Kept for consistent interface with factory. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation=None, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds=analyse_thresholds, + ) + + def _expected_number_of_ticks(self) -> float: + """ + Return the fixed E[T] value. + """ + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] diff --git a/src/RiskLabAI/data/structures/run_bars.py b/src/RiskLabAI/data/structures/run_bars.py new file mode 100644 index 0000000..ac1fb5c --- /dev/null +++ b/src/RiskLabAI/data/structures/run_bars.py @@ -0,0 +1,132 @@ +""" +Concrete implementations of Run Bars: +- ExpectedRunBars +- FixedRunBars +""" + +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_run_bars import AbstractRunBars +from RiskLabAI.utils.constants import ( + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class ExpectedRunBars(AbstractRunBars): + """ + Concrete class for Run Bars with a dynamic, EWMA-based + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_run', 'tick_run'. + window_size_for_expected_n_ticks_estimation : int + Window size for EWMA of E[T]. + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[theta]. + expected_ticks_number_bounds : Tuple[float, float], optional + (Lower, Upper) bounds to clamp E[T]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds, + ) + + if expected_ticks_number_bounds is None: + self.expected_ticks_number_lower_bound = 0.0 + self.expected_ticks_number_upper_bound = np.inf + else: + self.expected_ticks_number_lower_bound = expected_ticks_number_bounds[0] + self.expected_ticks_number_upper_bound = expected_ticks_number_bounds[1] + + def _expected_number_of_ticks(self) -> float: + """ + Calculate E[T] using an EWMA of previous bar tick counts. + """ + prev_ticks_list = self.run_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST] + + window = self.window_size_for_expected_n_ticks_estimation + if window is None or window <= 0: + return np.mean(prev_ticks_list) + + if not prev_ticks_list: + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + + ewma_ticks = ewma( + np.array(prev_ticks_list[-window:], dtype=float), window=window + )[-1] + + return min( + max(ewma_ticks, self.expected_ticks_number_lower_bound), + self.expected_ticks_number_upper_bound, + ) + + +class FixedRunBars(AbstractRunBars): + """ + Concrete class for Run Bars with a fixed (constant) + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + analyse_thresholds: bool = False, + window_size_for_expected_n_ticks_estimation: Optional[int] = None, # Not used + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_run', 'tick_run'. + initial_estimate_of_expected_n_ticks_in_bar : int + The *fixed* value for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[theta]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + window_size_for_expected_n_ticks_estimation : int, optional + Ignored. Kept for consistent interface with factory. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation=None, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds=analyse_thresholds, + ) + + def _expected_number_of_ticks(self) -> float: + """ + Return the fixed E[T] value. + """ + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] diff --git a/src/RiskLabAI/data/structures/standard_bars.py b/src/RiskLabAI/data/structures/standard_bars.py new file mode 100644 index 0000000..110a3b9 --- /dev/null +++ b/src/RiskLabAI/data/structures/standard_bars.py @@ -0,0 +1,87 @@ +""" +Implements Standard Bars (Tick, Volume, Dollar). +""" + +from collections.abc import Iterable +from typing import Any + +from RiskLabAI.data.structures.abstract_bars import AbstractBars, TickData + + +class StandardBars(AbstractBars): + """ + Concrete class for Standard Bars (Tick, Volume, Dollar). + + Generates a new bar whenever a cumulative threshold of ticks, + volume, or dollars is reached. + """ + + def __init__(self, bar_type: str, threshold: float = 50000): + """ + StandardBars constructor. + + Parameters + ---------- + bar_type : str + Type of bar. Must be one of: + `CUMULATIVE_TICKS`, `CUMULATIVE_VOLUME`, `CUMULATIVE_DOLLAR`. + threshold : float, default=50000 + The threshold value to trigger bar construction. + """ + super().__init__(bar_type) + self.threshold = threshold + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs standard bars from input tick data. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is (date_time, price, volume). + + Returns + ------- + List[List[Any]] + A list of the constructed standard bars. + """ + bars_list = [] + + # Keep track of last timestamp for final bar + date_time = None + + for tick_data in data: + self.tick_counter += 1 + + # Unpack data + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price # Update close price continuously + + # Check if bar construction condition is met + if self._bar_construction_condition(self.threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + self.threshold, + ) + bars_list.append(next_bar) + + # Reset cached fields for the next bar + self._reset_cached_fields() + + return bars_list + + def _bar_construction_condition(self, threshold: float) -> bool: + """ + Check if the cumulative value of the `bar_type` has + exceeded the threshold. + """ + return self.base_statistics[self.bar_type] >= threshold diff --git a/src/RiskLabAI/data/structures/time_bars.py b/src/RiskLabAI/data/structures/time_bars.py new file mode 100644 index 0000000..835e7a3 --- /dev/null +++ b/src/RiskLabAI/data/structures/time_bars.py @@ -0,0 +1,135 @@ +""" +Implements Time Bars. +""" + +from collections.abc import Iterable +from typing import Any + +import numpy as np +import pandas as pd + +from RiskLabAI.data.structures.abstract_bars import AbstractBars, TickData + + +class TimeBars(AbstractBars): + """ + Concrete class for Time Bars. + + Generates a new bar whenever a fixed time duration has passed. + """ + + def __init__(self, resolution_type: str, resolution_units: int): + """ + TimeBars constructor. + + Parameters + ---------- + resolution_type : str + Type of bar resolution: 'S', 'MIN', 'H', 'D', 'W'. + resolution_units : int + Number of units for the resolution (e.g., 5 for 5-minutes). + """ + super().__init__(bar_type="time") + + self.resolution_to_n_seconds = { + "S": 1, + "MIN": 60, + "H": 3600, + "D": 86400, + "W": 604800, + } + + if resolution_type.upper() not in self.resolution_to_n_seconds: + raise ValueError( + f"Invalid resolution_type. Use one of {list(self.resolution_to_n_seconds.keys())}" + ) + + self.resolution_type = resolution_type.upper() + self.resolution_units = resolution_units + self.threshold_in_seconds = ( + self.resolution_units * self.resolution_to_n_seconds[self.resolution_type] + ) + + self.current_bar_timestamp = np.nan + self.current_bar_end_timestamp = np.nan + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs time bars from input tick data. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is (date_time, price, volume). + `date_time` must be a pandas Timestamp. + + Returns + ------- + List[List[Any]] + A list of the constructed time bars. + """ + bars_list = [] + for tick_data in data: + self.tick_counter += 1 + + # Unpack data + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Get tick timestamp in seconds + try: + tick_timestamp_sec = date_time.timestamp() + except AttributeError as err: + raise TypeError( + "TimeBars require `date_time` to be a pandas Timestamp " + "or datetime object with a .timestamp() method." + ) from err + + # Determine the "floor" timestamp for this bar + bar_start_timestamp_sec = ( + int(tick_timestamp_sec // self.threshold_in_seconds) + * self.threshold_in_seconds + ) + + # Initialize first bar + if np.isnan(self.current_bar_timestamp): + self.current_bar_timestamp = bar_start_timestamp_sec + self.current_bar_end_timestamp = ( + bar_start_timestamp_sec + self.threshold_in_seconds + ) + + # Check if this tick belongs to a new bar + if self._bar_construction_condition(tick_timestamp_sec): + # Construct the *previous* bar + bar_end_time = pd.to_datetime(self.current_bar_end_timestamp, unit="s") + + next_bar = self._construct_next_bar( + bar_end_time, + self.tick_counter - 1, # Index of the *previous* tick + self.close_price, # Close price from *previous* tick + self.high_price, + self.low_price, + self.current_bar_end_timestamp, + ) + bars_list.append(next_bar) + + # Reset for the new bar + self._reset_cached_fields() + self.current_bar_timestamp = bar_start_timestamp_sec + self.current_bar_end_timestamp = ( + bar_start_timestamp_sec + self.threshold_in_seconds + ) + + # Update fields with current tick data + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + return bars_list + + def _bar_construction_condition(self, tick_timestamp_sec: float) -> bool: + """ + Check if the current tick's timestamp has crossed + the end-time of the current bar. + """ + return tick_timestamp_sec >= self.current_bar_end_timestamp diff --git a/src/RiskLabAI/data/synthetic_data/__init__.py b/src/RiskLabAI/data/synthetic_data/__init__.py new file mode 100644 index 0000000..b06e36e --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/__init__.py @@ -0,0 +1,35 @@ +""" +RiskLabAI Synthetic Data Module + +Provides tools for generating synthetic financial data, from +simple block-diagonal covariance matrices to complex, +regime-switching Heston-Merton price paths. +""" + +from .drift_burst_hypothesis import drift_volatility_burst +from .simulation import ( + form_block_matrix, + form_true_matrix, + random_cov, + simulates_cov_mu, +) +from .synthetic_controlled_environment import ( + align_params_length, + compute_log_returns, + generate_prices_from_regimes, + heston_merton_log_returns, + parallel_generate_prices, +) + +__all__ = [ + "drift_volatility_burst", + "compute_log_returns", + "heston_merton_log_returns", + "align_params_length", + "generate_prices_from_regimes", + "parallel_generate_prices", + "random_cov", + "form_block_matrix", + "form_true_matrix", + "simulates_cov_mu", +] diff --git a/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py b/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py new file mode 100644 index 0000000..7dd5156 --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py @@ -0,0 +1,94 @@ +""" +Implements the Drift-Burst Hypothesis (DBH) model for synthetic data. + +This model generates drift and volatility parameters for a bubble scenario, +featuring an "explosion" at the midpoint. +""" + +import numpy as np + + +def drift_volatility_burst( + bubble_length: int, + a_before: float, + a_after: float, + b_before: float, + b_after: float, + alpha: float, + beta: float, + explosion_filter_width: float = 0.1, +) -> tuple[np.ndarray, np.ndarray]: + r""" + Compute the drift and volatility for a DBH burst scenario. + + The model defines drift and volatility as a function of time `t` + (represented by `steps` from 0 to 1), with an explosion at `t=0.5`. + + .. math:: + drift(t) = \frac{a(t)}{|t - 0.5|^\alpha} + vol(t) = \frac{b(t)}{|t - 0.5|^\beta} + + To prevent division by zero, an `explosion_filter_width` is used + to clamp the denominator near `t=0.5`. + + Parameters + ---------- + bubble_length : int + The number of timesteps in the bubble. + a_before : float + Drift coefficient 'a' for `t < 0.5`. + a_after : float + Drift coefficient 'a' for `t > 0.5`. + b_before : float + Volatility coefficient 'b' for `t < 0.5`. + b_after : float + Volatility coefficient 'b' for `t > 0.5`. + alpha : float + Exponent for the drift denominator. + beta : float + Exponent for the volatility denominator. + explosion_filter_width : float, default=0.1 + Width of the "safe" area around the midpoint (0.5) + to prevent division by zero. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - drifts: The array of drift values. + - volatilities: The array of volatility values. + """ + steps = np.linspace(0, 1, bubble_length) + + # Clamp values near the midpoint to avoid division by zero + before_mask = (steps >= (0.5 - explosion_filter_width)) & (steps < 0.5) + after_mask = (steps > 0.5) & (steps <= (0.5 + explosion_filter_width)) + + steps_filtered = np.copy(steps) + steps_filtered[before_mask] = 0.5 - explosion_filter_width + steps_filtered[after_mask] = 0.5 + explosion_filter_width + + # Assign 'a' and 'b' parameters based on position + a_values = np.where(steps <= 0.5, a_before, a_after) + b_values = np.where(steps <= 0.5, b_before, b_after) + + # Calculate denominator, with NaN at the exact midpoint + denominators = np.abs(steps_filtered - 0.5) + denominators[steps == 0.5] = np.nan + + drifts = a_values / (denominators**alpha) + volatilities = b_values / (denominators**beta) + + # Handle the NaN at the midpoint by filling with 0 drift + # and the preceding volatility value. + nan_mask = np.isnan(denominators) + if np.any(nan_mask): + drifts[nan_mask] = 0.0 + # Find first NaN index and fill with previous value + nan_index = np.where(nan_mask)[0][0] + if nan_index > 0: + volatilities[nan_mask] = volatilities[nan_index - 1] + else: + # Handle case where midpoint is the first element + volatilities[nan_mask] = b_before # Fallback + + return drifts, volatilities diff --git a/src/RiskLabAI/data/synthetic_data/simulation.py b/src/RiskLabAI/data/synthetic_data/simulation.py new file mode 100644 index 0000000..973c1c2 --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/simulation.py @@ -0,0 +1,103 @@ +""" +Functions for generating synthetic, structured data for financial simulations. +Includes functions for creating random covariance matrices and +simulating multivariate normal observations. +""" + +import numpy as np +import pandas as pd +from scipy.linalg import block_diag +from sklearn.covariance import LedoitWolf + +# Import the utility from the denoising module +from RiskLabAI.data.denoise.denoising import corr_to_cov + + +def random_cov(num_columns: int, num_factors: int) -> np.ndarray: + """ + Generate a random covariance matrix. + + :param num_columns: Number of columns in the covariance matrix + :type num_columns: int + :param num_factors: Number of factors for random covariance matrix + :type num_factors: int + :return: Random covariance matrix + :rtype: np.ndarray + """ + w = np.random.normal(size=(num_columns, num_factors)) + cov = np.dot(w, w.T) + cov += np.diag(np.random.uniform(size=num_columns)) + return cov + + +def form_block_matrix( + n_blocks: int, block_size: int, block_correlation: float +) -> np.ndarray: + """ + Forms a block diagonal correlation matrix. + + :param n_blocks: Number of blocks + :type n_blocks: int + :param block_size: Size of each block + :type block_size: int + :param block_correlation: Correlation within each block + :type block_correlation: float + :return: Block diagonal correlation matrix + :rtype: np.ndarray + """ + block = np.ones((block_size, block_size)) * block_correlation + block[range(block_size), range(block_size)] = 1 + corr = block_diag(*([block] * n_blocks)) + return corr + + +def form_true_matrix( + n_blocks: int, block_size: int, block_correlation: float +) -> tuple[np.ndarray, np.ndarray]: + """ + Forms a shuffled block diagonal correlation matrix and the + corresponding covariance matrix. + + :param n_blocks: Number of blocks + :type n_blocks: int + :param block_size: Size of each block + :type block_size: int + :param block_correlation: Correlation within each block + :type block_correlation: float + :return: (mu0, cov0) Mean vector and Covariance matrix + :rtype: Tuple[np.ndarray, np.ndarray] + """ + corr0 = form_block_matrix(n_blocks, block_size, block_correlation) + corr0 = pd.DataFrame(corr0) + cols = corr0.columns.tolist() + np.random.shuffle(cols) + corr0 = corr0[cols].loc[cols].copy(deep=True).values + + std0 = np.random.uniform(0.05, 0.2, corr0.shape[0]) + cov0 = corr_to_cov(corr0, std0) + mu0 = np.random.normal(std0, std0, cov0.shape[0]).reshape(-1, 1) + return mu0, cov0 + + +def simulates_cov_mu( + mu0: np.ndarray, cov0: np.ndarray, n_obs: int, shrink: bool = False +) -> tuple[np.ndarray, np.ndarray]: + """ + Simulates multivariate normal observations and computes the + sample mean and covariance. + + :param mu0: True mean + :type mu0: np.ndarray + :param cov0: True covariance matrix + :type cov0: np.ndarray + :param n_obs: Number of observations + :type n_obs: int + :param shrink: Whether to use Ledoit-Wolf shrinkage, default False + :type shrink: bool + :return: (mu1, cov1) Sample mean and covariance matrix + :rtype: Tuple[np.ndarray, np.ndarray] + """ + x = np.random.multivariate_normal(mu0.flatten(), cov0, size=n_obs) + mu1 = x.mean(axis=0).reshape(-1, 1) + cov1 = LedoitWolf().fit(x).covariance_ if shrink else np.cov(x, rowvar=0) + return mu1, cov1 diff --git a/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py b/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py new file mode 100644 index 0000000..1372afe --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py @@ -0,0 +1,431 @@ +""" +Generates synthetic price data using a Heston-Merton model +with Markov-switching regimes. +""" + +from typing import Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + +# Type hint for regime parameters +RegimeParams = dict[str, Union[float, list[float]]] + + +@jit(nopython=True) +def compute_log_returns( + n_steps: int, + mu_vector: np.ndarray, + kappa_vector: np.ndarray, + theta_vector: np.ndarray, + xi_vector: np.ndarray, + dw_stock: np.ndarray, + dw_vol: np.ndarray, + jump_comp: np.ndarray, + poisson_var: np.ndarray, + dt: float, + sqrt_dt: float, + lambda_vector: np.ndarray, + m_vector: np.ndarray, + v_vector: np.ndarray, + regime_change: np.ndarray, +) -> np.ndarray: + r""" + Compute log returns using the Heston-Merton model (Numba-optimized). + + Heston Model (Volatility): + .. math:: + v_{i+1} = v_i + \kappa_i (\theta_i - \max(v_i, 0)) dt + + \xi_i \sqrt{\max(v_i, 0)} dwV_i \sqrt{dt} + + Merton Jump-Diffusion (Price): + .. math:: + \log(S_{i+1}/S_i) = \left(\mu_i - 0.5 v_i - \lambda_i (m_i + + \frac{v^2_i}{2})\right) dt + + \sqrt{v_i} dwS_i \sqrt{dt} + dJ_i + + Parameters + ---------- + n_steps : int + Number of steps (N). + mu_vector : np.ndarray + Drift vector. + kappa_vector : np.ndarray + Mean-reversion speed vector. + theta_vector : np.ndarray + Long-term mean vector for volatility. + xi_vector : np.ndarray + Volatility of volatility vector. + dw_stock : np.ndarray + Wiener process for stock. + dw_vol : np.ndarray + Wiener process for volatility. + jump_comp : np.ndarray + Jump component (Y). + poisson_var : np.ndarray + Poisson random variable vector (n). + dt : float + Time step. + sqrt_dt : float + Square root of the time step. + lambda_vector : np.ndarray + Jump intensity vector. + m_vector : np.ndarray + Mean of jump size vector. + v_vector : np.ndarray + Variance of jump size vector. + regime_change : np.ndarray + Boolean array, True if regime changed at this step. + + Returns + ------- + np.ndarray + Array of log returns. + """ + v = np.empty(n_steps + 1) + jump_events = np.multiply(poisson_var, jump_comp) + v[0] = theta_vector[0] # Initial volatility + log_returns = np.empty(n_steps) + + for i in range(n_steps): + if regime_change[i]: + v[i] = theta_vector[i] # Reset vol to new regime's long-term mean + + # Ensure v[i] is non-negative for sqrt + v_i_safe = max(v[i], 0.0) + + # Volatility process (Heston) + v[i + 1] = ( + v[i] + + kappa_vector[i] * (theta_vector[i] - v_i_safe) * dt + + xi_vector[i] * np.sqrt(v_i_safe) * dw_vol[i] * sqrt_dt + ) + + # Log return (Merton Jump-Diffusion) + log_returns[i] = ( + ( + mu_vector[i] + - 0.5 * v_i_safe + - lambda_vector[i] * (m_vector[i] + (v_vector[i] ** 2) / 2.0) + ) + * dt + + np.sqrt(v_i_safe) * dw_stock[i] * sqrt_dt + + jump_events[i] + ) + + return log_returns + + +def heston_merton_log_returns( + total_time: float, + n_steps: int, + mu_vector: np.ndarray, + kappa_vector: np.ndarray, + theta_vector: np.ndarray, + xi_vector: np.ndarray, + rho_vector: np.ndarray, + lambda_vector: np.ndarray, + m_vector: np.ndarray, + v_vector: np.ndarray, + regime_change: np.ndarray, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate Heston-Merton log returns. + + This function sets up the correlated random numbers and calls + the Numba-jitted `compute_log_returns`. + + Parameters + ---------- + total_time : float + Total time (T). + n_steps : int + Number of steps (N). + *..._vector : np.ndarray + Arrays of model parameters (see `compute_log_returns`). + regime_change : np.ndarray + Boolean array, True if regime changed at this step. + random_state : int, optional + Seed for the random number generator. + + Returns + ------- + np.ndarray + Array of log returns. + """ + # Ensure vectors are of length n_steps + params = [ + mu_vector, + kappa_vector, + theta_vector, + xi_vector, + rho_vector, + lambda_vector, + m_vector, + v_vector, + ] + if not all(len(p) == n_steps for p in params): + raise ValueError("All parameter vectors must have length `n_steps`") + + rng = np.random.default_rng(random_state) + dt = total_time / n_steps + sqrt_dt = np.sqrt(dt) + + # Generate correlated random numbers + z = np.zeros((n_steps, 3)) + n = np.zeros(n_steps) + for i in range(n_steps): + cov_matrix = np.array( + [ + [1.0, rho_vector[i], 0.0], + [rho_vector[i], 1.0, 0.0], + [0.0, 0.0, v_vector[i] ** 2], + ] + ) + z[i] = rng.multivariate_normal([0.0, 0.0, m_vector[i]], cov_matrix) + n[i] = rng.poisson(lambda_vector[i] * dt) + + dw_stock = z[:, 0] + dw_vol = z[:, 1] + jump_comp = z[:, 2] + + return compute_log_returns( + n_steps, + mu_vector, + kappa_vector, + theta_vector, + xi_vector, + dw_stock, + dw_vol, + jump_comp, + n, + dt, + sqrt_dt, + lambda_vector, + m_vector, + v_vector, + regime_change, + ) + + +def align_params_length( + regime_params: RegimeParams, +) -> tuple[dict[str, list[float]], int]: + """ + Align the parameter lists within a regime to be the same length. + + If a parameter is a float, it's broadcast to a list. + If a parameter is a list shorter than the max, it's extended + by repeating its last value. + + Parameters + ---------- + regime_params : RegimeParams + Dictionary of regime parameters. + + Returns + ------- + Tuple[Dict[str, List[float]], int] + - The aligned regime parameter dictionary. + - The maximum length (number of steps) for this regime. + """ + max_len = max(len(v) if isinstance(v, list) else 1 for v in regime_params.values()) + + aligned_params: dict[str, list[float]] = {} + for key, value in regime_params.items(): + if isinstance(value, list): + if len(value) < max_len: + # Extend list by repeating last value + aligned_params[key] = value + [value[-1]] * (max_len - len(value)) + else: + aligned_params[key] = value[:max_len] # Truncate if too long + else: + # Broadcast float to list + aligned_params[key] = [value] * max_len + + return aligned_params, max_len + + +def generate_prices_from_regimes( + regimes: dict[str, RegimeParams], + transition_matrix: np.ndarray, + total_time: float, + n_steps: int, + random_state: Optional[int] = None, +) -> tuple[pd.Series, np.ndarray]: + """ + Generate a price series from a Markov-switching regime model. + + Parameters + ---------- + regimes : Dict[str, RegimeParams] + Dictionary mapping regime names to their parameter sets. + transition_matrix : np.ndarray + The Markov Chain transition matrix. + total_time : float + Total time for the simulation (e.g., 1.0 for 1 year). + n_steps : int + Number of discrete steps in the simulation. + random_state : int, optional + Seed for random number generation. + + Returns + ------- + Tuple[pd.Series, np.ndarray] + - The generated price series, indexed by business days. + - The array of simulated regime names for each step. + """ + rng = np.random.default_rng(random_state) + + # 1. Simulate the Markov Chain + import quantecon.markov as qe # optional dependency: RiskLabAI[synth] + + regime_names = list(regimes.keys()) + markov_chain = qe.MarkovChain(transition_matrix, state_values=regime_names) + simulated_regimes = markov_chain.simulate(ts_length=n_steps, random_state=rng) + + # 2. Unpack parameters based on simulated regimes + param_lists: dict[str, list[float]] = { + "mu": [], + "kappa": [], + "theta": [], + "xi": [], + "rho": [], + "lam": [], + "m": [], + "v": [], + } + + regime_path_expanded = [] + + current_step = 0 + while current_step < n_steps: + regime_name = simulated_regimes[current_step] + params, regime_len = align_params_length(regimes[regime_name].copy()) + + steps_to_take = min(regime_len, n_steps - current_step) + + for key in param_lists: + param_lists[key].extend(params[key][:steps_to_take]) + + regime_path_expanded.extend([regime_name] * steps_to_take) + current_step += steps_to_take + + # 3. Finalize parameter arrays and regime path + simulated_regimes_final = np.array(regime_path_expanded) + param_arrays: dict[str, np.ndarray] = { + key: np.array(val) for key, val in param_lists.items() + } + + # 4. Identify regime change points + regime_change = simulated_regimes_final[1:] != simulated_regimes_final[:-1] + regime_change = np.pad( + regime_change, (1, 0), mode="constant", constant_values=False + ) + + # 5. Generate log returns + log_returns = heston_merton_log_returns( + total_time, + n_steps, + param_arrays["mu"], + param_arrays["kappa"], + param_arrays["theta"], + param_arrays["xi"], + param_arrays["rho"], + param_arrays["lam"], + param_arrays["m"], + param_arrays["v"], + regime_change, + random_state=rng.integers(0, 1e6), + ) + + # 6. Create price series with a Business Day index + start_day = "2000-01-01" + business_days = pd.date_range(start=start_day, periods=n_steps, freq="B") + + price_series = pd.Series(log_returns, index=business_days).ffill() + prices = 100 * np.exp(price_series.cumsum()) + prices.name = "Price" + + return prices, simulated_regimes_final + + +def parallel_generate_prices( + number_of_paths: int, + regimes: dict[str, RegimeParams], + transition_matrix: np.ndarray, + total_time: float, + n_steps: int, + random_state: Optional[int] = None, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Parallel generation of price paths. + + Parameters + ---------- + number_of_paths : int + The number of paths to generate. + regimes : Dict[str, RegimeParams] + Dictionary of regime parameters. + transition_matrix : np.ndarray + Markov Chain transition matrix. + total_time : float + Total time for the simulation. + number_of_steps : int + Number of steps in the simulation. + random_state : int, optional + Seed for *main* random number generator (which generates seeds). + n_jobs : int, default=1 + Number of parallel jobs to run. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - prices_df: DataFrame of (n_steps x number_of_paths) prices. + - regimes_df: DataFrame of (n_steps x number_of_paths) regimes. + """ + rng = np.random.default_rng(random_state) + # Generate unique seeds for each parallel job + random_states = rng.integers(0, 10 * number_of_paths, size=number_of_paths) + + results = Parallel(n_jobs=n_jobs)( + delayed(generate_prices_from_regimes)( + regimes, + transition_matrix, + total_time, + n_steps, + seed, + ) + for seed in random_states + ) + + prices, simulated_regimes = zip(*results) + + prices_df = pd.concat(prices, axis=1) + prices_df.columns = range(number_of_paths) + + simulated_regimes_df = pd.DataFrame(simulated_regimes).T + simulated_regimes_df.columns = range(number_of_paths) + simulated_regimes_df.index = prices_df.index + + return prices_df, simulated_regimes_df diff --git a/src/RiskLabAI/data/weights/__init__.py b/src/RiskLabAI/data/weights/__init__.py new file mode 100644 index 0000000..5788242 --- /dev/null +++ b/src/RiskLabAI/data/weights/__init__.py @@ -0,0 +1,24 @@ +""" +RiskLabAI Sample Weights Module + +Implements sample weighting techniques for financial machine learning, +focusing on uniqueness (concurrency) and time decay. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +from .sample_weights import ( + calculate_average_uniqueness, + calculate_time_decay, + expand_label_for_meta_labeling, + sample_weight_absolute_return_meta_labeling, +) + +__all__ = [ + "expand_label_for_meta_labeling", + "calculate_average_uniqueness", + "sample_weight_absolute_return_meta_labeling", + "calculate_time_decay", +] diff --git a/src/RiskLabAI/data/weights/sample_weights.py b/src/RiskLabAI/data/weights/sample_weights.py new file mode 100644 index 0000000..d85988d --- /dev/null +++ b/src/RiskLabAI/data/weights/sample_weights.py @@ -0,0 +1,234 @@ +""" +Implements sample weighting techniques for financial machine learning, +focusing on uniqueness (concurrency) and time decay. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +import numpy as np +import pandas as pd + + +def expand_label_for_meta_labeling( + close_index: pd.Index, + timestamp: pd.Series, + molecule: pd.Index, +) -> pd.Series: + """ + Compute the number of concurrent events for each timestamp. + + This function expands the event start/end times (`timestamp`) + to a full series indicating how many events are active + at each point in time. + + Reference: + Based on Snippet 4.1, Page 60 (label concurrency). + + Parameters + ---------- + close_index : pd.Index + The master index of all timestamps (e.g., from price bars). + timestamp : pd.Series + Series where index is event start time, value is event end time. + molecule : pd.Index + The subset of event start times to process. + + Returns + ------- + pd.Series + A Series indexed by `close_index` where each value is the + count of active events at that timestamp. + """ + # Filter events that are relevant to this molecule + ts = timestamp.fillna(close_index[-1]) + ts = ts[ts.index.isin(molecule)] + ts = ts[ts > molecule[0]] + + if ts.empty: + # Return an empty series; align in the caller will handle it + return pd.Series(dtype=float) + + # Find min/max index locations + iloc_min = close_index.searchsorted(ts.index[0]) + iloc_max = close_index.searchsorted(ts.max()) + + # Create a count series over the relevant time span + count = pd.Series(0, index=close_index[iloc_min : iloc_max + 1]) + + for t_in, t_out in ts.items(): + # Ensure t_out is within the count index + # (it might be larger than iloc_max if fillna was used) + t_out = min(t_out, count.index[-1]) + count.loc[t_in:t_out] += 1 + + return count.loc[molecule[0] : ts.max()] + + +def calculate_average_uniqueness( + index_matrix: pd.DataFrame, +) -> pd.Series: + """ + Calculate the average uniqueness of each event. + + Uniqueness is calculated as 1/c_t, averaged over the + duration of the event. + + Reference: + Snippet 4.2, Page 62. + + Parameters + ---------- + index_matrix : pd.DataFrame + An indicator matrix (T x N) where T is timestamps + and N is the number of events. + + Returns + ------- + pd.Series + A Series of average uniqueness for each event (column). + """ + # c_t: Concurrency at each timestamp + concurrency = index_matrix.sum(axis=1) + + # 1/c_t: Uniqueness at each timestamp + # This is a (T x N) DataFrame, 0 where event is not active + uniqueness = index_matrix.div(concurrency, axis=0).fillna(0) + + # Sum of 1/c_t for each event + total_uniqueness = uniqueness.sum(axis=0) + + # Number of active periods for each event + event_duration = (index_matrix > 0).sum(axis=0) + + # Average uniqueness: sum(1/c_t) / sum(I) + average_uniqueness = total_uniqueness / event_duration + + # Handle events that never occurred (duration 0) + average_uniqueness = average_uniqueness.fillna(0) + + return average_uniqueness + + +def sample_weight_absolute_return_meta_labeling( + timestamp: pd.Series, price: pd.Series, molecule: pd.Index +) -> pd.Series: + """ + Calculate sample weights based on absolute log-return attribution. + + The weight of a sample is proportional to the sum of the absolute + log-returns that occurred during the event, divided by the + concurrency of those returns. + + w_i = sum_{t=t_i,0}^{t_i,1} [ |r_t| / c_t ] + + Weights are then normalized to sum to the number of samples. + + Reference: + Based on Snippet 4.10, Page 70. + + Parameters + ---------- + timestamp : pd.Series + Series where index is event start time, value is event end time. + price : pd.Series + Series of prices. + molecule : pd.Index + The subset of event start times to process. + + Returns + ------- + pd.Series + A Series of sample weights, normalized to sum to N. + """ + # 1. Compute concurrency + concurrency_events = expand_label_for_meta_labeling( + price.index, timestamp, molecule + ) + + # 2. Compute absolute log returns + log_return = np.log(price).diff().abs() + + # Align returns and concurrency + # Use 'left' join to keep the log_return (price) index + log_return, concurrency_events = log_return.align( + concurrency_events, join="left", fill_value=0 + ) + + weight = pd.Series(index=molecule, dtype=float) + + # 3. Calculate weighted returns + for t_in, t_out in timestamp.loc[weight.index].items(): + if t_out not in log_return.index: + # Find the closest preceding index + t_out = log_return.index[log_return.index.searchsorted(t_out) - 1] + + # r_t / c_t + # Filter concurrency > 0 to avoid division by zero + relevant_concurrency = concurrency_events.loc[t_in:t_out] + relevant_log_return = log_return.loc[t_in:t_out] + + active_periods = relevant_concurrency > 0 + if active_periods.any(): + weighted_return = ( + relevant_log_return[active_periods] + / relevant_concurrency[active_periods] + ) + weight.loc[t_in] = weighted_return.sum() + else: + weight.loc[t_in] = 0.0 + + weight = weight.abs() + + # 4. Normalize + if weight.sum() == 0: + # Avoid division by zero if all weights are 0 + return pd.Series(1.0, index=molecule) + + weight *= len(weight) / weight.sum() + return weight + + +def calculate_time_decay(weight: pd.Series, clf_last_weight: float = 1.0) -> pd.Series: + """ + Apply a time-decay factor to sample weights. + + This function applies a linearly decaying weight, where the + most recent observation has weight 1.0 and the oldest + has weight `clf_last_weight`. + + Reference: + Snippet 4.11, Page 71. + + Parameters + ---------- + weight : pd.Series + The original sample weights (e.g., from uniqueness). + clf_last_weight : float, default=1.0 + The weight to assign to the oldest observation. + If 1.0, all weights are 1 (no decay). + If 0.0, weights decay linearly to 0. + + Returns + ------- + pd.Series + The new weights with time decay applied. + """ + clf_weight = weight.sort_index().cumsum() + + if clf_last_weight < 0 or clf_last_weight > 1: + raise ValueError("clf_last_weight must be between 0 and 1") + + # Calculate slope + if clf_last_weight == 1.0 or clf_weight.empty or clf_weight.iloc[-1] == 0: + slope = 0.0 + const = 1.0 + else: + slope = (1.0 - clf_last_weight) / clf_weight.iloc[-1] + const = 1.0 - slope * clf_weight.iloc[-1] + + clf_weight = const + slope * clf_weight + clf_weight[clf_weight < 0] = 0.0 # Should not happen if clf_last_weight >= 0 + + return clf_weight diff --git a/src/RiskLabAI/ensemble/__init__.py b/src/RiskLabAI/ensemble/__init__.py new file mode 100644 index 0000000..8ff0b5c --- /dev/null +++ b/src/RiskLabAI/ensemble/__init__.py @@ -0,0 +1,24 @@ +""" +RiskLabAI Ensemble Module + +Implements ensemble-related methods, such as the theoretical +accuracy of bagging classifiers. +""" + +# From the theoretical function file (bagging_classifier_accuracy.py) +from .bagging_classifier_accuracy import bagging_classifier_accuracy + +# From the empirical class file (empirical_bagging_accuracy.py) +from .empirical_bagging_accuracy import ( + BaggingClassifierAccuracy, + calculate_bootstrap_accuracy, + plot_bootstrap_accuracy_distribution, +) + +# Expose all names for import +__all__ = [ + "bagging_classifier_accuracy", # The function + "BaggingClassifierAccuracy", # The class + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution", +] diff --git a/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py b/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py new file mode 100644 index 0000000..141d580 --- /dev/null +++ b/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py @@ -0,0 +1,45 @@ +# RiskLabAI/ensemble/bagging_classifier_accuracy.py +""" +Implements the theoretical accuracy of a bagging classifier, +as described in "Advances in Financial Machine Learning" by de Prado (2018), +Chapter 6, Section 6.2, p. 86. +""" + +# import numpy as np <-- Removed unused import +from scipy.stats import binom + + +def bagging_classifier_accuracy(N: int, p: float) -> float: + """ + Calculates the theoretical accuracy of a bagging classifier + assuming independent classifiers. + + The bagging classifier is correct if the majority of + classifiers are correct. + + Parameters + ---------- + N : int + Number of classifiers (must be odd). + p : float + Accuracy of a single classifier (0 <= p <= 1). + + Returns + ------- + float + The accuracy of the ensemble. + """ + if N % 2 == 0: + raise ValueError(f"Number of estimators N must be odd. Got {N}.") + + # The majority threshold. + # e.g., if N=101, k=50. We need 51 or more correct votes. + k = (N - 1) // 2 + + # Probability of k or fewer successes (P(X <= k)) + prob_majority_wrong = binom.cdf(k, N, p) + + # Probability of more than k successes (P(X > k)) + prob_majority_correct = 1.0 - prob_majority_wrong + + return prob_majority_correct diff --git a/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py b/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py new file mode 100644 index 0000000..e59cb5d --- /dev/null +++ b/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py @@ -0,0 +1,377 @@ +""" +Implements the BaggingClassifierAccuracy class to evaluate the +performance of a bagging classifier using various weighting schemes, +as described in "Advances in Financial Machine Learning" by de Prado (2018), +Chapter 6, Section 6.5, pp. 91-92. +""" + +from typing import TYPE_CHECKING, Optional + +import numpy as np +import pandas as pd +from scipy.stats import norm + +if TYPE_CHECKING: + import matplotlib.pyplot as plt # type hints only; optional at runtime +from sklearn.ensemble import BaggingClassifier +from sklearn.exceptions import NotFittedError +from sklearn.metrics import accuracy_score +from sklearn.tree import DecisionTreeClassifier + + +class BaggingClassifierAccuracy: + """ + Evaluates a bagging classifier's accuracy using different + weighting schemes based on decision tree c_i scores. + + Methods: + - fit: Fits the bagging classifier. + - calculate_c_i: Calculates the c_i score for each tree. + - calculate_weights: Computes weights (uniform, c_i, 1-c_i^2). + - predict: Predicts class labels using specified weights. + - evaluate_all_schemes: Gets accuracy for all weighting schemes. + """ + + def __init__( + self, + n_estimators: int = 1000, + max_samples: int = 100, + max_features: float = 1.0, + random_state: Optional[int] = None, + ): + """ + Initializes the BaggingClassifier. + + Note: This class uses a specific base_estimator: + DecisionTreeClassifier(criterion='entropy', max_features=1, + class_weight='balanced') as per the book's context. + + Parameters + ---------- + n_estimators : int, default=1000 + Number of trees in the ensemble. + max_samples : int, default=100 + Number of samples to draw for training each tree. + max_features : float, default=1.0 + Number of features to draw for training each tree. + random_state : int, optional + Seed for reproducibility. + """ + self.n_estimators = n_estimators + self.max_samples = max_samples + self.max_features = max_features + self.random_state = random_state + + self.base_estimator = DecisionTreeClassifier( + criterion="entropy", + max_features=1, # Trees vote on one feature + class_weight="balanced", + ) + + self.clf = BaggingClassifier( + estimator=self.base_estimator, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + random_state=self.random_state, + ) + + self.estimators_ = None + self.weights_ = None + self.c_i_scores_ = None + # <-- ADDED: Store class labels + self.class_0_ = None + self.class_1_ = None + + def fit(self, X: pd.DataFrame, y: pd.Series) -> "BaggingClassifierAccuracy": + """ + Fits the bagging classifier on the training data. + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + self + Fitted object. + """ + self.clf.fit(X, y) + self.estimators_ = self.clf.estimators_ + + # <-- ADDED: Check for binary classification and store classes + if len(self.clf.classes_) != 2: + raise ValueError("This class only supports binary classification.") + + self.class_0_ = self.clf.classes_[0] + self.class_1_ = self.clf.classes_[1] + + return self + + def calculate_c_i(self, X: pd.DataFrame, y: pd.Series) -> np.ndarray: + """ + Calculates the c_i score (accuracy) for each decision tree + on the full training set. + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + np.ndarray + Array of c_i scores for each estimator. + """ + if self.estimators_ is None: + raise NotFittedError("Classifier must be fitted first. Call .fit()") + + c_i_scores = [] + for tree in self.estimators_: + y_pred = tree.predict(X) + acc = accuracy_score(y, y_pred) + c_i_scores.append(acc) + + self.c_i_scores_ = np.array(c_i_scores) + return self.c_i_scores_ + + def calculate_weights(self, X: pd.DataFrame, y: pd.Series) -> dict[str, np.ndarray]: + """ + Calculates weights for each estimator based on three schemes: + 1. Uniform (w_i = 1/N) + 2. c_i (w_i = c_i / sum(c_i)) + 3. 1 - c_i^2 (w_i = (1 - c_i^2) / sum(1 - c_i^2)) + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + Dict[str, np.ndarray] + Dictionary of weight arrays for each scheme. + """ + if self.c_i_scores_ is None: + # calculate_c_i also checks if model is fitted + self.calculate_c_i(X, y) + + c_i = self.c_i_scores_ + n = len(c_i) + + # 1. Uniform weights + w_uniform = np.ones(n) / n + + # 2. c_i weights + sum_c_i = np.sum(c_i) + w_c_i = c_i / sum_c_i if sum_c_i != 0 else w_uniform + + # 3. 1 - c_i^2 weights (proportional to variance) + c_i_squared = c_i**2 + w_variance = 1.0 - c_i_squared + sum_w_var = np.sum(w_variance) + w_variance = w_variance / sum_w_var if sum_w_var != 0 else w_uniform + + self.weights_ = {"uniform": w_uniform, "c_i": w_c_i, "variance": w_variance} + return self.weights_ + + def predict(self, X: pd.DataFrame, weight_scheme: str = "uniform") -> np.ndarray: + """ + Predicts class labels for X using the specified weighting scheme. + + Parameters + ---------- + X : pd.DataFrame + Features to predict on. + weight_scheme : str, default='uniform' + The weighting scheme to use ('uniform', 'c_i', 'variance'). + + Returns + ------- + np.ndarray + Predicted class labels. + """ + # <-- UPDATED: Check fit status first + if self.estimators_ is None: + raise NotFittedError("Classifier must be fitted first. Call .fit()") + + if self.weights_ is None: + # <-- UPDATED: More specific error + raise NotFittedError( + "Weights must be calculated first. Call .calculate_weights()" + ) + + if weight_scheme not in self.weights_: + raise ValueError( + f"Unknown weight_scheme: {weight_scheme}. " + f"Must be one of {list(self.weights_.keys())}" + ) + + weights = self.weights_[weight_scheme] + + # Get predictions from each tree + # (N_samples, N_estimators) + tree_preds = np.array([tree.predict(X) for tree in self.estimators_]).T + + # <-- UPDATED: Convert labels {class_0, class_1} to {-1, 1} + # Map class_1 to 1, and class_0 to -1 + tree_preds_signed = np.where(tree_preds == self.class_1_, 1, -1) + + # Calculate weighted average vote + # (N_samples, N_estimators) * (N_estimators,) -> (N_samples,) + weighted_votes = np.dot(tree_preds_signed, weights) + + # <-- UPDATED: Convert vote back to {class_0, class_1} + # Positive vote -> class_1, Negative or Zero vote -> class_0 + y_pred = np.where(weighted_votes > 0, self.class_1_, self.class_0_) + + return y_pred + + def evaluate_all_schemes( + self, + X_test: pd.DataFrame, + y_test: pd.Series, + X_train: pd.DataFrame, + y_train: pd.Series, + ) -> dict[str, float]: + """ + Fits, calculates weights, and evaluates accuracy for all + three weighting schemes. + + Parameters + ---------- + X_test : pd.DataFrame + Test features. + y_test : pd.Series + Test labels. + X_train : pd.DataFrame + Training features (for calculating weights). + y_train : pd.Series + Training labels (for calculating weights). + + Returns + ------- + Dict[str, float] + Dictionary of accuracy scores for each scheme. + """ + # Fit classifier + self.fit(X_train, y_train) + + # Calculate weights (uses X_train, y_train implicitly) + self.calculate_weights(X_train, y_train) + + accuracies = {} + for scheme in self.weights_.keys(): + y_pred = self.predict(X_test, weight_scheme=scheme) + acc = accuracy_score(y_test, y_pred) + accuracies[scheme] = acc + + return accuracies + + +# --- Standalone Functions for Bootstrap Analysis --- + + +def calculate_bootstrap_accuracy( + clf: BaggingClassifier, X: pd.DataFrame, y: pd.Series, n_bootstraps: int = 1000 +) -> tuple[np.ndarray, float, float]: + """ + Calculates the accuracy of a bagging classifier over multiple + bootstrapped samples of the test set. + + This helps estimate the standard deviation of the accuracy (std(a_n)). + + Parameters + ---------- + clf : BaggingClassifier + A fitted BaggingClassifier instance. + X : pd.DataFrame + Test features. + y : pd.Series + Test labels. + n_bootstraps : int, default=1000 + Number of bootstrap iterations. + + Returns + ------- + Tuple[np.ndarray, float, float] + - a_n_values: Array of accuracy scores from each bootstrap. + - a_n_mean: The mean accuracy. + - a_n_std: The standard deviation of the accuracies. + """ + a_n_values = [] + n_samples = len(y) + + # Use indices from the original X/y DataFrames/Series + indices = X.index + + # --- CHANGE: Fixed typo n_bootstraMps -> n_bootstraps --- + for _ in range(n_bootstraps): + # --- END CHANGE --- + # Sample test set with replacement + boot_indices = np.random.choice(indices, n_samples, replace=True) + X_boot = X.loc[boot_indices] + y_boot = y.loc[boot_indices] + + # Predict on the bootstrapped sample + y_pred = clf.predict(X_boot) + acc = accuracy_score(y_boot, y_pred) + a_n_values.append(acc) + + a_n_values = np.array(a_n_values) + a_n_mean = np.mean(a_n_values) + a_n_std = np.std(a_n_values, ddof=1) + + return a_n_values, a_n_mean, a_n_std + + +def plot_bootstrap_accuracy_distribution( + a_n_values: np.ndarray, + a_n_mean: float, + a_n_std: float, + ax: Optional["plt.Axes"] = None, +) -> "plt.Axes": + """ + Plots the distribution of bootstrapped accuracy scores. + + Parameters + ---------- + a_n_values : np.ndarray + Array of accuracy scores from each bootstrap. + a_n_mean : float + The mean accuracy. + a_n_std : float + The standard deviation of the accuracies. + ax : plt.Axes, optional + Matplotlib Axes object to plot on. If None, creates a new one. + + Returns + ------- + plt.Axes + The Axes object with the plot. + """ + import matplotlib.pyplot as plt # optional dependency: RiskLabAI[plot] + import seaborn as sns # optional dependency: RiskLabAI[plot] + + if ax is None: + fig, ax = plt.subplots(figsize=(10, 6)) + + sns.histplot( + a_n_values, kde=True, ax=ax, stat="density", label="Empirical Distribution" + ) + + # Overlay a normal distribution + x_min, x_max = ax.get_xlim() + x = np.linspace(x_min, x_max, 100) + p = norm.pdf(x, a_n_mean, a_n_std) + ax.plot(x, p, "k", linewidth=2, label=f"Normal(μ={a_n_mean:.3f}, σ={a_n_std:.3f})") + + ax.legend() + return ax diff --git a/src/RiskLabAI/features/__init__.py b/src/RiskLabAI/features/__init__.py new file mode 100644 index 0000000..511d313 --- /dev/null +++ b/src/RiskLabAI/features/__init__.py @@ -0,0 +1,6 @@ +from . import ( + entropy_features, + feature_importance, + microstructural_features, + structural_breaks, +) diff --git a/src/RiskLabAI/features/entropy_features/__init__.py b/src/RiskLabAI/features/entropy_features/__init__.py new file mode 100644 index 0000000..96576ca --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/__init__.py @@ -0,0 +1,31 @@ +""" +RiskLabAI Entropy Features Module + +Provides various entropy estimators for financial time series, such +as Shannon, Lempel-Ziv, and Kontoyiannis, plus the bias-corrected family +(Miller-Madow, Grassberger, NSB) for undersampled symbol counts. +""" + +from .bias_corrected import ( + grassberger_entropy, + miller_madow_entropy, + nsb_entropy, +) +from .kontoyiannis import kontoyiannis_entropy, longest_match_length +from .lempel_ziv import lempel_ziv_entropy +from .plug_in import plug_in_entropy_estimator +from .pmf import ngram_counts, probability_mass_function +from .shannon import shannon_entropy + +__all__ = [ + "shannon_entropy", + "lempel_ziv_entropy", + "probability_mass_function", + "ngram_counts", + "plug_in_entropy_estimator", + "kontoyiannis_entropy", + "longest_match_length", + "miller_madow_entropy", + "grassberger_entropy", + "nsb_entropy", +] diff --git a/src/RiskLabAI/features/entropy_features/bias_corrected.py b/src/RiskLabAI/features/entropy_features/bias_corrected.py new file mode 100644 index 0000000..507c8ed --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/bias_corrected.py @@ -0,0 +1,293 @@ +""" +Bias-corrected Shannon entropy estimators (Miller-Madow, Grassberger, NSB). + +de Prado's plug-in / maximum-likelihood entropy (`plug_in_entropy_estimator`, AFML ch.18) is +systematically biased downward when the number of distinct symbols (the alphabet, here the number of +possible n-grams) is large relative to the sample, because unobserved symbols contribute zero. The +bias is approximately -(K - 1) / (2N), growing with the alphabet K and shrinking with the sample N. +These estimators add back the missing mass: Miller-Madow analytically to first order, Grassberger to +higher order, and NSB by integrating over a near-uniform-entropy Bayesian prior. + +All three operate on the same n-gram counts as the plug-in (via +:func:`~RiskLabAI.features.entropy_features.pmf.ngram_counts`) and return entropy in bits, normalized +by the word length, so they are drop-in comparable with `plug_in_entropy_estimator`. The plug-in's +behaviour is unchanged. + +Admitted in Appraisal 06 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag (preferred-when / avoid-when), +verbatim from the ledger: + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + NSB is most accurate in deep undersampling; Grassberger is close at lower cost (the practical + default when undersampled); Miller-Madow is the cheap first-order fix. All converge to the plug-in + when N is much larger than K with no over-correction, so a correction is a safe default and the + plug-in remains fine when well-sampled. The gain is negligible for coarse encodings (e.g. binary), + and a gap caused by non-stationarity (e.g. sigma encoding) is not an entropy-bias problem the + correction can fix. + +Evidence and caveats: appraisals/06_verdict.md. + +References +---------- +Miller, G. (1955) Note on the bias of information estimates. In Information Theory in Psychology. +Grassberger, P. (2008) Entropy estimates from insufficient samplings. arXiv:physics/0307138. +Nemenman, I., Shafee, F. and Bialek, W. (2002) Entropy and inference, revisited. NIPS 14. +Wolpert, D. H. and Wolf, D. R. (1995) Estimating functions of probability distributions from a + finite set of samples. Physical Review E, 52(6), 6841. +""" + +from functools import lru_cache +from typing import Optional + +import numpy as np +from scipy import integrate, optimize +from scipy.special import gammaln, psi + +from .pmf import ngram_counts + +_LN2 = np.log(2.0) + + +def _counts_array(message: str, approximate_word_length: int) -> np.ndarray: + """Positive n-gram counts of a message as a float array (empty if the message is too short).""" + counts = ngram_counts(message, approximate_word_length) + if not counts: + return np.zeros(0) + return np.asarray(list(counts.values()), dtype=np.float64) + + +def _plugin_bits(counts: np.ndarray) -> float: + """Plug-in / maximum-likelihood entropy (bits) of a count vector: H = -sum p log2 p.""" + n = counts.sum() + if n <= 0: + return 0.0 + p = counts / n + return float(-np.sum(p * np.log2(p))) + + +def miller_madow_entropy(message: str, approximate_word_length: int = 1) -> float: + """ + Miller-Madow bias-corrected Shannon entropy, normalized by word length (bits). + + The plug-in entropy plus the first-order analytic correction (K_hat - 1) / (2N), where K_hat is + the number of observed n-grams and N the number of n-grams counted. The cheapest correction; it + captures most of the gain at moderate undersampling. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + Miller-Madow is the cheap first-order fix; it converges to the plug-in when N is much larger than + K, with no over-correction. The gain is negligible for coarse encodings (e.g. binary). See the + module docstring for the full regime tag and appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string (e.g. a discretized time series "110100..."). + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it, matching `plug_in_entropy_estimator`. + + Returns + ------- + float + The Miller-Madow entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import miller_madow_entropy + >>> miller_madow_entropy("110100110101", approximate_word_length=1) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + n = counts.sum() + k_hat = counts.size + corrected = _plugin_bits(counts) + (k_hat - 1.0) / (2.0 * n * _LN2) + return corrected / approximate_word_length + + +def _grassberger_g(n: np.ndarray) -> np.ndarray: + """G(n) = psi(n) + 0.5 (-1)^n [psi((n+1)/2) - psi(n/2)] (Grassberger 2008), in nats.""" + sign = np.where(n.astype(np.int64) % 2 == 0, 1.0, -1.0) + return psi(n) + 0.5 * sign * (psi((n + 1.0) / 2.0) - psi(n / 2.0)) + + +def grassberger_entropy(message: str, approximate_word_length: int = 1) -> float: + """ + Grassberger (2008) bias-corrected Shannon entropy, normalized by word length (bits). + + Uses the digamma finite-sample correction H = ln N - (1/N) sum n_i G(n_i), converted to bits. A + higher-order correction with no tuning, intermediate in cost between Miller-Madow and NSB. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + Grassberger is close to NSB at lower cost (the practical default when undersampled); it converges + to the plug-in when N is much larger than K, with no over-correction. The gain is negligible for + coarse encodings (e.g. binary). See the module docstring for the full regime tag and + appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it. + + Returns + ------- + float + The Grassberger entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import grassberger_entropy + >>> grassberger_entropy("110100110101", approximate_word_length=1) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + n = counts.sum() + h_nats = np.log(n) - (1.0 / n) * np.sum(counts * _grassberger_g(counts)) + return float(h_nats / _LN2) / approximate_word_length + + +# --- NSB (Nemenman-Shafee-Bialek 2002) ------------------------------------------------------- +# +# Posterior mean entropy at a fixed Dirichlet concentration beta (Wolpert-Wolf 1995), integrated over +# beta against the NSB prior P(beta) proportional to d/dbeta E[H | beta], which makes the a priori mean +# entropy xi(beta) = psi0(K beta + 1) - psi0(beta + 1) near-uniform on (0, ln K). K is the true +# alphabet size; integration is in the xi variable (uniform prior) for stability, and the beta(xi) +# inversion depends only on (K, n_points) so it is cached. + + +def _xi_of_beta(beta: float, k: int) -> float: + """A priori mean entropy (nats) implied by concentration beta over K symbols.""" + return psi(k * beta + 1.0) - psi(beta + 1.0) + + +@lru_cache(maxsize=256) +def _nsb_beta_grid( + k: int, n_points: int +) -> tuple[tuple[float, ...], tuple[float, ...]]: + """ + Quadrature nodes for the NSB xi-integral over (0, ln K): cached ``(xi_nodes, beta_nodes)``. + + xi(beta) is monotone increasing from 0 (beta -> 0) to ln K (beta -> inf); each node is inverted + once with Brent's method. Depends only on (K, n_points), so it is cached across calls (the result + is returned as tuples to keep it hashable / immutable). + """ + xi_max = np.log(k) + xi_nodes = np.linspace( + xi_max / (n_points + 1), xi_max * n_points / (n_points + 1), n_points + ) + beta_nodes = np.empty(n_points) + for i, xi in enumerate(xi_nodes): + + def _f(beta: float, target: float = xi) -> float: + return _xi_of_beta(beta, k) - target + + beta_nodes[i] = optimize.brentq(_f, 1e-8, 1e8, xtol=1e-12) + return tuple(xi_nodes.tolist()), tuple(beta_nodes.tolist()) + + +def _log_evidence(beta: np.ndarray, k: int, n: float, counts: np.ndarray) -> np.ndarray: + """Unnormalised log marginal likelihood at each concentration in ``beta`` (nats, vectorised).""" + b = beta[:, None] + obs_term = np.sum(gammaln(counts[None, :] + b) - gammaln(b), axis=1) + return gammaln(k * beta) - gammaln(n + k * beta) + obs_term + + +def _mean_entropy_given_beta( + beta: np.ndarray, k: int, n: float, counts: np.ndarray, k_obs: int +) -> np.ndarray: + """ + Posterior mean entropy (nats) at each concentration in ``beta`` (Wolpert-Wolf 1995). + + As beta -> 0 this tends to psi(N + 1) - (1/N) sum n_i psi(n_i + 1), the maximum-likelihood-like + Bayesian limit; as beta -> inf it tends to the entropy of the uniform distribution over K. + """ + denom = n + k * beta + b = beta[:, None] + observed = np.sum((counts[None, :] + b) * psi(counts[None, :] + b + 1.0), axis=1) + unobserved = (k - k_obs) * beta * psi(beta + 1.0) + return psi(denom + 1.0) - (observed + unobserved) / denom + + +def _nsb_from_counts(counts: np.ndarray, k: int, n_points: int) -> float: + """NSB posterior mean entropy (bits) for a count vector and true alphabet size K.""" + n = counts.sum() + if n <= 0 or k <= 1: + return 0.0 + k_obs = counts.size + + xi_tuple, beta_tuple = _nsb_beta_grid(k, n_points) + xi_nodes = np.asarray(xi_tuple) + beta = np.asarray(beta_tuple) + + log_ev = _log_evidence(beta, k, n, counts) + mean_h = _mean_entropy_given_beta(beta, k, n, counts, k_obs) + + # Uniform prior in xi; stabilise the evidence by its maximum before exponentiating. + log_ev -= np.max(log_ev) + weight = np.exp(log_ev) + numerator = integrate.trapezoid(weight * mean_h, xi_nodes) + normaliser = integrate.trapezoid(weight, xi_nodes) + if normaliser <= 0 or not np.isfinite(normaliser): + return float("nan") + return float((numerator / normaliser) / _LN2) + + +def nsb_entropy( + message: str, + approximate_word_length: int = 1, + alphabet_size: Optional[int] = None, + n_points: int = 80, +) -> float: + """ + NSB (Nemenman-Shafee-Bialek 2002) Bayesian Shannon entropy, normalized by word length (bits). + + The Wolpert-Wolf posterior mean entropy integrated over the Dirichlet concentration against a + prior chosen so the implied prior on entropy is near-uniform. NSB needs the true alphabet size; it + is the most accurate estimator in deep undersampling, and the most expensive. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + NSB is most accurate in deep undersampling; it converges to the plug-in when N is much larger than + K, with no over-correction. The gain is negligible for coarse encodings (e.g. binary), and a gap + caused by non-stationarity (e.g. sigma encoding) is not an entropy-bias problem the correction can + fix. See the module docstring for the full regime tag and appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it. + alphabet_size : int, optional + The number of distinct base symbols the encoding can produce. The effective alphabet of words + is ``alphabet_size ** approximate_word_length``. Defaults to the number of distinct symbols + observed in the message (a lower bound; pass the encoding's true alphabet when known). + n_points : int, default 80 + Number of quadrature nodes for the Bayesian integral. + + Returns + ------- + float + The NSB entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import nsb_entropy + >>> nsb_entropy("110100110101", approximate_word_length=1, alphabet_size=2) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + base = alphabet_size if alphabet_size is not None else len(set(message)) + base = max(int(base), 1) + k_eff = base**approximate_word_length + # The effective alphabet cannot be smaller than the number of distinct words observed. + k_eff = max(k_eff, counts.size) + return _nsb_from_counts(counts, k_eff, n_points) / approximate_word_length diff --git a/src/RiskLabAI/features/entropy_features/kontoyiannis.py b/src/RiskLabAI/features/entropy_features/kontoyiannis.py new file mode 100644 index 0000000..aa976fe --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/kontoyiannis.py @@ -0,0 +1,109 @@ +""" +Implements the Kontoyiannis Entropy estimator (LZ-based). +""" + +from math import log2 +from typing import Optional + + +def longest_match_length(message: str, i: int, n: int) -> tuple[int, str]: + """ + Find the length of the longest match for the substring starting at `i`. + + This function searches for the longest match of `message[i:i+l]` + within the preceding window `message[max(0, i-n):i]`. + + Parameters + ---------- + message : str + Input string. + i : int + The starting index of the substring to match. + n : int + The length of the look-back window. + + Returns + ------- + Tuple[int, str] + - The length of the longest match (L_i) + 1. + - The matched substring. + """ + longest_match = "" + # Iterate through possible match lengths + for length in range(1, n + 1): + pattern = message[i : i + length] + + # Stop if pattern goes beyond message length + if i + length > len(message): + break + + found = False + # Look back in the window [max(0, i-n), i-1] + for j in range(max(0, i - n), i): + candidate = message[j : j + length] + if pattern == candidate: + longest_match = pattern + found = True + break + + # If a pattern of this length was not found, the + # longest match was one shorter. + if not found: + break + + return len(longest_match) + 1, longest_match + + +def kontoyiannis_entropy(message: str, window: Optional[int] = None) -> float: + r""" + Calculate Kontoyiannis Entropy (an LZ78-based estimator). + + .. math:: + H_k(n) = \frac{1}{\sum_{i} 1} \sum_{i} \frac{\log_2(n_i)}{L_i(n)} + + Reference: + Kontoyiannis, I. (1998). "Pointwise redundancy in Lempel-Ziv + parsing." + + Parameters + ---------- + message : str + Input string. + window : int, optional + If None, uses an expanding window (full lookback, n_i = i). + If set, uses a rolling window (n_i = window). + + Returns + ------- + float + The calculated Kontoyiannis Entropy (H_k). + """ + message_length = len(message) + sum_h = 0.0 + num_points = 0 + + if window is None: + # Expanding window: n = i + points = range(2, message_length) + else: + # Rolling window: n = window + window = min(window, message_length - 1) + points = range(window, message_length) + + if not points: + return 0.0 + + for i in points: + n = i if window is None else window + if n == 0: + continue # Avoid log2(0) + + l_i, _ = longest_match_length(message, i, n) + + sum_h += log2(n) / l_i + num_points += 1 + + if num_points == 0: + return 0.0 + + return sum_h / num_points diff --git a/src/RiskLabAI/features/entropy_features/lempel_ziv.py b/src/RiskLabAI/features/entropy_features/lempel_ziv.py new file mode 100644 index 0000000..38b5887 --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/lempel_ziv.py @@ -0,0 +1,41 @@ +""" +Implements the Lempel-Ziv (LZ) Entropy estimator. +""" + + +def lempel_ziv_entropy(message: str) -> float: + """ + Calculate the Lempel-Ziv (LZ) complexity as an entropy estimator. + + This function calculates the number of unique substrings encountered + during a one-pass traversal of the message, normalized by the + message length. + + Parameters + ---------- + message : str + Input string. + + Returns + ------- + float + The calculated Lempel-Ziv Entropy. + """ + if not message: + return 0.0 + + library = set() + message_length = len(message) + i = 0 + + while i < message_length: + j = i + # Find the longest substring starting at `i` that is *not* in the library + while j < message_length and message[i : j + 1] in library: + j += 1 + + # Add the new, unseen substring to the library + library.add(message[i : j + 1]) + i = j + 1 + + return len(library) / message_length diff --git a/src/RiskLabAI/features/entropy_features/plug_in.py b/src/RiskLabAI/features/entropy_features/plug_in.py new file mode 100644 index 0000000..ae5052e --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/plug_in.py @@ -0,0 +1,39 @@ +""" +Implements the Plug-in Entropy estimator. +""" + +from math import log2 + +from .pmf import probability_mass_function + + +def plug_in_entropy_estimator(message: str, approximate_word_length: int = 1) -> float: + """ + Calculate the Plug-in Entropy Estimator (based on n-gram PMF). + + This is the Shannon entropy applied to the PMF of n-grams + (words of `approximate_word_length`). + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default=1 + The n-gram length. + + Returns + ------- + float + The calculated Plug-in Entropy, normalized by word length. + """ + if not message: + return 0.0 + + pmf = probability_mass_function(message, approximate_word_length) + if not pmf: + return 0.0 + + plug_in_entropy = -sum(p * log2(p) for p in pmf.values() if p > 0) + + # Normalize by word length + return plug_in_entropy / approximate_word_length diff --git a/src/RiskLabAI/features/entropy_features/pmf.py b/src/RiskLabAI/features/entropy_features/pmf.py new file mode 100644 index 0000000..75294dc --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/pmf.py @@ -0,0 +1,64 @@ +""" +Implements the n-gram count and Probability Mass Function (PMF) calculation. +""" + +from collections import Counter + + +def ngram_counts(message: str, approximate_word_length: int) -> dict[str, int]: + """ + Count the overlapping n-grams (words) in a message. + + This is the shared counting path used by both the plug-in estimator (via + :func:`probability_mass_function`) and the bias-corrected estimators, which need + the raw counts and the sample size rather than the normalized probabilities. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int + The length of the "words" or n-grams to analyze (e.g., 1, 2, 3). + + Returns + ------- + Dict[str, int] + A dictionary mapping each observed n-gram to its integer count. + """ + if not message or len(message) < approximate_word_length: + return {} + + return dict( + Counter( + message[i : i + approximate_word_length] + for i in range(len(message) - approximate_word_length + 1) + ) + ) + + +def probability_mass_function( + message: str, approximate_word_length: int +) -> dict[str, float]: + """ + Calculate the Probability Mass Function (PMF) of n-grams. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int + The length of the "words" or n-grams to analyze (e.g., 1, 2, 3). + + Returns + ------- + Dict[str, float] + A dictionary mapping each n-gram to its probability. + """ + counts = ngram_counts(message, approximate_word_length) + + # The total number of n-grams (windows). + num_windows = float(sum(counts.values())) + if num_windows == 0: + return {} + + return {key: count / num_windows for key, count in counts.items()} diff --git a/src/RiskLabAI/features/entropy_features/shannon.py b/src/RiskLabAI/features/entropy_features/shannon.py new file mode 100644 index 0000000..937fb63 --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/shannon.py @@ -0,0 +1,34 @@ +""" +Implements the Shannon Entropy estimator. +""" + +from collections import Counter +from math import log2 + + +def shannon_entropy(message: str) -> float: + """ + Calculate the Shannon Entropy of a message. + + Parameters + ---------- + message : str + Input string (e.g., a discretized time series "1-1101..."). + + Returns + ------- + float + The calculated Shannon Entropy (in bits). + """ + if not message: + return 0.0 + + character_counts = Counter(message) + message_length = len(message) + + entropy = -sum( + (count / message_length) * log2(count / message_length) + for count in character_counts.values() + ) + + return entropy diff --git a/src/RiskLabAI/features/feature_importance/__init__.py b/src/RiskLabAI/features/feature_importance/__init__.py new file mode 100644 index 0000000..118b1cc --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/__init__.py @@ -0,0 +1,66 @@ +""" +RiskLabAI Feature Importance Module + +Provides a framework for calculating feature importance using +various methods (MDI, MDA, SFI) and clustering. +""" + +# Import from other RiskLabAI modules +from RiskLabAI.cluster.clustering import ( + cluster_k_means_top, + random_block_correlation, +) +from RiskLabAI.data.synthetic_data.simulation import ( + form_true_matrix, + simulates_cov_mu, +) + +from .clustered_feature_importance_mda import ClusteredFeatureImportanceMDA +from .clustered_feature_importance_mdi import ClusteredFeatureImportanceMDI +from .debiased_importance import ( + conditional_predictive_impact, + mdi_plus_importance, +) +from .feature_importance_controller import FeatureImportanceController +from .feature_importance_factory import FeatureImportanceFactory +from .feature_importance_mda import FeatureImportanceMDA + +# Strategy Implementations +from .feature_importance_mdi import FeatureImportanceMDI +from .feature_importance_sfi import FeatureImportanceSFI + +# Core Strategy Pattern +from .feature_importance_strategy import FeatureImportanceStrategy + +# Utility Functions for this module +from .generate_synthetic_data import get_test_dataset +from .orthogonal_features import _compute_eigenvectors, orthogonal_features +from .weighted_tau import calculate_weighted_tau + +# Placeholder for future modules (from original file) +# from . import swefi + +__all__ = [ + # Core Pattern + "FeatureImportanceStrategy", + "FeatureImportanceFactory", + "FeatureImportanceController", + # Strategy Implementations + "FeatureImportanceMDI", + "ClusteredFeatureImportanceMDI", + "FeatureImportanceMDA", + "ClusteredFeatureImportanceMDA", + "FeatureImportanceSFI", + # Debiased / conditional importance (Appraisal 10) + "mdi_plus_importance", + "conditional_predictive_impact", + # Imported utilities from *other* modules + "cluster_k_means_top", + "random_block_correlation", + "form_true_matrix", + "simulates_cov_mu", + # Utilities *from this* module + "get_test_dataset", + "orthogonal_features", + "calculate_weighted_tau", +] diff --git a/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py new file mode 100644 index 0000000..a88bb0c --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py @@ -0,0 +1,157 @@ +""" +Computes Clustered Mean Decrease Accuracy (MDA) feature importance. +""" + +import logging +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + +logger = logging.getLogger(__name__) + + +class ClusteredFeatureImportanceMDA(FeatureImportanceStrategy): + """ + Computes clustered feature importance using MDA. + + This method shuffles entire *clusters* of features at a time + and measures the decrease in model performance. + """ + + def __init__( + self, + classifier: object, + clusters: dict[str, list[str]], + n_splits: int = 10, + random_state: int = 42, + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + clusters : Dict[str, List[str]] + Dictionary mapping cluster names to lists of feature names. + n_splits : int, default=10 + Number of splits for cross-validation. + random_state : int, default=42 + Seed for KFold and shuffling for reproducibility. + """ + self.classifier = classifier + self.clusters = clusters + self.n_splits = n_splits + self.random_state = random_state + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute Clustered MDA feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance + for each *cluster*. + """ + train_weights = kwargs.get("train_sample_weights") + score_weights = kwargs.get("score_sample_weights") + + if train_weights is None: + train_weights = np.ones(x.shape[0]) + if score_weights is None: + score_weights = np.ones(x.shape[0]) + + cv_generator = KFold( + n_splits=self.n_splits, shuffle=True, random_state=self.random_state + ) + baseline_scores = pd.Series(dtype=float) + shuffled_scores = pd.DataFrame(columns=self.clusters.keys(), dtype=float) + + for i, (train_idx, test_idx) in enumerate(cv_generator.split(X=x)): + logger.debug("Fold %d start ...", i) + + x_train, y_train, w_train = ( + x.iloc[train_idx, :], + y.iloc[train_idx], + train_weights[train_idx], + ) + x_test, y_test, w_test = ( + x.iloc[test_idx, :], + y.iloc[test_idx], + score_weights[test_idx], + ) + + classifier_fit = self.classifier.fit( + X=x_train, y=y_train, sample_weight=w_train + ) + prediction_probability = classifier_fit.predict_proba(x_test) + + baseline_scores.loc[i] = -log_loss( + y_test, + prediction_probability, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Get scores for each shuffled *cluster* + rng = np.random.default_rng(self.random_state + i) + for cluster_name in shuffled_scores.columns: + x_test_shuffled = x_test.copy(deep=True) + + # --- CORRECTED SHUFFLING LOGIC --- + # Get all feature names for this cluster + cluster_cols = self.clusters[cluster_name] + + if not cluster_cols: # Skip if cluster is empty + shuffled_scores.loc[i, cluster_name] = baseline_scores.loc[i] + continue + + # Get the underlying numpy array for these columns + cluster_data = x_test_shuffled[cluster_cols].values.copy() + + # Shuffle the rows of this array in-place. + # This applies the *same* permutation to all features + # in the cluster, preserving intra-cluster correlation. + rng.shuffle(cluster_data) + + # Assign the shuffled data back + x_test_shuffled[cluster_cols] = cluster_data + # --- END CORRECTION --- + + prob = classifier_fit.predict_proba(x_test_shuffled) + shuffled_scores.loc[i, cluster_name] = -log_loss( + y_test, prob, labels=self.classifier.classes_, sample_weight=w_test + ) + + # Calculate importance as the simple drop in score + importances = shuffled_scores.rsub(baseline_scores, axis=0) + + # Central Limit Theorem for standard deviation + importances_summary = pd.concat( + { + "Mean": importances.mean(), + "StandardDeviation": ( + importances.std() * (importances.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + importances_summary.index = [f"C_{i}" for i in importances_summary.index] + return importances_summary diff --git a/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py new file mode 100644 index 0000000..dd7092d --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py @@ -0,0 +1,108 @@ +""" +Computes Clustered Mean Decrease Impurity (MDI) feature importance. +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaseEnsemble + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class ClusteredFeatureImportanceMDI(FeatureImportanceStrategy): + """ + Computes Clustered MDI feature importance. + + Aggregates MDI importance for pre-defined clusters of features. + """ + + def __init__( + self, + classifier: BaseEnsemble, + clusters: dict[str, list[str]], + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : BaseEnsemble + An *untrained* scikit-learn ensemble model. + clusters : Dict[str, List[str]] + Dictionary mapping cluster names to lists of feature names. + """ + if not isinstance(classifier, BaseEnsemble): + raise TypeError("Classifier must be an ensemble (e.g., RandomForest).") + + self.classifier = classifier + self.clusters = clusters + + def _group_mean_std( + self, dataframe: pd.DataFrame, clusters: dict[str, list[str]] + ) -> pd.DataFrame: + """ + Calculate the mean and standard deviation for cluster importances. + """ + output = pd.DataFrame(columns=["Mean", "StandardDeviation"]) + + for cluster_name, feature_names in clusters.items(): + # Sum importance for all features in the cluster + cluster_data = dataframe[feature_names].sum(axis=1) + + cluster_mean = cluster_data.mean() + cluster_std = cluster_data.std() + + output.loc[f"C_{cluster_name}", "Mean"] = cluster_mean + output.loc[f"C_{cluster_name}", "StandardDeviation"] = cluster_std * ( + cluster_data.shape[0] ** -0.5 + ) + return output + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute Clustered MDI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Keyword arguments for the classifier's `fit` method + (e.g., `sample_weight`). + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance + for each *cluster*. + """ + train_sample_weights = kwargs.get("sample_weight") + + # Fit the classifier + self.classifier.fit(x, y, sample_weight=train_sample_weights) + + # Get importance from each tree + importances_dict = { + i: tree.feature_importances_ + for i, tree in enumerate(self.classifier.estimators_) + } + importances_df = pd.DataFrame.from_dict(importances_dict, orient="index") + + if hasattr(self.classifier, "feature_names_in_"): + importances_df.columns = self.classifier.feature_names_in_ + else: + importances_df.columns = x.columns + + # Replace 0 with NaN + importances_df.replace(0, np.nan, inplace=True) + + # Group by cluster + aggregated_importances = self._group_mean_std(importances_df, self.clusters) + + # Normalize + aggregated_importances /= aggregated_importances["Mean"].sum() + return aggregated_importances diff --git a/src/RiskLabAI/features/feature_importance/debiased_importance.py b/src/RiskLabAI/features/feature_importance/debiased_importance.py new file mode 100644 index 0000000..a601ebc --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/debiased_importance.py @@ -0,0 +1,277 @@ +r""" +Debiased and conditional feature importance: MDI+ and the Conditional Predictive Impact (CPI). + +de Prado's MDI counts impurity reduction, which inflates with split opportunities (high cardinality) +and with in-sample reuse, so it over-credits noisy / continuous features and can rank noise above +signal; MDA has no significance test. Two published fixes extend the baselines in +`features.feature_importance`: + +- MDI+ (Agarwal et al. 2023) recasts each tree's MDI as a regularized GLM on the tree's decision-stump + representation and scores a feature by its out-of-sample partial contribution, removing the + in-sample / cardinality inflation; and +- the Conditional Predictive Impact (CPI, Watson and Wright 2021) measures a feature's importance as + the model-loss increase when the true feature is replaced by a conditionally-resampled knockoff, with + a paired significance test, the calibrated test MDA lacks. + +Admitted in Appraisal 10 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tags, verbatim from the verdict: + + MDI+: prefer MDI+ over MDI when features are noisy, high-cardinality, or mixed-type - it rejects + noise/cardinality inflation (noise-reject 0.94 vs 0.50) and recovers true relevance better; + converges to MDI when features are orthogonal and high-SNR. + + CPI: prefer CPI when you need a statistically valid test of whether a feature matters (MDA has + none); it holds nominal size with good power. + +The implemented MDI+ is the simplified faithful variant admitted (per-tree decision-stump ridge GLM +with an out-of-bag partial-variance score), NOT the full leave-one-out / similarity-weighted paper +algorithm, which is a logged refinement. CPI's test holds nominal size in the analysis regimes; on the +hardest held-out corner (high correlation x low SNR) it is mildly elevated (~0.067 vs 0.05), a small +finite-sample effect. Evidence and caveats: appraisals/10_verdict.md. + +References +---------- +Agarwal, A. et al. (2023) MDI+: A flexible random forest based feature importance framework. + arXiv:2307.01932. +Watson, D. S. and Wright, M. N. (2021) Testing conditional independence in supervised learning + algorithms. Machine Learning, 110, 2107-2129. (The Conditional Predictive Impact.) +""" + +from __future__ import annotations + +from numbers import Integral + +import numpy as np +import pandas as pd +from scipy.stats import ttest_1samp +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import Ridge +from sklearn.model_selection import KFold +from sklearn.utils import check_random_state + +_DEFAULT_N_ESTIMATORS = 100 +_DEFAULT_MAX_DEPTH = 6 +_DEFAULT_TREES_DECOMP = 60 + + +def _bootstrap_sample_indices( + random_state: int, n_samples: int, max_samples: int | float | None +) -> np.ndarray: + """Reproduce documented bootstrap sampling without private library APIs.""" + if max_samples is None: + n_bootstrap = n_samples + elif isinstance(max_samples, Integral): + n_bootstrap = int(max_samples) + else: + n_bootstrap = max(round(n_samples * float(max_samples)), 1) + return check_random_state(random_state).randint( + 0, n_samples, n_bootstrap + ) + + +def _default_forest( + random_state: int, n_estimators: int = _DEFAULT_N_ESTIMATORS +) -> RandomForestClassifier: + return RandomForestClassifier( + n_estimators=n_estimators, + max_depth=_DEFAULT_MAX_DEPTH, + random_state=random_state, + n_jobs=1, + bootstrap=True, + ) + + +def mdi_plus_importance( + x: pd.DataFrame, + y: pd.Series, + random_state: int = 0, + n_trees: int = _DEFAULT_TREES_DECOMP, + classifier: RandomForestClassifier | None = None, +) -> pd.Series: + r""" + MDI+ feature importance (Agarwal et al. 2023, simplified faithful variant). + + For each tree it builds a centered decision-stump basis (one indicator ``1{x_f <= threshold}`` per + internal node, grouped by the splitting feature), fits a ridge GLM of the label on that basis using + the tree's in-bag rows, and scores each feature by the variance of its block's partial prediction + evaluated on the tree's out-of-bag rows; the per-tree scores are averaged. The tree representation + plus regularized GLM plus out-of-sample partial contribution remove the in-sample / cardinality + inflation of plain MDI. + + Prefer MDI+ over MDI when features are noisy, high-cardinality, or mixed-type; it converges to MDI + when features are orthogonal and high-SNR. This is the simplified faithful variant (not the full + leave-one-out paper algorithm). See the module docstring for the full regime tag and + appraisals/10_verdict.md. + + Parameters + ---------- + x : pd.DataFrame + The design matrix. + y : pd.Series + The binary target. + random_state : int, default 0 + Forest seed. + n_trees : int, default 60 + Number of trees used by the per-tree decomposition. + classifier : RandomForestClassifier, optional + A bootstrap RandomForest to use instead of the default (depth-6, 100-tree) forest. + + Returns + ------- + pd.Series + MDI+ importance per feature (indexed by column; higher is more important). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> x = pd.DataFrame(rng.standard_normal((400, 4)), columns=list("abcd")) + >>> y = pd.Series((x["a"] + 0.3 * rng.standard_normal(400) > 0).astype(int)) + >>> imp = mdi_plus_importance(x, y) + >>> imp.idxmax() == "a" + True + """ + forest = classifier if classifier is not None else _default_forest(random_state) + forest.fit(x, y) + xv = x.to_numpy(float) + yv = y.to_numpy(float) + n, n_features = xv.shape + accumulated = np.zeros(n_features) + used = 0 + for estimator in forest.estimators_[:n_trees]: + tree = estimator.tree_ + internal = np.where(tree.feature >= 0)[0] + if internal.size == 0: + continue + basis = (xv[:, tree.feature[internal]] <= tree.threshold[internal]).astype( + float + ) + basis = basis - basis.mean(axis=0, keepdims=True) + groups = tree.feature[internal] + in_bag = np.unique( + _bootstrap_sample_indices(estimator.random_state, n, forest.max_samples) + ) + out_of_bag = np.setdiff1d(np.arange(n), in_bag, assume_unique=False) + if out_of_bag.size < 5: + continue + ridge = Ridge(alpha=1.0, fit_intercept=True) + ridge.fit(basis[in_bag], yv[in_bag]) + coefficients = ridge.coef_ + for feature in np.unique(groups): + mask = groups == feature + partial = basis[out_of_bag][:, mask] @ coefficients[mask] + accumulated[feature] += partial.var() + used += 1 + accumulated = accumulated / max(used, 1) + return pd.Series(accumulated, index=x.columns) + + +def _per_sample_log_loss( + y_true: np.ndarray, proba: np.ndarray, classes: np.ndarray +) -> np.ndarray: + eps = 1e-15 + proba = np.clip(proba, eps, 1 - eps) + class_to_column = {c: k for k, c in enumerate(classes)} + columns = np.array([class_to_column[v] for v in y_true]) + chosen = proba[np.arange(len(y_true)), columns] + return -np.log(chosen) + + +def conditional_predictive_impact( + x: pd.DataFrame, + y: pd.Series, + random_state: int = 0, + n_splits: int = 4, + n_estimators: int = 80, + classifier: RandomForestClassifier | None = None, +) -> pd.DataFrame: + r""" + Conditional Predictive Impact (CPI; Watson and Wright 2021), with a calibrated significance test. + + For each cross-validation fold a classifier is fit on the training rows; on the test rows each + feature is replaced by a conditional Gaussian knockoff (the feature regressed on the others by ridge + plus resampled residual noise), and the per-sample increase in log loss is recorded. Pooled across + folds, the mean increase is the CPI statistic and a one-sided paired t-test gives a p-value, so the + method tests whether a feature matters - the calibrated test MDA lacks. + + Prefer CPI when you need a statistically valid test of whether a feature matters; it holds nominal + size with good power (mildly elevated on the hardest high-correlation x low-SNR corner). See the + module docstring for the full regime tag and appraisals/10_verdict.md. + + Parameters + ---------- + x : pd.DataFrame + The design matrix. + y : pd.Series + The target. + random_state : int, default 0 + Seed for the folds, forest, and knockoff noise. + n_splits : int, default 4 + Cross-validation folds. + n_estimators : int, default 80 + Trees in the per-fold forest (used when ``classifier`` is None). + classifier : RandomForestClassifier, optional + A classifier template cloned per fold (defaults to a depth-6 forest). + + Returns + ------- + pd.DataFrame + Indexed by feature with columns ``importance`` (mean log-loss increase; higher = more important) + and ``p_value`` (one-sided paired t-test of importance > 0). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(1) + >>> x = pd.DataFrame(rng.standard_normal((300, 3)), columns=list("abc")) + >>> y = pd.Series((x["a"] + 0.3 * rng.standard_normal(300) > 0).astype(int)) + >>> cpi = conditional_predictive_impact(x, y) + >>> bool(cpi.loc["a", "p_value"] < cpi.loc["c", "p_value"]) + True + """ + from sklearn.base import clone + + columns = list(x.columns) + folds = KFold(n_splits=n_splits, shuffle=True, random_state=random_state) + xv = x.to_numpy(float) + yv = y.to_numpy() + differences: dict[str, list] = {c: [] for c in columns} + template = ( + classifier + if classifier is not None + else _default_forest(random_state, n_estimators) + ) + for i, (train, test) in enumerate(folds.split(x)): + model = clone(template).fit(x.iloc[train], y.iloc[train]) + classes = model.classes_ + loss_true = _per_sample_log_loss( + yv[test], model.predict_proba(x.iloc[test]), classes + ) + rng = np.random.default_rng(random_state + 100 + i) + for j, c in enumerate(columns): + others = [k for k in range(len(columns)) if k != j] + ridge = Ridge(alpha=1.0).fit(xv[train][:, others], xv[train][:, j]) + residual = xv[train][:, j] - ridge.predict(xv[train][:, others]) + sigma = residual.std() + 1e-9 + knockoff = ridge.predict(xv[test][:, others]) + sigma * rng.standard_normal( + len(test) + ) + x_knockoff = xv[test].copy() + x_knockoff[:, j] = knockoff + loss_knockoff = _per_sample_log_loss( + yv[test], + model.predict_proba(pd.DataFrame(x_knockoff, columns=columns)), + classes, + ) + differences[c].append(loss_knockoff - loss_true) + importance, p_value = {}, {} + for c in columns: + d = np.concatenate(differences[c]) + importance[c] = float(d.mean()) + if d.std() < 1e-12: + p_value[c] = 1.0 + else: + t_stat, p_two = ttest_1samp(d, 0.0) + p_value[c] = float(p_two / 2 if t_stat > 0 else 1.0 - p_two / 2) + return pd.DataFrame( + {"importance": pd.Series(importance), "p_value": pd.Series(p_value)} + ).reindex(columns) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_controller.py b/src/RiskLabAI/features/feature_importance/feature_importance_controller.py new file mode 100644 index 0000000..8c9a568 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_controller.py @@ -0,0 +1,42 @@ +""" +Deprecated controller for managing feature-importance strategies. + +Use the core component registry (:data:`RiskLabAI.core.FEATURE_IMPORTANCE`) +directly instead. This controller is retained for backward compatibility and +delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +import pandas as pd + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceController: + """ + Deprecated. Use ``RiskLabAI.core.FEATURE_IMPORTANCE.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + def __init__(self, strategy_type: str, **kwargs: Any): + warnings.warn( + "FeatureImportanceController is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.FEATURE_IMPORTANCE.create(strategy_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import FEATURE_IMPORTANCE + + self.strategy_instance: FeatureImportanceStrategy = FEATURE_IMPORTANCE.create( + strategy_type, filter_unknown_kwargs=True, **kwargs + ) + + def calculate_importance( + self, x: pd.DataFrame, y: pd.Series, **kwargs: Any + ) -> pd.DataFrame: + """Calculate feature importance using the configured strategy.""" + return self.strategy_instance.compute(x, y, **kwargs) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_factory.py b/src/RiskLabAI/features/feature_importance/feature_importance_factory.py new file mode 100644 index 0000000..6065abb --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_factory.py @@ -0,0 +1,50 @@ +""" +Deprecated factory for creating feature-importance strategy objects. + +The core component registry (:data:`RiskLabAI.core.FEATURE_IMPORTANCE`) is now +the single source of truth. This factory is retained for backward compatibility +and delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceFactory: + """ + Deprecated. Use ``RiskLabAI.core.FEATURE_IMPORTANCE.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + @staticmethod + def create_feature_importance( + strategy_type: str, **kwargs: Any + ) -> FeatureImportanceStrategy: + """ + Deprecated. Create a feature-importance strategy via the core registry. + + Raises + ------ + ValueError + If ``strategy_type`` is not a known strategy. + """ + warnings.warn( + "FeatureImportanceFactory is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.FEATURE_IMPORTANCE.create(strategy_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import FEATURE_IMPORTANCE + + if strategy_type.lower() not in FEATURE_IMPORTANCE: + valid = list(FEATURE_IMPORTANCE.available()) + raise ValueError( + f"Invalid strategy_type: {strategy_type}. Valid types are: {valid}" + ) + return FEATURE_IMPORTANCE.create( + strategy_type, filter_unknown_kwargs=True, **kwargs + ) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_mda.py b/src/RiskLabAI/features/feature_importance/feature_importance_mda.py new file mode 100644 index 0000000..b989249 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_mda.py @@ -0,0 +1,135 @@ +""" +Computes Mean Decrease Accuracy (MDA) feature importance. +""" + +import logging +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + +logger = logging.getLogger(__name__) + + +class FeatureImportanceMDA(FeatureImportanceStrategy): + """ + Computes feature importance using Mean Decrease Accuracy (MDA). + + This method shuffles each feature one by one and measures how + much the model's performance (e.g., log loss) decreases. + """ + + def __init__(self, classifier: object, n_splits: int = 10, random_state: int = 42): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + n_splits : int, default=10 + Number of splits for cross-validation. + """ + self.classifier = classifier + self.n_splits = n_splits + self.random_state = random_state + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute MDA feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance. + """ + train_weights = kwargs.get("train_sample_weights") + score_weights = kwargs.get("score_sample_weights") + + if train_weights is None: + train_weights = np.ones(x.shape[0]) + if score_weights is None: + score_weights = np.ones(x.shape[0]) + + cv_generator = KFold( + n_splits=self.n_splits, shuffle=True, random_state=self.random_state + ) + baseline_scores = pd.Series(dtype=float) + shuffled_scores = pd.DataFrame(columns=x.columns, dtype=float) + + for i, (train_idx, test_idx) in enumerate(cv_generator.split(x)): + logger.debug("Fold %d start ...", i) + + x_train, y_train, w_train = ( + x.iloc[train_idx, :], + y.iloc[train_idx], + train_weights[train_idx], + ) + x_test, y_test, w_test = ( + x.iloc[test_idx, :], + y.iloc[test_idx], + score_weights[test_idx], + ) + + # Fit classifier and get baseline score + fitted_classifier = self.classifier.fit( + X=x_train, y=y_train, sample_weight=w_train + ) + pred_proba = fitted_classifier.predict_proba(x_test) + + baseline_scores.loc[i] = -log_loss( + y_test, + pred_proba, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Get scores for each shuffled feature + rng = np.random.default_rng(self.random_state + i) + for feature in x.columns: + x_test_shuffled = x_test.copy(deep=True) + # Shuffle a copy and assign back: with pandas copy-on-write + # (default since pandas 3.0), column .values is a read-only + # view and in-place shuffling raises ValueError. + shuffled_values = x_test_shuffled[feature].to_numpy(copy=True) + rng.shuffle(shuffled_values) # seeded shuffle + x_test_shuffled[feature] = shuffled_values + + shuffled_proba = fitted_classifier.predict_proba(x_test_shuffled) + + shuffled_scores.loc[i, feature] = -log_loss( + y_test, + shuffled_proba, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Calculate importance as the simple drop in score + importances = shuffled_scores.rsub(baseline_scores, axis=0) + + # Calculate mean and std dev + importances_summary = pd.concat( + { + "Mean": importances.mean(), + "StandardDeviation": ( + importances.std() * (importances.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + return importances_summary diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py b/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py new file mode 100644 index 0000000..c1c1be9 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py @@ -0,0 +1,88 @@ +""" +Computes Mean Decrease Impurity (MDI) feature importance. +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaseEnsemble + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceMDI(FeatureImportanceStrategy): + """ + Computes feature importance using Mean Decrease Impurity (MDI). + + This method is specific to tree-based ensembles (like RandomForest) + and measures importance as the average impurity decrease. + """ + + def __init__(self, classifier: BaseEnsemble): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : BaseEnsemble + An *untrained* scikit-learn ensemble model (e.g., RandomForestClassifier). + """ + if not isinstance(classifier, BaseEnsemble): + raise TypeError("Classifier must be an ensemble (e.g., RandomForest).") + + self.classifier = classifier + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute MDI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Keyword arguments for the classifier's `fit` method + (e.g., `sample_weight`). + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance. + """ + train_sample_weights = kwargs.get("sample_weight") + + # Fit the classifier + self.classifier.fit(x, y, sample_weight=train_sample_weights) + + # Get importance from each tree + importances_dict = { + i: tree.feature_importances_ + for i, tree in enumerate(self.classifier.estimators_) + } + importances_df = pd.DataFrame.from_dict(importances_dict, orient="index") + + # Ensure correct feature names + if hasattr(self.classifier, "feature_names_in_"): + importances_df.columns = self.classifier.feature_names_in_ + else: + importances_df.columns = x.columns + + # Replace 0 with NaN (as per user's original code) + importances_df.replace(0, np.nan, inplace=True) + + importances = pd.concat( + { + "Mean": importances_df.mean(), + "StandardDeviation": ( + importances_df.std() * (importances_df.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + # Normalize + importances /= importances["Mean"].sum() + return importances diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py b/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py new file mode 100644 index 0000000..e311d96 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py @@ -0,0 +1,119 @@ +""" +Computes Single Feature Importance (SFI). +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import accuracy_score, log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceSFI(FeatureImportanceStrategy): + """ + Computes Single Feature Importance (SFI). + + This method calculates the importance of each feature by training + and evaluating a model using *only that single feature*. + """ + + def __init__( + self, + classifier: object, + n_splits: int = 10, + scoring: str = "log_loss", + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + n_splits : int, default=10 + Number of splits for cross-validation. + scoring : str, default="log_loss" + Scoring method ("log_loss" or "accuracy"). + """ + self.classifier = classifier + self.n_splits = n_splits + self.scoring = scoring + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute SFI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "FeatureName", "Mean", and "StandardDeviation" + of the SFI scores. + """ + train_sample_weights = kwargs.get("train_sample_weights") + score_sample_weights = kwargs.get("score_sample_weights") + + if train_sample_weights is None: + train_sample_weights = np.ones(x.shape[0]) + if score_sample_weights is None: + score_sample_weights = np.ones(x.shape[0]) + + cv_generator = KFold(n_splits=self.n_splits) + feature_names = x.columns + importances = [] + + for feature_name in feature_names: + scores = [] + feature_data = x[[feature_name]] + + for train_idx, test_idx in cv_generator.split(feature_data): + x_train, y_train, w_train = ( + feature_data.iloc[train_idx], + y.iloc[train_idx], + train_sample_weights[train_idx], + ) + x_test, y_test, w_test = ( + feature_data.iloc[test_idx], + y.iloc[test_idx], + score_sample_weights[test_idx], + ) + + self.classifier.fit(x_train, y_train, sample_weight=w_train) + + if self.scoring == "log_loss": + pred_proba = self.classifier.predict_proba(x_test) + score = -log_loss( + y_test, + pred_proba, + sample_weight=w_test, + labels=self.classifier.classes_, + ) + elif self.scoring == "accuracy": + pred = self.classifier.predict(x_test) + score = accuracy_score(y_test, pred, sample_weight=w_test) + else: + raise ValueError(f"'{self.scoring}' method not defined.") + + scores.append(score) + + importances.append( + { + "FeatureName": feature_name, + "Mean": np.mean(scores), + "StandardDeviation": np.std(scores, ddof=1) * (len(scores) ** -0.5), + } + ) + + return pd.DataFrame(importances).set_index("FeatureName") diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py b/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py new file mode 100644 index 0000000..bd59e3c --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py @@ -0,0 +1,37 @@ +""" +Abstract Base Class for feature importance strategies. +""" + +from abc import ABC, abstractmethod +from typing import Any + +import pandas as pd + + +class FeatureImportanceStrategy(ABC): + """ + Abstract Base Class for computing feature importance. + + Derived classes must implement the `compute` method. + """ + + @abstractmethod + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Abstract method to compute feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Additional keyword arguments (e.g., sample_weights). + + Returns + ------- + pd.DataFrame + A DataFrame containing feature importances. + """ + pass diff --git a/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py b/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py new file mode 100644 index 0000000..67e625e --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py @@ -0,0 +1,80 @@ +""" +Generates a synthetic dataset for feature importance testing. + +Based on the method from De Prado (2018). +""" + +import numpy as np +import pandas as pd +from sklearn.datasets import make_classification + + +def get_test_dataset( + n_features: int = 100, + n_informative: int = 25, + n_redundant: int = 25, + n_samples: int = 10000, + random_state: int = 0, + sigma_std: float = 0.0, +) -> tuple[pd.DataFrame, pd.Series]: + """ + Generate a synthetic dataset with informative, redundant, and noise features. + + Parameters + ---------- + n_features : int, default=100 + Total number of features to generate. + n_informative : int, default=25 + Number of informative features. + n_redundant : int, default=25 + Number of redundant features. These are created by adding + Gaussian noise to informative features. + n_samples : int, default=10000 + Number of samples (rows) to generate. + random_state : int, default=0 + Random state for reproducibility. + sigma_std : float, default=0.0 + Standard deviation of the noise added to create redundant features. + + Returns + ------- + Tuple[pd.DataFrame, pd.Series] + - X (pd.DataFrame): The generated feature matrix. + - y (pd.Series): The generated labels. + """ + rng = np.random.default_rng(random_state) + + # 1. Generate informative and noise features + n_noise = n_features - n_informative - n_redundant + x_array, y_array = make_classification( + n_samples=n_samples, + n_features=n_informative + n_noise, + n_informative=n_informative, + n_redundant=0, + shuffle=False, + random_state=random_state, + ) + + # 2. Create DataFrame + columns = [f"I_{i}" for i in range(n_informative)] + columns += [f"N_{i}" for i in range(n_noise)] + x_df = pd.DataFrame(x_array, columns=columns) + y_series = pd.Series(y_array, name="Target") + + # 3. Create redundant features + # Randomly pick informative features to copy + redundant_indices = rng.choice(range(n_informative), size=n_redundant, replace=True) + + for i, orig_idx in enumerate(redundant_indices): + orig_feature_name = f"I_{orig_idx}" + new_feature_name = f"R_{i}" + + # Add noise to the original informative feature + noise = ( + rng.normal(loc=0.0, scale=sigma_std, size=n_samples) + * x_df[orig_feature_name].std() + ) + + x_df[new_feature_name] = x_df[orig_feature_name] + noise + + return x_df[sorted(x_df.columns)], y_series diff --git a/src/RiskLabAI/features/feature_importance/orthogonal_features.py b/src/RiskLabAI/features/feature_importance/orthogonal_features.py new file mode 100644 index 0000000..8136c8b --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/orthogonal_features.py @@ -0,0 +1,99 @@ +""" +Implements a method for feature orthogonalization using PCA. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +import numpy as np +import pandas as pd + + +def _compute_eigenvectors( + dot_product: np.ndarray, explained_variance_threshold: float +) -> pd.DataFrame: + """ + Compute eigenvalues and eigenvectors and filter by explained variance. + + Parameters + ---------- + dot_product : np.ndarray + The dot product matrix (e.g., X.T @ X). + explained_variance_threshold : float + The cumulative variance threshold to filter eigenvectors. + + Returns + ------- + pd.DataFrame + DataFrame with sorted eigenvalues, eigenvectors, and cumulative variance. + """ + eigenvalues, eigenvectors = np.linalg.eigh(dot_product) + + # Sort in descending order + indices_sorted = eigenvalues.argsort()[::-1] + eigenvalues = eigenvalues[indices_sorted] + eigenvectors = eigenvectors[:, indices_sorted] + + eigen_dataframe = pd.DataFrame( + { + "EigenValue": eigenvalues, + "EigenVector": [ev for ev in eigenvectors.T], + }, + index=[f"PC_{i+1}" for i in range(len(eigenvalues))], + ) + + # Calculate cumulative variance + cumulative_variance = eigenvalues.cumsum() / eigenvalues.sum() + eigen_dataframe["CumulativeVariance"] = cumulative_variance + + # Find the index where cumulative variance crosses the threshold + index = cumulative_variance.searchsorted(explained_variance_threshold) + + # Keep components up to and including the one that crosses the threshold + eigen_dataframe = eigen_dataframe.iloc[: index + 1, :] + + return eigen_dataframe + + +def orthogonal_features( + features: pd.DataFrame, variance_threshold: float = 0.95 +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Compute orthogonal features using PCA. + + Parameters + ---------- + features : pd.DataFrame + The feature DataFrame. + variance_threshold : float, default=0.95 + Cumulative explained variance threshold to keep. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - orthogonal_features_df: The transformed (orthogonal) features. + - eigen_dataframe: DataFrame with eigenvalues and vectors. + """ + # 1. Normalize features (z-score) + normalized_features = (features - features.mean(axis=0)) / features.std(axis=0) + normalized_features = normalized_features.dropna(axis=1) # Drop constant cols + + # 2. Compute dot product (proportional to covariance) + dot_product = normalized_features.T @ normalized_features + + # 3. Get principal components + eigen_dataframe = _compute_eigenvectors(dot_product, variance_threshold) + + # 4. Get transformation matrix (stacking eigenvectors) + transformation_matrix = np.vstack(eigen_dataframe["EigenVector"].values).T + + # 5. Transform features + orthogonal_features_arr = normalized_features.values @ transformation_matrix + + orthogonal_features_df = pd.DataFrame( + orthogonal_features_arr, + index=features.index, + columns=eigen_dataframe.index, + ) + + return orthogonal_features_df, eigen_dataframe diff --git a/src/RiskLabAI/features/feature_importance/weighted_tau.py b/src/RiskLabAI/features/feature_importance/weighted_tau.py new file mode 100644 index 0000000..b5f271e --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/weighted_tau.py @@ -0,0 +1,36 @@ +""" +Calculates the weighted Kendall's tau. +""" + +import numpy as np +import scipy.stats as stats + + +def calculate_weighted_tau( + feature_importances: np.ndarray, principal_component_ranks: np.ndarray +) -> float: + r""" + Calculate the weighted Kendall's tau (\(\tau\)). + + This function calculates a weighted correlation between two rankings. + Here, it's used to compare feature importances to their ranks, + weighted by the inverse of the rank (giving more weight to + disagreements at the top). + + Parameters + ---------- + feature_importances : np.ndarray + Vector of feature importances. + principal_component_ranks : np.ndarray + Vector of ranks (e.g., 1, 2, 3, ...). + + Returns + ------- + float + The weighted Kendall's tau correlation coefficient. + """ + # Weights are the inverse of the rank + weights = 1.0 / principal_component_ranks + + tau, _ = stats.weightedtau(feature_importances, weights) + return tau diff --git a/src/RiskLabAI/features/microstructural_features/__init__.py b/src/RiskLabAI/features/microstructural_features/__init__.py new file mode 100644 index 0000000..ed25a06 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/__init__.py @@ -0,0 +1,29 @@ +""" +RiskLabAI Microstructural Features Module + +Implements estimators for market microstructure features, such as the +Corwin-Schultz spread, the Bekker-Parkinson volatility, and the EDGE +effective bid-ask spread estimator. +""" + +from .bekker_parkinson_volatility_estimator import ( + bekker_parkinson_volatility_estimates, + sigma_estimates, +) +from .corwin_schultz import ( + alpha_estimates, + beta_estimates, + corwin_schultz_estimator, + gamma_estimates, +) +from .edge import edge_estimator + +__all__ = [ + "beta_estimates", + "gamma_estimates", + "alpha_estimates", + "corwin_schultz_estimator", + "sigma_estimates", + "bekker_parkinson_volatility_estimates", + "edge_estimator", +] diff --git a/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py b/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py new file mode 100644 index 0000000..ad513f0 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py @@ -0,0 +1,77 @@ +""" +Implements the Bekker-Parkinson volatility estimator, which adjusts +the Parkinson volatility by the Corwin-Schultz spread. + +Reference: + De Prado, M. (2018) Advances in Financial Machine Learning, + page 286, snippet 19.2. +""" + +from math import pi + +import numpy as np +import pandas as pd + +from .corwin_schultz import _DENOMINATOR, beta_estimates, gamma_estimates + + +def sigma_estimates(beta: pd.Series, gamma: pd.Series) -> pd.Series: + r""" + Compute Bekker-Parkinson volatility \(\sigma\) estimates. + + .. math:: + k_2 = \sqrt{8 / \pi} + d = 3 - 2\sqrt{2} + \sigma = \frac{(\sqrt{2} - 1) \sqrt{\beta}}{d} + + \sqrt{\frac{\gamma}{k_2^2 d}} + + Parameters + ---------- + beta : pd.Series + \(\beta\) estimates vector from Corwin-Schultz. + gamma : pd.Series + \(\gamma\) estimates vector from Corwin-Schultz. + + Returns + ------- + pd.Series + Bekker-Parkinson volatility \(\sigma\) estimates. + """ + k2 = (8 / pi) ** 0.5 + + term1 = (2**0.5 - 1) * (beta**0.5) / _DENOMINATOR + term2 = (gamma / (k2**2 * _DENOMINATOR)) ** 0.5 + + # Floor at zero + sigma = np.maximum(term1 + term2, 0) + + return sigma + + +def bekker_parkinson_volatility_estimates( + high_prices: pd.Series, low_prices: pd.Series, window_span: int = 20 +) -> pd.Series: + r""" + Compute Bekker-Parkinson volatility estimates from high and low prices. + + This function first calculates the Corwin-Schultz \(\beta\) and \(\gamma\) + parameters and then uses them to compute the volatility estimates. + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int, default=20 + Rolling window span for \(\beta\) estimation. + + Returns + ------- + pd.Series + Bekker-Parkinson volatility estimates. + """ + beta = beta_estimates(high_prices, low_prices, window_span) + gamma = gamma_estimates(high_prices, low_prices) + + return sigma_estimates(beta, gamma) diff --git a/src/RiskLabAI/features/microstructural_features/corwin_schultz.py b/src/RiskLabAI/features/microstructural_features/corwin_schultz.py new file mode 100644 index 0000000..b773650 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/corwin_schultz.py @@ -0,0 +1,135 @@ +""" +Implements the Corwin and Schultz (2012) bid-ask spread estimator. + +Reference: + Corwin, S. A., & Schultz, P. (2012). A simple way to estimate + bid-ask spreads from daily high and low prices. + The Journal of Finance, 67(2), 719-760. +""" + +import numpy as np +import pandas as pd + +# Constant 'd' from the paper (denominator) +_DENOMINATOR = 3 - 2 * (2**0.5) + + +def beta_estimates( + high_prices: pd.Series, low_prices: pd.Series, window_span: int +) -> pd.Series: + r""" + Estimate \(\beta\) (sum of squared high-low log-ratios). + + .. math:: + \beta_t = \sum_{j=t-1}^{t} [\ln(H_j / L_j)]^2 + + This is then averaged over the `window_span`. + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int + Rolling window span for averaging. + + Returns + ------- + pd.Series + The estimated \(\beta\) vector. + """ + log_ratios_sq = np.log(high_prices / low_prices) ** 2 + + # Sum of current and previous day's squared log-ratio + beta = log_ratios_sq.rolling(window=2).sum() + + # Average over the window span + beta = beta.rolling(window=window_span).mean() + return beta + + +def gamma_estimates(high_prices: pd.Series, low_prices: pd.Series) -> pd.Series: + r""" + Estimate \(\gamma\) (squared log-ratio of two-day high/low). + + .. math:: + \gamma_t = [\ln(\max(H_t, H_{t-1}) / \min(L_t, L_{t-1}))]^2 + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + + Returns + ------- + pd.Series + The estimated \(\gamma\) vector. + """ + high_prices_max = high_prices.rolling(window=2).max() + low_prices_min = low_prices.rolling(window=2).min() + gamma = np.log(high_prices_max / low_prices_min) ** 2 + return gamma + + +def alpha_estimates(beta: pd.Series, gamma: pd.Series) -> pd.Series: + r""" + Estimate \(\alpha\) from \(\beta\) and \(\gamma\). + + .. math:: + d = 3 - 2\sqrt{2} + \alpha = \frac{(\sqrt{2} - 1)\sqrt{\beta}}{d} + - \sqrt{\frac{\gamma}{d}} + + Parameters + ---------- + beta : pd.Series + \(\beta\) estimates vector. + gamma : pd.Series + \(\gamma\) estimates vector. + + Returns + ------- + pd.Series + The estimated \(\alpha\) vector, floored at 0. + """ + term1 = ((2**0.5) - 1) * (beta**0.5) / _DENOMINATOR + term2 = (gamma / _DENOMINATOR) ** 0.5 + + # Floor at zero + alpha = np.maximum(term1 - term2, 0) + return alpha + + +def corwin_schultz_estimator( + high_prices: pd.Series, low_prices: pd.Series, window_span: int = 20 +) -> pd.Series: + r""" + Estimate the bid-ask spread using the Corwin and Schultz (2012) method. + + .. math:: + S = \frac{2(e^\alpha - 1)}{1 + e^\alpha} + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int, default=20 + Rolling window span for \(\beta\) estimation. + + Returns + ------- + pd.Series + The estimated spread vector. + """ + beta = beta_estimates(high_prices, low_prices, window_span) + gamma = gamma_estimates(high_prices, low_prices) + alpha = alpha_estimates(beta, gamma) + + # Calculate spread + spread = 2 * (np.exp(alpha) - 1) / (1 + np.exp(alpha)) + return spread diff --git a/src/RiskLabAI/features/microstructural_features/edge.py b/src/RiskLabAI/features/microstructural_features/edge.py new file mode 100644 index 0000000..8366f0a --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/edge.py @@ -0,0 +1,181 @@ +""" +Implements the EDGE effective bid-ask spread estimator +(Ardia, Guidotti & Kroencke 2024). + +Admitted in Appraisal 03 (CONTRIBUTIONS_LEDGER row 1, 2026-06-26). Regime tag, +verbatim from the verdict: + + Prefer EDGE over Roll and Abdi-Ranaldo for low-frequency spread estimation in + all regimes; over Corwin-Schultz at small spreads (the edge narrows at very high + illiquidity and very large spreads). + +EDGE pools the open, high, low and close prices and corrects for discrete, +infrequently traded data, giving lower bias and variance than the close-to-close +Roll (1984) estimator and the two-day high-low Corwin-Schultz (2012) estimator, and +never returning an invalid (negative) point estimate. The result is a proportional +spread (0.01 is a 1% spread). Real-data confirmation is a logged follow-up pending an +adequate public intraday / quote dataset (DECISION_LOG 2026-06-26). Evidence and +caveats: appraisals/03_verdict.md (03_results/RESULTS.md + HELDOUT.md). + +Reference: + Ardia, D., Guidotti, E., & Kroencke, T. A. (2024). Efficient estimation of + bid-ask spreads from open, high, low, and close prices. Journal of Financial + Economics, 161, 103916. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np + +_ArrayLike = Sequence[float] | np.ndarray + + +def _lag_nan(values: np.ndarray) -> np.ndarray: + """Shift by one period; the first element (no predecessor) becomes NaN.""" + lagged = np.empty_like(values) + lagged[0] = np.nan + lagged[1:] = values[:-1] + return lagged + + +def edge_estimator( + open: _ArrayLike, + high: _ArrayLike, + low: _ArrayLike, + close: _ArrayLike, + sign: bool = False, +) -> float: + r""" + Estimate the effective bid-ask spread using the EDGE estimator + (Ardia, Guidotti & Kroencke 2024). + + The estimator forms two unbiased squared-spread moment conditions from the log + open/high/low/close prices and a tick-rule trade indicator, then combines them + with the GMM-optimal, variance-weighted average: + + .. math:: + \hat{S}^2 = \frac{v_2 \, e_1 + v_1 \, e_2}{v_1 + v_2}, \qquad + \hat{S} = \sqrt{|\hat{S}^2|} + + where :math:`e_k` and :math:`v_k` are the mean and variance of the two estimators + (equal weight if the total variance is not positive). + + Parameters + ---------- + open, high, low, close : array-like + Four equal-length sequences of bar open, high, low and close prices + (arrays of bar prices, not a DataFrame). + sign : bool, default=False + If True, return a signed estimate (negative when the estimated squared + spread is negative); otherwise the spread magnitude is returned. + + Returns + ------- + float + The estimated proportional spread (0.01 is a 1% spread), or ``NaN`` when the + estimate is undefined (fewer than three observations, fewer than two traded + periods, or a degenerate / zero-variance input). + + Raises + ------ + ValueError + If the open, high, low and close sequences are not all the same length. + + Examples + -------- + >>> o = [100.0, 101.2, 100.5, 102.1, 101.8, 103.0] + >>> h = [101.0, 101.9, 101.3, 102.8, 102.5, 103.6] + >>> l = [99.4, 100.6, 99.9, 101.3, 101.0, 102.2] + >>> c = [101.1, 100.7, 101.0, 102.3, 101.6, 103.2] + >>> round(edge_estimator(o, h, l, c), 6) # doctest: +SKIP + 0.011176 + """ + open_prices = np.asarray(open, dtype=float) + high_prices = np.asarray(high, dtype=float) + low_prices = np.asarray(low, dtype=float) + close_prices = np.asarray(close, dtype=float) + + n = open_prices.shape[0] + if not ( + high_prices.shape[0] == n + and low_prices.shape[0] == n + and close_prices.shape[0] == n + ): + raise ValueError("open, high, low, and close must have the same length.") + if n < 3: + return float("nan") + + o = np.log(open_prices) + h = np.log(high_prices) + ll = np.log(low_prices) + c = np.log(close_prices) + m = (h + ll) / 2.0 + + h1 = _lag_nan(h) + l1 = _lag_nan(ll) + c1 = _lag_nan(c) + m1 = _lag_nan(m) + + # Log-returns; r1's first element is masked to align with the lagged quantities. + r1 = m - o + r1[0] = np.nan + r2 = o - m1 + r3 = m - c1 + r4 = c1 - m1 + r5 = o - c1 + + # Trade indicator: the bar traded if its range is non-zero or the low differs + # from the previous close. NaN where any required input is missing. + valid_tau = ~(np.isnan(h) | np.isnan(ll) | np.isnan(c1)) + tau = np.full(n, np.nan) + tau[valid_tau] = ( + (h[valid_tau] != ll[valid_tau]) | (ll[valid_tau] != c1[valid_tau]) + ).astype(float) + + def _indicator(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """(tau and a != b) as 0/1, NaN where tau or any required input is missing.""" + out = np.full(n, np.nan) + valid = ~(np.isnan(tau) | np.isnan(a) | np.isnan(b)) + out[valid] = ((tau[valid] == 1.0) & (a[valid] != b[valid])).astype(float) + return out + + po1 = _indicator(o, h) + po2 = _indicator(o, ll) + pc1 = _indicator(c1, h1) + pc2 = _indicator(c1, l1) + + pt = np.nanmean(tau) + po = np.nanmean(po1) + np.nanmean(po2) + pc = np.nanmean(pc1) + np.nanmean(pc2) + + if np.nansum(tau) < 2 or po == 0.0 or pc == 0.0: + return float("nan") + + # De-meaned log-returns, weighted by the trade indicator and trade probability. + d1 = r1 - (np.nanmean(r1) / pt) * tau + d3 = r3 - (np.nanmean(r3) / pt) * tau + d5 = r5 - (np.nanmean(r5) / pt) * tau + + # Two unbiased squared-spread estimators, from the open and the previous close. + x1 = (-4.0 / po) * d1 * r2 + (-4.0 / pc) * d3 * r4 + x2 = (-4.0 / po) * d1 * r5 + (-4.0 / pc) * d5 * r4 + + e1 = np.nanmean(x1) + e2 = np.nanmean(x2) + v1 = np.nanmean(x1 * x1) - e1 * e1 + v2 = np.nanmean(x2 * x2) - e2 * e2 + + # Variance-weighted (GMM-optimal) average of the two estimators; equal weight if + # the total variance is not positive. + total_variance = v1 + v2 + if total_variance > 0.0: + squared_spread = (v2 * e1 + v1 * e2) / total_variance + else: + squared_spread = (e1 + e2) / 2.0 + + spread = float(np.sqrt(np.abs(squared_spread))) + if sign and squared_spread < 0.0: + spread = -spread + return spread diff --git a/src/RiskLabAI/features/structural_breaks/__init__.py b/src/RiskLabAI/features/structural_breaks/__init__.py new file mode 100644 index 0000000..3e64182 --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/__init__.py @@ -0,0 +1,41 @@ +""" +RiskLabAI Structural Breaks Features Module + +Implements methods for detecting structural breaks in time series, +including the Augmented Dickey-Fuller (ADF) and (G)SADF tests +as described by de Prado, plus the Phillips-Shi-Yu (2015) GSADF / BSADF +multiple-bubble detector and date-stamping. +""" + +from .pelt_change_points import pelt_change_points +from .structural_breaks import ( + compute_beta, + get_bsadf_sequence, + get_bsadf_statistic, + get_bubble_episodes, + get_expanding_window_adf, + get_gsadf_statistic, + get_sadf_sequence, + lag_dataframe, + prepare_data, + psy_minimum_window, + simulate_psy_critical_values, +) +from .volatility_robust_sadf import volatility_robust_sadf + +__all__ = [ + "lag_dataframe", + "prepare_data", + "compute_beta", + "get_expanding_window_adf", + "get_bsadf_statistic", + "psy_minimum_window", + "get_sadf_sequence", + "get_bsadf_sequence", + "get_gsadf_statistic", + "get_bubble_episodes", + "simulate_psy_critical_values", + # Appraisal 26 admits (alongside SADF/GSADF/CUSUM) + "volatility_robust_sadf", + "pelt_change_points", +] diff --git a/src/RiskLabAI/features/structural_breaks/pelt_change_points.py b/src/RiskLabAI/features/structural_breaks/pelt_change_points.py new file mode 100644 index 0000000..dbb9a2b --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/pelt_change_points.py @@ -0,0 +1,98 @@ +r""" +PELT exact multiple change-point detection (Killick, Fearnhead and Eckley 2012). + +de Prado's CUSUM (Chu-Stinchcombe-White) structural-break test targets a single mean shift and is, by +construction, blind to a pure variance change. PELT (Pruned Exact Linear Time) finds the exact set of +multiple change-points that minimizes a segment cost plus a per-change penalty, so with a Gaussian +(mean-and-variance) cost it detects and dates multiple and variance change-points that CUSUM misses, +without over-segmenting when penalized appropriately. + +Admitted in Appraisal 26 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer it over CUSUM for detecting and dating multiple and/or variance change-points (which CUSUM + misses), without over-segmenting; CUSUM remains adequate for a single mean shift. + +Held-out confirmed (appraisals/26_results, HELDOUT.md): on the sealed mixed mean+variance corner (3 true +change-points) PELT recovers 2.65 of 3 on average with a 0.000 over-segmentation rate (it never exceeds +the true count), where CUSUM recovers far fewer of these mean+variance change-points. The CUSUM baseline +is unchanged. Evidence: appraisals/26_verdict.md. + +PELT is provided through the ``ruptures`` package (BSD-2-Clause), an optional dependency imported lazily; +the GPL-3 ``exuber`` reference was not used. Install it with ``pip install ruptures``. + +References +---------- +Killick, R., Fearnhead, P. and Eckley, I. A. (2012) Optimal detection of changepoints with a linear + computational cost. Journal of the American Statistical Association, 107(500), 1590-1598. +""" + +from __future__ import annotations + +import numpy as np + + +def pelt_change_points( + series, + model: str = "normal", + penalty_multiplier: float = 3.0, + min_size: int = 10, + jump: int = 5, +) -> list: + r""" + Detect and date change-points with PELT (exact multiple change-point detection). + + Fits a PELT model with a BIC-style penalty ``penalty_multiplier * log(T)`` and returns the interior + change indices (the segment boundaries, excluding the series endpoints). With ``model="normal"`` the + Gaussian cost captures both mean and variance changes, so PELT recovers multiple and variance + change-points that CUSUM misses, without over-segmenting under an adequate penalty. + + Prefer PELT over CUSUM for detecting and dating multiple and/or variance change-points; CUSUM remains + adequate for a single mean shift. See the module docstring for the verbatim regime tag and + appraisals/26_verdict.md. + + Parameters + ---------- + series : array-like + The 1-D series to segment. + model : str, default "normal" + ``ruptures`` cost model (``"normal"`` for mean+variance, ``"l2"`` for mean only, ``"rbf"`` ...). + penalty_multiplier : float, default 3.0 + Penalty ``= penalty_multiplier * log(T)`` (higher penalizes more change-points). + min_size : int, default 10 + Minimum segment length. + jump : int, default 5 + Grid stride for candidate change-points (``ruptures`` speed knob). + + Returns + ------- + list of int + The interior change-point indices, ascending (empty if none). + + Raises + ------ + ImportError + If ``ruptures`` is not installed (it is an optional dependency). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = np.concatenate([rng.standard_normal(120), rng.standard_normal(120) + 3.0]) + >>> cps = pelt_change_points(x) # doctest: +SKIP + >>> any(100 < c < 140 for c in cps) # doctest: +SKIP + True + """ + try: + import ruptures as rpt + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError( + "pelt_change_points requires ruptures (pip install ruptures)" + ) from exc + + x = np.asarray(series, dtype=float).reshape(-1, 1) + n = x.shape[0] + if n < 2 * int(min_size): + return [] + algorithm = rpt.Pelt(model=model, min_size=int(min_size), jump=int(jump)).fit(x) + breakpoints = algorithm.predict(pen=float(penalty_multiplier) * np.log(n)) + return [int(b) for b in breakpoints if 0 < b < n] diff --git a/src/RiskLabAI/features/structural_breaks/structural_breaks.py b/src/RiskLabAI/features/structural_breaks/structural_breaks.py new file mode 100644 index 0000000..4ce4a5e --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/structural_breaks.py @@ -0,0 +1,708 @@ +""" +Implements the core logic for the (G)SADF (Generalized) Supreme +Augmented Dickey-Fuller test for structural breaks. + +References: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 17. (SADF, Phillips-Wu-Yu 2011.) + Phillips, P. C. B., Shi, S. and Yu, J. (2015) Testing for multiple bubbles: + historical episodes of exuberance and collapse in the S&P 500. + International Economic Review, 56(4), 1043-1078. (GSADF / BSADF.) +""" + +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd + + +def lag_dataframe( + market_data: pd.DataFrame, lags: Union[int, list[int]] +) -> pd.DataFrame: + """ + Apply lags to a DataFrame. + + Reference: + Snippet 17.3, Page 253. + + Parameters + ---------- + market_data : pd.DataFrame + DataFrame of price or log price. + lags : Union[int, List[int]] + An integer number of lags (e.g., 3 creates lags 0, 1, 2, 3) + or a specific list of lags to create. + + Returns + ------- + pd.DataFrame + A DataFrame with lagged columns, e.g., 'price_0', 'price_1', ... + """ + lagged_parts = [] + + if isinstance(lags, int): + lags_list = range(lags + 1) + else: + lags_list = [int(lag) for lag in lags] + + for lag in lags_list: + lagged_data = market_data.shift(lag) + lagged_data.columns = [f"{col}_{lag}" for col in market_data.columns] + lagged_parts.append(lagged_data) + + return pd.concat(lagged_parts, axis=1) + + +def prepare_data( + log_price_series: pd.Series, constant: str, lags: int +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Prepare the y and X matrices for ADF regression. + + Reference: + Snippet 17.3, Page 253. + + Parameters + ---------- + log_price_series : pd.Series # <-- CHANGED: Accept Series + Series of log prices. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - y_df: The dependent variable (delta log price). + - x_df: The independent variables (lagged level, lagged deltas, constants). + """ + # <-- ADDED: Convert univariate series to frame for internal processing + log_price = log_price_series.to_frame() + + price_diff = log_price.diff().dropna() + y_df = price_diff + + # 1. Lagged level + x_df = log_price.shift(1).copy() + x_df.columns = ["level_l1"] + + # 2. Lagged deltas + if lags > 0: + lagged_deltas = price_diff.shift(1) + lagged_deltas.columns = ["delta_l1"] + + if lags > 1: + for i in range(2, lags + 1): + lagged_deltas[f"delta_l{i}"] = price_diff.shift(i) + + x_df = x_df.join(lagged_deltas, how="outer") + + # 3. Add constants + if constant == "c": + x_df["constant"] = 1 + elif constant == "ct": + x_df["constant"] = 1 + x_df["trend"] = np.arange(1, len(x_df) + 1) + elif constant == "ctt": + x_df["constant"] = 1 + x_df["trend"] = np.arange(1, len(x_df) + 1) + x_df["trend_sq"] = x_df["trend"] ** 2 + + # Align y and X by dropping NaNs created by lagging + combined = y_df.join(x_df, how="inner").dropna() + + y_df = combined.iloc[:, [0]] + x_df = combined.iloc[:, 1:] + + return y_df, x_df + + +def compute_beta( + y_window: np.ndarray, x_window: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """ + Compute OLS beta coefficients and their variance. + + Reference: + Snippet 17.2, Page 251. + + Parameters + ---------- + y_window : np.ndarray + Window of the dependent variable. + x_window : np.ndarray + Window of the independent variables. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - beta_mean: The OLS coefficients. + - beta_variance: The variance-covariance matrix of the coefficients. + """ + try: + xt_x_inv = np.linalg.inv(x_window.T @ x_window) + xt_y = x_window.T @ y_window + + beta_mean = xt_x_inv @ xt_y + + error = y_window - (x_window @ beta_mean) + variance_e = (error.T @ error) / (x_window.shape[0] - x_window.shape[1]) + beta_variance = variance_e * xt_x_inv + + return beta_mean, beta_variance + + except np.linalg.LinAlgError: + # Handle singular matrix + return np.full((x_window.shape[1], 1), np.nan), np.full( + (x_window.shape[1], x_window.shape[1]), np.nan + ) + + +def get_expanding_window_adf( + log_price: pd.Series, + min_sample_length: int, + constant: str, + lags: int, +) -> pd.Series: + """ + Compute the ADF t-statistic over an expanding window. + + This is useful for plotting the evolution of the test statistic over time. + + Reference: + Based on Snippet 17.2, Page 251. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. + min_sample_length : int + The minimum number of samples to start the expanding window. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include in the regression. + + Returns + ------- + pd.Series + A Series of ADF t-statistics, indexed by timestamp. + """ + # <-- CHANGED: Pass Series directly to prepare_data + y_df, x_df = prepare_data(log_price, constant=constant, lags=lags) + + adf_stats = [] + timestamps = [] + + for i in range(min_sample_length, y_df.shape[0] + 1): + y_window = y_df.iloc[:i].values + x_window = x_df.iloc[:i].values + + beta_mean, beta_variance = compute_beta(y_window, x_window) + + if np.isnan(beta_variance[0, 0]): + t_stat = np.nan + else: + beta_std_level = beta_variance[0, 0] ** 0.5 + if beta_std_level == 0: + t_stat = -np.inf if beta_mean[0, 0] < 0 else np.inf + else: + t_stat = beta_mean[0, 0] / beta_std_level + + adf_stats.append(t_stat) + timestamps.append(y_df.index[i - 1]) + + return pd.Series(adf_stats, index=timestamps) + + +def get_bsadf_statistic( + log_price: pd.Series, # <-- CHANGED: Accept Series + min_sample_length: int, + constant: str, + lags: int, +) -> dict[str, Any]: + """ + Compute the Backward Supremum ADF (BSADF) statistic. + + This test runs an expanding ADF test starting from every possible + point in the series and finds the supremum (highest) t-statistic. + This is used to detect the *origination* of a bubble. + + Reference: + Snippet 17.4, Page 253. (Renamed from `adf` for clarity). + + Parameters + ---------- + log_price : pd.Series # <-- CHANGED + Series of log prices. + min_sample_length : int + Minimum sample length for each ADF test. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include. + + Returns + ------- + Dict[str, Any] + A dictionary containing: + - 'Time': The timestamp of the end of the series. + - 'bsadf': The BSADF statistic (the supremum ADF). + """ + # 1. Prepare the full X, y matrices + # <-- CHANGED: Pass Series directly + y, x = prepare_data(log_price, constant=constant, lags=lags) + + # 2. Define all possible start points + # <-- BUG FIX: Removed '+ lags' from the range + start_points = range(0, y.shape[0] - min_sample_length + 1) + bsadf = -np.inf # Supremum ADF + + y_np, x_np = y.values, x.values + + # 3. Loop over all expanding windows + for start in start_points: + y_window, x_window = y_np[start:], x_np[start:] + + # 4. Compute ADF regression for this window + beta_mean, beta_variance = compute_beta(y_window, x_window) + + if np.isnan(beta_variance[0, 0]): + continue + + # 5. Get t-statistic for the first coefficient (the level) + beta_mean_level = beta_mean[0, 0] + beta_std_level = beta_variance[0, 0] ** 0.5 + + if beta_std_level == 0: + t_stat = -np.inf if beta_mean_level < 0 else np.inf + else: + t_stat = beta_mean_level / beta_std_level + + if t_stat > bsadf: + bsadf = t_stat + + return {"Time": log_price.index[-1], "bsadf": bsadf} + + +# ---------------------------------------------------------------------------------- +# GSADF / BSADF (Phillips-Shi-Yu 2015): the generalized sup-ADF and its backward +# sequence, with date-stamping and a finite-sample critical-value simulator. +# +# de Prado (AFML ch.17) ships the single-window SADF: the forward-expanding ADF is +# anchored at the sample origin, so several explosive episodes dilute and mask one +# another and SADF collapses them to about one. GSADF varies *both* window endpoints +# and the backward BSADF sequence dates each origination and collapse, recovering and +# counting multiple bubbles where SADF sees one. The functions below build directly +# on the validated ``get_bsadf_statistic`` / ``get_expanding_window_adf`` above (and +# agree with them to ~1e-9 on shared windows; see the structural-breaks tests). +# +# Admitted in Appraisal 05 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim: +# Prefer GSADF/BSADF over single-window SADF when a series may contain more than +# one explosive episode (it recovers and counts each). For a single suspected +# bubble, SADF is at least as good. Use seasonally-adjusted data and the simulated +# finite-sample critical values; GSADF over-flags strongly trending or seasonal +# series and is mildly oversized in large samples, so treat broad flags cautiously. +# Evidence and caveats: appraisals/05_verdict.md. +# ---------------------------------------------------------------------------------- + + +def psy_minimum_window(sample_length: int) -> int: + """ + Phillips-Shi-Yu minimum window length, in observations. + + PSY (2015) set the minimum window fraction to r0 = 0.01 + 1.8 / sqrt(T) and take + the minimum window as round(r0 * T), the standard rule used to simulate the + finite-sample critical values. + + Parameters + ---------- + sample_length : int + The series length T (number of price observations). + + Returns + ------- + int + The minimum window length in observations (at least 3). + """ + r0 = 0.01 + 1.8 / (sample_length**0.5) + return max(int(round(r0 * sample_length)), 3) + + +def _psy_sadf_bsadf_sequences( + y: np.ndarray, min_sample_length: int +) -> tuple[np.ndarray, np.ndarray]: + """ + Fast prefix-sum kernel for the PSY-standard ADF specification (intercept, no lag + augmentation: Delta y_t = alpha + beta y_{t-1} + e_t). + + Returns ``(sadf_seq, bsadf_seq)``, each of length ``len(y)`` and indexed by the + window endpoint (price index), with ``NaN`` before the first valid endpoint: + + - ``sadf_seq[r2]`` = ADF(0, r2) (forward-expanding, PWY) + - ``bsadf_seq[r2]`` = sup_{r1 in [0, r2 - nmin]} ADF(r1, r2) (backward sup, PSY) + + The ADF t-statistic of the lagged-level coefficient for any window is computed in + O(1) from prefix sums of the regression observations, so the whole sequence is + O(T^2). Clean-room from Phillips-Shi-Yu (2015); it agrees with the loop-based + ``get_bsadf_statistic`` / ``get_expanding_window_adf`` to ~1e-9 on shared windows. + """ + y = np.ascontiguousarray(np.asarray(y, dtype=np.float64)) + length = y.shape[0] + sadf = np.full(length, np.nan) + bsadf = np.full(length, np.nan) + if length < min_sample_length + 1: + return sadf, bsadf + + # Regression observation i (0-based) uses level x_i = y[i] and delta d_i = y[i+1]-y[i]. + x = y[:-1] + d = y[1:] - y[:-1] + + # Prefix sums (length m+1, leading 0) so any window's sufficient statistics are O(1). + cx = np.concatenate(([0.0], np.cumsum(x))) + cxx = np.concatenate(([0.0], np.cumsum(x * x))) + cd = np.concatenate(([0.0], np.cumsum(d))) + cxd = np.concatenate(([0.0], np.cumsum(x * d))) + cdd = np.concatenate(([0.0], np.cumsum(d * d))) + + for r2 in range(min_sample_length, length): + # Window [r1, r2] uses regression observations [r1, r2); n = r2 - r1. + r1 = np.arange(0, r2 - min_sample_length + 1) + n = (r2 - r1).astype(np.float64) + sx = cx[r2] - cx[r1] + sxx = cxx[r2] - cxx[r1] + sd = cd[r2] - cd[r1] + sxd = cxd[r2] - cxd[r1] + sdd = cdd[r2] - cdd[r1] + den = n * sxx - sx * sx + with np.errstate(divide="ignore", invalid="ignore"): + beta = (n * sxd - sx * sd) / den + alpha = (sd - beta * sx) / n + ssr = np.maximum(sdd - alpha * sd - beta * sxd, 0.0) + var_beta = (ssr / (n - 2)) * n / den + t_stat = beta / np.sqrt(var_beta) + valid = (den > 0.0) & (n >= 3) & (var_beta > 0.0) + t_stat = np.where(valid, t_stat, -np.inf) + best = t_stat.max() + bsadf[r2] = best if np.isfinite(best) else np.nan + # r1 == 0 is the forward-expanding (origin-anchored) ADF = SADF date-stamp. + sadf[r2] = t_stat[0] if valid[0] else np.nan + + return sadf, bsadf + + +def get_sadf_sequence( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> pd.Series: + """ + Forward-expanding ADF (SADF) sequence, indexed by window endpoint. + + This is the date-stamping sequence for de Prado's single-window SADF: each point + is the ADF t-statistic of an expanding window anchored at the sample origin. It is + a thin wrapper over :func:`get_expanding_window_adf` (and the fast prefix-sum + kernel for the standard intercept-only, no-lag specification) so the SADF and GSADF + date-stamps share an aligned index. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values (e.g. FRED holiday gaps) are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms (PSY simulate with 0). + + Returns + ------- + pd.Series + The SADF t-statistic at each endpoint, indexed by timestamp. + """ + log_price = log_price.dropna() + if constant == "c" and lags == 0: + sadf, _ = _psy_sadf_bsadf_sequences(log_price.to_numpy(), min_sample_length) + endpoints = log_price.index[min_sample_length:] + return pd.Series(sadf[min_sample_length:], index=endpoints) + return get_expanding_window_adf(log_price, min_sample_length, constant, lags) + + +def get_bsadf_sequence( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> pd.Series: + """ + Backward sup-ADF (BSADF) sequence, indexed by window endpoint (Phillips-Shi-Yu). + + For each endpoint the statistic is the supremum of the ADF t-statistic over all + window start points, so a later episode is tested on its own sub-sample rather than + diluted by the earlier sample. Comparing this sequence against a critical-value + sequence (see :func:`simulate_psy_critical_values`) date-stamps each bubble in real + time; see :func:`get_bubble_episodes`. + + For the standard intercept-only, no-lag specification this uses a fast O(T^2) + prefix-sum kernel; for other constants or ``lags > 0`` it reuses the validated + :func:`get_bsadf_statistic` on expanding prefixes. Both agree to ~1e-9. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + + Returns + ------- + pd.Series + The BSADF t-statistic at each endpoint, indexed by timestamp. + """ + log_price = log_price.dropna() + if constant == "c" and lags == 0: + _, bsadf = _psy_sadf_bsadf_sequences(log_price.to_numpy(), min_sample_length) + endpoints = log_price.index[min_sample_length:] + return pd.Series(bsadf[min_sample_length:], index=endpoints) + + values, timestamps = [], [] + first_end = min_sample_length + lags + 1 + for end in range(first_end, len(log_price) + 1): + result = get_bsadf_statistic( + log_price.iloc[:end], min_sample_length, constant, lags + ) + values.append(result["bsadf"]) + timestamps.append(result["Time"]) + return pd.Series(values, index=timestamps) + + +def get_gsadf_statistic( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> float: + """ + Generalized sup-ADF (GSADF) statistic (Phillips-Shi-Yu 2015). + + The GSADF is the supremum of the backward sup-ADF (BSADF) sequence over all + flexible windows, i.e. ``max`` of :func:`get_bsadf_sequence`. A series is flagged + as containing at least one explosive episode when the GSADF exceeds its + finite-sample critical value (:func:`simulate_psy_critical_values`). + + Prefer GSADF/BSADF over single-window SADF when a series may contain more than one + explosive episode (it recovers and counts each); for a single suspected bubble, + SADF is at least as good. Use seasonally-adjusted data and the simulated + finite-sample critical values; GSADF over-flags strongly trending or seasonal + series and is mildly oversized in large samples, so treat broad flags cautiously. + (Admitted in Appraisal 05; see appraisals/05_verdict.md.) + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + + Returns + ------- + float + The GSADF statistic, or ``NaN`` if the series is too short. + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> y = np.cumsum(rng.standard_normal(200)) # random walk + >>> for t in range(120, 150): # embed an explosive episode + ... y[t] = 1.06 * y[t - 1] + rng.standard_normal() + >>> price = pd.Series(y) + >>> nmin = psy_minimum_window(len(price)) + >>> gsadf = get_gsadf_statistic(price, nmin) + >>> cv = simulate_psy_critical_values(len(price), nmin, n_simulations=200) + >>> bool(gsadf > cv["gsadf_global_cv"]) # bubble detected + True + """ + bsadf_seq = get_bsadf_sequence(log_price, min_sample_length, constant, lags) + values = bsadf_seq.to_numpy() + if values.size == 0 or not np.isfinite(values).any(): + return float("nan") + return float(np.nanmax(values)) + + +def get_bubble_episodes( + statistic_sequence: pd.Series, + critical_value_sequence: Union[pd.Series, np.ndarray, float], + min_duration: int = 1, +) -> list[tuple[Any, Any]]: + """ + Date-stamp explosive episodes from a sup-ADF sequence (Phillips-Shi-Yu 2015). + + An episode is a maximal run of endpoints where the statistic exceeds its + critical value and that lasts at least ``min_duration`` periods. The origination + is the first such period; the collapse is the first period the statistic falls + back below the critical value (or the last endpoint if it is still above at the + end of the sample). The same rule applied to the BSADF sequence date-stamps GSADF + episodes and applied to the SADF sequence date-stamps the single-window detector, + so the two are directly comparable. + + Parameters + ---------- + statistic_sequence : pd.Series + A BSADF or SADF sequence (see :func:`get_bsadf_sequence`, + :func:`get_sadf_sequence`). + critical_value_sequence : pd.Series, np.ndarray or float + The per-endpoint critical-value sequence (aligned with + ``statistic_sequence``) or a single scalar critical value. + min_duration : int, default 1 + Minimum run length, in periods, for a run to count as an episode. PSY use a + small minimum such as ``round(log(T))``. + + Returns + ------- + list of (origination, collapse) + Index labels (timestamps) bounding each detected episode, in order. + """ + stat = statistic_sequence.to_numpy(dtype=float) + index = statistic_sequence.index + n = stat.shape[0] + if isinstance(critical_value_sequence, pd.Series): + cv = critical_value_sequence.reindex(index).to_numpy(dtype=float) + elif np.isscalar(critical_value_sequence): + cv = np.full(n, float(critical_value_sequence)) + else: + cv = np.asarray(critical_value_sequence, dtype=float) + + above = np.isfinite(stat) & np.isfinite(cv) & (stat > cv) + episodes: list[tuple[Any, Any]] = [] + t = 0 + while t < n: + if above[t]: + start = t + while t < n and above[t]: + t += 1 + end = t # first endpoint back below the critical value (exclusive) + if end - start >= min_duration: + collapse = index[end] if end < n else index[-1] + episodes.append((index[start], collapse)) + else: + t += 1 + return episodes + + +_CRITICAL_VALUE_CACHE: dict[tuple, dict[str, Any]] = {} + + +def simulate_psy_critical_values( + sample_length: int, + min_sample_length: Optional[int] = None, + constant: str = "c", + lags: int = 0, + n_simulations: int = 2000, + level: float = 0.95, + seed: int = 0, +) -> dict[str, Any]: + """ + Finite-sample critical values for SADF and GSADF, by simulating the random-walk + null (the standard Phillips-Shi-Yu approach), with caching. + + The ADF-family statistics are pivotal under the random-walk null, so the values + depend only on the sample length, the minimum window and the regression + specification, not on the data's level or scale. For each of ``n_simulations`` + independent random-walk paths the SADF and BSADF sequences are computed; the + ``level`` quantile of the global statistic across paths gives the global critical + value (used for detection), and the ``level`` quantile per endpoint gives the + critical-value sequence (used for date-stamping; see :func:`get_bubble_episodes`). + + Parameters + ---------- + sample_length : int + The series length T to simulate. + min_sample_length : int, optional + Minimum window length in observations. Defaults to + :func:`psy_minimum_window`. + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + n_simulations : int, default 2000 + Number of random-walk paths. + level : float, default 0.95 + Quantile of the null distribution (a 95% test targets a 5% size). + seed : int, default 0 + Base seed; path ``s`` uses ``seed + s`` for reproducibility. + + Returns + ------- + dict + Keys ``sample_length``, ``min_sample_length``, ``level``, + ``sadf_global_cv``, ``gsadf_global_cv`` (floats), and + ``sadf_sequence_cv``, ``bsadf_sequence_cv`` (per-endpoint critical-value + arrays aligned with :func:`get_sadf_sequence` / :func:`get_bsadf_sequence`). + """ + if min_sample_length is None: + min_sample_length = psy_minimum_window(sample_length) + + cache_key = ( + sample_length, + min_sample_length, + constant, + lags, + n_simulations, + level, + seed, + ) + if cache_key in _CRITICAL_VALUE_CACHE: + return _CRITICAL_VALUE_CACHE[cache_key] + + fast = constant == "c" and lags == 0 + sadf_paths = np.full((n_simulations, sample_length), np.nan) + bsadf_paths = np.full((n_simulations, sample_length), np.nan) + for s in range(n_simulations): + rng = np.random.default_rng(seed + s) + y = np.cumsum(rng.standard_normal(sample_length)) # random-walk null + if fast: + sadf, bsadf = _psy_sadf_bsadf_sequences(y, min_sample_length) + else: + price = pd.Series(y) + sadf = ( + get_sadf_sequence(price, min_sample_length, constant, lags) + .reindex(range(sample_length)) + .to_numpy() + ) + bsadf = ( + get_bsadf_sequence(price, min_sample_length, constant, lags) + .reindex(range(sample_length)) + .to_numpy() + ) + sadf_paths[s] = sadf + bsadf_paths[s] = bsadf + + global_sadf = np.nanmax(sadf_paths, axis=1) + global_gsadf = np.nanmax(bsadf_paths, axis=1) + # Quantiles only over the valid endpoint columns (the leading columns are all NaN). + sadf_seq_cv = np.nanquantile(sadf_paths[:, min_sample_length:], level, axis=0) + bsadf_seq_cv = np.nanquantile(bsadf_paths[:, min_sample_length:], level, axis=0) + + result = { + "sample_length": sample_length, + "min_sample_length": min_sample_length, + "level": level, + "sadf_global_cv": float(np.nanquantile(global_sadf, level)), + "gsadf_global_cv": float(np.nanquantile(global_gsadf, level)), + "sadf_sequence_cv": sadf_seq_cv, + "bsadf_sequence_cv": bsadf_seq_cv, + } + _CRITICAL_VALUE_CACHE[cache_key] = result + return result diff --git a/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py b/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py new file mode 100644 index 0000000..c8096a7 --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py @@ -0,0 +1,126 @@ +r""" +Volatility-robust SADF / GSADF via wild-bootstrap critical values (Harvey-Leybourne-Sollis-Taylor 2016). + +The admitted SADF / GSADF bubble detector (`structural_breaks.get_sadf_sequence` / +`get_gsadf_statistic`) uses a sup-ADF statistic whose null distribution assumes homoskedastic errors. +Under non-stationary volatility the test over-rejects, flagging spurious bubbles (the mild oversizing +noted in Appraisal 05). The volatility-robust variant computes the SAME sup-ADF statistics but calibrates +the critical values by a wild bootstrap (Rademacher sign-flip of the first-difference residuals), which +preserves the series' own volatility pattern while destroying any explosive autocorrelation, restoring +correct size under non-stationary volatility. + +Admitted in Appraisal 26 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer it over plain SADF/GSADF when the series' volatility may be non-stationary: it holds nominal + size where plain SADF over-rejects ~9x, at a modest power cost, and converges to plain SADF under + constant volatility. Pairs with the admitted GSADF/BSADF. + +Held-out confirmed (appraisals/26_results, HELDOUT.md): on the sealed variance-break-6x bubble null the +volatility-robust SADF holds nominal size (~0.067) where plain SADF over-rejects (~0.667, the homoskedastic +critical value flags spurious bubbles), and it matches plain SADF under constant volatility at a modest +power cost. The plain SADF / GSADF baseline is unchanged. Evidence: appraisals/26_verdict.md. + +References +---------- +Harvey, D. I., Leybourne, S. J., Sollis, R. and Taylor, A. M. R. (2016) Tests for explosive financial + bubbles in the presence of non-stationary volatility. Journal of Empirical Finance, 38, 548-574. +Phillips, P. C. B., Shi, S. and Yu, J. (2015) Testing for multiple bubbles. International Economic Review. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from .structural_breaks import _psy_sadf_bsadf_sequences, psy_minimum_window + + +def _sup_adf_statistics(log_price: np.ndarray, min_length: int) -> tuple: + sadf_seq, bsadf_seq = _psy_sadf_bsadf_sequences( + np.asarray(log_price, float), min_length + ) + sadf = float(np.nanmax(sadf_seq)) if np.isfinite(sadf_seq).any() else np.nan + gsadf = float(np.nanmax(bsadf_seq)) if np.isfinite(bsadf_seq).any() else np.nan + return sadf, gsadf + + +def volatility_robust_sadf( + log_price: np.ndarray | pd.Series, + min_length: int | None = None, + n_bootstrap: int = 99, + random_state: int | None = None, +) -> dict: + r""" + Volatility-robust SADF / GSADF wild-bootstrap test for an explosive (bubble) episode. + + Computes the observed SADF and GSADF sup-ADF statistics, then calibrates their p-values by a wild + bootstrap: each bootstrap path multiplies the demeaned first-difference residuals by an independent + Rademacher (+/- 1) sign and re-cumulates, preserving the series' (possibly non-stationary) volatility + while removing any explosive autocorrelation; the sup-ADF statistics are recomputed on each path to + form the null distribution. The p-value is the share of bootstrap statistics at least as large as the + observed one. + + Prefer this over plain SADF / GSADF when the series' volatility may be non-stationary: it holds nominal + size where the plain test over-rejects, at a modest power cost, and converges to the plain test under + constant volatility. It pairs with the admitted GSADF / BSADF (the same sup-ADF statistics). See the + module docstring for the verbatim regime tag and appraisals/26_verdict.md. + + Parameters + ---------- + log_price : np.ndarray or pd.Series + The log-price level series. + min_length : int, optional + Minimum (PSY) window for the sup-ADF recursion; defaults to ``psy_minimum_window(len)``. + n_bootstrap : int, default 99 + Wild-bootstrap replications. + random_state : int, optional + Seed for the Rademacher signs. + + Returns + ------- + dict + ``sadf``, ``gsadf`` (observed statistics), ``sadf_pvalue``, ``gsadf_pvalue`` (wild-bootstrap), + and ``reject_sadf`` / ``reject_gsadf`` (at the 5% level). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> y = np.cumsum(rng.standard_normal(200)) # a random walk (no bubble) + >>> res = volatility_robust_sadf(y, n_bootstrap=49, random_state=0) + >>> set(res) == {"sadf", "gsadf", "sadf_pvalue", "gsadf_pvalue", "reject_sadf", "reject_gsadf"} + True + """ + y = np.asarray( + log_price.values if isinstance(log_price, pd.Series) else log_price, dtype=float + ) + if y.ndim != 1 or y.size < 8: + raise ValueError("log_price must be a 1-D series of length >= 8") + nmin = int(min_length) if min_length is not None else psy_minimum_window(y.size) + sadf_obs, gsadf_obs = _sup_adf_statistics(y, nmin) + + residual = np.diff(y) + residual = residual - residual.mean() + y0 = y[0] + rng = np.random.default_rng(random_state) + ge_sadf = ge_gsadf = 0 + for _ in range(n_bootstrap): + signs = rng.integers(0, 2, size=residual.shape[0]) * 2 - 1 + path = np.empty_like(y) + path[0] = y0 + path[1:] = y0 + np.cumsum(signs * residual) + sadf_b, gsadf_b = _sup_adf_statistics(path, nmin) + if np.isfinite(sadf_b) and np.isfinite(sadf_obs) and sadf_b >= sadf_obs: + ge_sadf += 1 + if np.isfinite(gsadf_b) and np.isfinite(gsadf_obs) and gsadf_b >= gsadf_obs: + ge_gsadf += 1 + p_sadf = (1 + ge_sadf) / (n_bootstrap + 1) + p_gsadf = (1 + ge_gsadf) / (n_bootstrap + 1) + return { + "sadf": sadf_obs, + "gsadf": gsadf_obs, + "sadf_pvalue": float(p_sadf), + "gsadf_pvalue": float(p_gsadf), + "reject_sadf": bool(p_sadf < 0.05), + "reject_gsadf": bool(p_gsadf < 0.05), + } diff --git a/src/RiskLabAI/hpc/__init__.py b/src/RiskLabAI/hpc/__init__.py new file mode 100644 index 0000000..b62b19e --- /dev/null +++ b/src/RiskLabAI/hpc/__init__.py @@ -0,0 +1,27 @@ +""" +RiskLabAI High-Performance Computing (HPC) Module + +Provides utilities for multiprocessing and parallel execution. +""" + +from .hpc import ( + expand_call, + linear_partitions, + mp_pandas_obj, + nested_partitions, + parallel_run, + process_jobs, + process_jobs_sequential, + report_progress, +) + +__all__ = [ + "report_progress", + "expand_call", + "process_jobs", + "process_jobs_sequential", + "linear_partitions", + "nested_partitions", + "mp_pandas_obj", + "parallel_run", +] diff --git a/src/RiskLabAI/hpc/hpc.py b/src/RiskLabAI/hpc/hpc.py new file mode 100644 index 0000000..03b3026 --- /dev/null +++ b/src/RiskLabAI/hpc/hpc.py @@ -0,0 +1,379 @@ +""" +High-Performance Computing (HPC) utilities for parallel processing. + +Provides a set of functions to parallelize operations, especially +on pandas objects, using Python's multiprocessing and Joblib. +""" + +import datetime as dt +import logging +import multiprocessing as mp +import time +from collections.abc import Iterable +from typing import Any, Callable, Optional, Union + +import joblib # <-- Added missing import +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + + +def parallel_run( + func: Callable[..., Any], + iterable: Iterable[Any], + num_cpus: int = -1, + lin_partition: bool = False, + **kwargs, +) -> list[Any]: + """ + Executes a function in parallel over an iterable using Joblib. + + Parameters + ---------- + func : Callable + The function to be parallelized. + iterable : Iterable + The iterable to loop over. + num_cpus : int, default=-1 + The number of CPUs to use. -1 means all available CPUs. + lin_partition : bool, default=False + How to partition the work: + - If False (default): Item-by-item. `func` receives one item + from `iterable` at a time. + - If True: Chunked. `iterable` is split into `num_cpus` chunks + (by index). `func` must be designed to receive a list of + indices (e.g., [0, 1, 2]) and process them. + **kwargs : + Additional keyword arguments to be passed to `func`. + + Returns + ------- + List[Any] + A list of the results from all parallel executions. + """ + if num_cpus == -1: + num_cpus = mp.cpu_count() + + if lin_partition: + # --- Chunked Partitioning --- + # func must accept a list of indices + # NOTE: This implies `iterable` must support `len()`. + try: + iterable_len = len(iterable) + except TypeError: + iterable = list(iterable) + iterable_len = len(iterable) + + num_atoms = num_cpus + iterable_partition = np.array_split(range(iterable_len), num_atoms) + jobs = (iterable_partition[i] for i in range(num_atoms)) + + results = joblib.Parallel(n_jobs=num_cpus)( + joblib.delayed(func)(job, **kwargs) for job in jobs + ) + + # Flatten the list of lists + return [item for sublist in results for item in sublist] + + else: + # --- Item-by-Item Partitioning (Standard) --- + # func accepts a single item + # <-- FIXED: Pass iterable directly. Joblib handles generators + # and lists efficiently. The original implementation was buggy. + results = joblib.Parallel(n_jobs=num_cpus)( + joblib.delayed(func)(job, **kwargs) for job in iterable + ) + return results + + +def report_progress( + job_number: int, total_jobs: int, start_time: float, task: str +) -> None: + """ + Report the progress of a computing task to the console. + + Parameters + ---------- + job_number : int + The current job number (e.g., `i` in a loop). + total_jobs : int + The total number of jobs. + start_time : float + The time the process started (e.g., `time.time()`). + task : str + A description of the task being performed. + """ + progress = job_number / total_jobs + elapsed_time_min = (time.time() - start_time) / 60.0 + + # Avoid division by zero if progress is 0 + remaining_time_min = 0.0 + if progress > 0: + remaining_time_min = elapsed_time_min * (1 / progress - 1) + + timestamp = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + message = ( + f"{timestamp} {progress*100:.2f}% {task} done after " + f"{elapsed_time_min:.2f} minutes. Remaining {remaining_time_min:.2f} minutes." + ) + + # Progress is emitted via logging (configure the 'RiskLabAI' logger to see + # it; the library is silent by default). + logger.info(message) + + +def expand_call(kargs: dict[str, Any]) -> Any: + """ + Wrapper function to expand keyword arguments for a callback. + + This is used by `process_jobs` to unpack a dictionary of arguments + and call the target function. + + Parameters + ---------- + kargs : dict + A dictionary of arguments, which *must* include a 'func' key + mapping to the function to be called. + + Returns + ------- + Any + The output of the callback function. + """ + func = kargs.pop("func") + return func(**kargs) + + +def process_jobs( + jobs: list[dict[str, Any]], task: Optional[str] = None, num_threads: int = -1 +) -> list[Any]: + """ + Process a list of jobs in parallel using multiprocessing. + + Parameters + ---------- + jobs : List[Dict[str, Any]] + A list of job dictionaries. Each dict must contain a 'func' key + and all arguments required by that function. + task : str, optional + A name for the task, used for progress reporting. If None, + the function name from the first job is used. + num_threads : int, default=-1 + The number of parallel processes to spawn. + -1 means use all available CPUs. + 1 means run sequentially (for debugging). + + Returns + ------- + List[Any] + A list containing the results from all jobs. + """ + if task is None: + task = jobs[0]["func"].__name__ + + # <-- Handle sequential case for debugging + if num_threads == 1: + logger.debug("Running %d jobs sequentially for debugging.", len(jobs)) + return process_jobs_sequential(jobs) + + # <-- Handle -1 for all CPUs + if num_threads == -1: + num_threads = mp.cpu_count() + + with mp.Pool(processes=num_threads) as pool: + outputs = [] + start_time = time.time() + + # Use imap_unordered for efficient processing + imap_results = pool.imap_unordered(expand_call, jobs) + + # Process results as they complete + for i, result in enumerate(imap_results, 1): + outputs.append(result) + report_progress(i, len(jobs), start_time, task) + + return outputs + + +def process_jobs_sequential(jobs: list[dict[str, Any]]) -> list[Any]: + """ + Run jobs sequentially (single-thread). Useful for debugging. + + Parameters + ---------- + jobs : List[Dict[str, Any]] + A list of job dictionaries. + + Returns + ------- + List[Any] + A list containing the results from all jobs. + """ + output = [] + start_time = time.time() + task = "Sequential Processing" + if jobs: + task = jobs[0].get("func", lambda: None).__name__ + + for i, job in enumerate(jobs, 1): + output_ = expand_call(job) + output.append(output_) + report_progress(i, len(jobs), start_time, task) + + return output + + +def linear_partitions(num_atoms: int, num_threads: int) -> np.ndarray: + """ + Generate linear partitions (indices) for parallel computation. + + Splits `num_atoms` into `num_threads` (or fewer) roughly equal parts. + + Parameters + ---------- + num_atoms : int + The total number of items to split. + num_threads : int + The desired number of partitions (threads). + + Returns + ------- + np.ndarray + An array of partition boundary indices. + """ + n_parts = min(num_threads, num_atoms) + if n_parts == 0: + return np.array([0]) + + partitions = np.linspace(0, num_atoms, n_parts + 1) + partitions = np.ceil(partitions).astype(int) + return partitions + + +def nested_partitions( + num_atoms: int, num_threads: int, upper_triangle: bool = False +) -> np.ndarray: + r""" + Generate nested partitions, useful for tasks with nested loops. + + The formula is designed to create partitions of varying sizes, + which can be more efficient for tasks where work is not + distributed linearly (e.g., matrix operations). + + .. math:: + p_i = \frac{-1 + \sqrt{1 + 4(p_{i-1}^2 + p_{i-1} + N(N+1)/K)}}{2} + + Parameters + ---------- + num_atoms : int + Number of atoms (N). + num_threads : int + Number of threads (K). + upper_triangle : bool, default=False + If True, sort partitions to be heaviest first. + + Returns + ------- + np.ndarray + An array of partition boundary indices. + """ + partitions = [0] + n_threads_ = min(num_threads, num_atoms) + + if n_threads_ == 0: + return np.array([0]) + + for _ in range(n_threads_): + last_part = partitions[-1] + part_size = 1 + 4 * ( + last_part**2 + last_part + num_atoms * (num_atoms + 1.0) / n_threads_ + ) + part_val = (-1 + part_size**0.5) / 2.0 + partitions.append(part_val) + + partitions = np.round(partitions).astype(int) + + if upper_triangle: # The first rows are the heaviest + partitions = np.cumsum(np.diff(partitions)[::-1]) + partitions = np.append(np.array([0]), partitions) + + return partitions + + +def mp_pandas_obj( + func: Callable[..., pd.Series], + pandas_object: tuple[str, pd.Index], + num_threads: int = -1, + mp_batches: int = 1, + linear_partition: bool = True, + **kwargs: Any, +) -> Union[pd.DataFrame, pd.Series, list[Any]]: + """ + Parallelize a function call on a pandas object (DataFrame/Series). + + This function splits the `pandas_object` index into partitions and + calls `func` on each partition in parallel. + + Parameters + ---------- + func : Callable + The function to be parallelized. It must accept a pandas + object as its first argument. + pandas_object : Tuple[str, pd.Index] + A tuple where: + - [0] (str): The name of the argument in `func` that + receives the partition (e.g., 'molecule'). + - [1] (pd.Index): The index to be partitioned. + num_threads : int, default=-1 + Number of parallel processes. + -1 means use all available CPUs. + 1 means run sequentially (for debugging). + mp_batches : int, default=1 + Number of batches to split the jobs into. + linear_partition : bool, default=True + If True, use `linear_partitions`. + If False, use `nested_partitions`. + **kwargs : Any + Other keyword arguments to be passed to `func`. + + Returns + ------- + Union[pd.DataFrame, pd.Series, List[Any]] + The concatenated results from all parallel calls. + Returns a List if the output is not a pandas object. + """ + # <-- Resolve num_threads here to correctly handle mp_batches + if num_threads == -1: + num_threads = mp.cpu_count() + + total_parts = num_threads * mp_batches + + if linear_partition: + parts = linear_partitions(len(pandas_object[1]), total_parts) + else: + parts = nested_partitions(len(pandas_object[1]), total_parts) + + jobs = [] + for i in range(1, len(parts)): + job = { + pandas_object[0]: pandas_object[1][parts[i - 1] : parts[i]], + "func": func, + } + job.update(kwargs) + jobs.append(job) + + # <-- Pass num_threads to process_jobs, which now handles 1 correctly + out = process_jobs(jobs, num_threads=num_threads) + + if not out: + return pd.DataFrame() # Return empty DataFrame if no results + + if isinstance(out[0], pd.DataFrame): + result_df = pd.concat(out) + elif isinstance(out[0], pd.Series): + result_df = pd.concat(out) + else: + return out # Return list of other objects + + return result_df.sort_index() diff --git a/src/RiskLabAI/optimization/__init__.py b/src/RiskLabAI/optimization/__init__.py new file mode 100644 index 0000000..eab6496 --- /dev/null +++ b/src/RiskLabAI/optimization/__init__.py @@ -0,0 +1,48 @@ +""" +RiskLabAI Portfolio Optimization Module + +Implements advanced portfolio optimization techniques, including: +- Hierarchical Risk Parity (HRP) +- Nested Clustered Optimisation (NCO) +- PCA-Based Hedging +- Custom Hyper-Parameter Tuning +""" + +from .hedging import ( + pca_weights, +) +from .hrp import ( + cluster_variance, + hrp, + quasi_diagonal, + recursive_bisection, +) +from .hyper_parameter_tuning import ( + MyPipeline, # deprecated alias (scheduled for removal in 4.0.0) + SampleWeightedPipeline, + clf_hyper_fit, +) +from .nco import ( + # cluster_kmeans_base is imported into nco.py, not defined there. + # It should be imported from RiskLabAI.cluster.clustering directly + # in any file that needs it, not from here. + get_optimal_portfolio_weights, + get_optimal_portfolio_weights_nco, +) + +__all__ = [ + # hrp.py + "cluster_variance", + "quasi_diagonal", + "recursive_bisection", + "hrp", + # nco.py + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco", + # hedging.py + "pca_weights", + # hyper_parameter_tuning.py + "SampleWeightedPipeline", + "MyPipeline", # deprecated alias (scheduled for removal in 4.0.0) + "clf_hyper_fit", +] diff --git a/src/RiskLabAI/optimization/hedging.py b/src/RiskLabAI/optimization/hedging.py new file mode 100644 index 0000000..b4a4986 --- /dev/null +++ b/src/RiskLabAI/optimization/hedging.py @@ -0,0 +1,69 @@ +""" +Implements PCA-based portfolio hedging techniques. +""" + +from typing import Optional + +import numpy as np + + +def pca_weights( + cov: np.ndarray, + risk_distribution: Optional[np.ndarray] = None, + risk_target: float = 1.0, +) -> np.ndarray: + r""" + Calculate hedging weights using PCA. + + This function uses PCA to determine portfolio weights that + match a target risk profile. + + Reference: + De Prado, M. (2018) Advances in financial machine learning. + Methodology 36. + + .. math:: + w = V \cdot \sqrt{\frac{\rho T}{\lambda}} + + Where: + - \(w\) are the weights. + - \(V\) is the matrix of eigenvectors. + - \(\rho\) is the risk distribution. + - \(T\) is the risk target. + - \(\lambda\) are the eigenvalues. + + Parameters + ---------- + cov : np.ndarray + Covariance matrix. + risk_distribution : np.ndarray, optional + Target risk distribution across principal components. + If None, all risk is allocated to the component with the + smallest eigenvalue (minimum variance portfolio). + risk_target : float, default=1.0 + The total risk target (T). + + Returns + ------- + np.ndarray + The calculated portfolio weights. + """ + # Calculate eigenvalues and eigenvectors + eigen_values, eigen_vectors = np.linalg.eigh(cov) + + # Sort in descending order + indices = eigen_values.argsort()[::-1] + eigen_values, eigen_vectors = eigen_values[indices], eigen_vectors[:, indices] + + # If no risk distribution, target minimum variance + if risk_distribution is None: + risk_distribution = np.zeros(cov.shape[0]) + risk_distribution[-1] = 1.0 # All risk on last component + + # Compute loads (allocation in the orthogonal basis) + loads = risk_target * (risk_distribution / eigen_values) ** 0.5 + + # Calculate weights in the original basis + weights = np.dot(eigen_vectors, np.reshape(loads, (-1, 1))) + + return weights.flatten() diff --git a/src/RiskLabAI/optimization/hrp.py b/src/RiskLabAI/optimization/hrp.py new file mode 100644 index 0000000..6e1b461 --- /dev/null +++ b/src/RiskLabAI/optimization/hrp.py @@ -0,0 +1,215 @@ +""" +Implements the Hierarchical Risk Parity (HRP) portfolio +optimization algorithm. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +import numpy as np +import pandas as pd +import scipy.cluster.hierarchy as sch +import scipy.spatial.distance as scd + + +def inverse_variance_weights(covariance_matrix: pd.DataFrame) -> np.ndarray: + """ + Compute the inverse-variance portfolio weights. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Covariance matrix of asset returns. + + Returns + ------- + np.ndarray + Array of portfolio weights. + """ + inv_var_weights = 1.0 / np.diag(covariance_matrix.values) + inv_var_weights /= inv_var_weights.sum() + return inv_var_weights + + +def cluster_variance( + covariance_matrix: pd.DataFrame, clustered_items: list[str] +) -> float: + """ + Compute the variance of a cluster using inverse-variance weighting. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Full covariance matrix of asset returns. + clustered_items : List[str] + List of asset names (index/columns) in the cluster. + + Returns + ------- + float + Variance of the cluster. + """ + cov_slice = covariance_matrix.loc[clustered_items, clustered_items] + weights = inverse_variance_weights(cov_slice).reshape(-1, 1) + + # V_cluster = w' * C * w + cluster_var = np.dot(np.dot(weights.T, cov_slice), weights)[0, 0] + return cluster_var + + +def quasi_diagonal(linkage_matrix: np.ndarray) -> list[int]: + """ + Return a sorted list of original item indices for a quasi-diagonal matrix. + + Parameters + ---------- + linkage_matrix : np.ndarray + The linkage matrix from `scipy.cluster.hierarchy.linkage`. + + Returns + ------- + List[int] + Sorted list of original item indices. + """ + link = linkage_matrix.astype(int) + num_items = link[-1, 3] # Total number of original items + + # Get the top-level clusters + items_to_process = [link[-1, 0], link[-1, 1]] + sorted_items = [] + + while len(items_to_process) > 0: + item = items_to_process.pop(0) # Process items recursively (depth-first) + + if item >= num_items: + # This is a cluster, add its children to the processing list + cluster_id = item - num_items + # Add children in their linkage order + items_to_process.insert(0, link[cluster_id, 1]) # Right child + items_to_process.insert(0, link[cluster_id, 0]) # Left child + else: + # This is an original item + sorted_items.append(item) + + return sorted_items + + +def recursive_bisection( + covariance_matrix: pd.DataFrame, sorted_items: list[str] +) -> pd.Series: + """ + Compute the Hierarchical Risk Parity (HRP) weights + using recursive bisection. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Covariance matrix of asset returns. + sorted_items : List[str] + Sorted list of asset names from `quasi_diagonal`. + + Returns + ------- + pd.Series + DataFrame of asset weights. + """ + weights = pd.Series(1.0, index=sorted_items) + clustered_items = [sorted_items] + + while len(clustered_items) > 0: + # Bisection + clustered_items = [ + i[j:k] + for i in clustered_items + for j, k in ((0, len(i) // 2), (len(i) // 2, len(i))) + if len(i) > 1 + ] + + # Process pairs of clusters + for i in range(0, len(clustered_items), 2): + cluster_0 = clustered_items[i] + cluster_1 = clustered_items[i + 1] + + # 1. Calculate variance for each cluster + variance_0 = cluster_variance(covariance_matrix, cluster_0) + variance_1 = cluster_variance(covariance_matrix, cluster_1) + + # 2. Calculate allocation factor (alpha) + if variance_0 + variance_1 == 0: + alpha = 0.5 # Default to equal weight if both variances are zero + else: + alpha = 1 - variance_0 / (variance_0 + variance_1) + + # 3. Apply weights + weights[cluster_0] *= alpha + weights[cluster_1] *= 1 - alpha + + return weights + + +def distance_corr(corr_matrix: np.ndarray) -> np.ndarray: + """ + Compute the distance matrix based on correlation. + d = sqrt(0.5 * (1 - p)) + + Parameters + ---------- + corr_matrix : np.ndarray + Correlation matrix. + + Returns + ------- + np.ndarray + Distance matrix. + """ + distance_matrix = ((1 - corr_matrix) / 2.0) ** 0.5 + return distance_matrix + + +def hrp(cov: pd.DataFrame, corr: pd.DataFrame) -> pd.Series: + """ + Main HRP algorithm. + + Constructs a hierarchical portfolio by: + 1. Calculating correlation-based distance. + 2. Hierarchical clustering (single linkage). + 3. Quasi-diagonalization of the linkage matrix. + 4. Recursive bisection to determine weights. + + Parameters + ---------- + cov : pd.DataFrame + Covariance matrix. + corr : pd.DataFrame + Correlation matrix. + + Returns + ------- + pd.Series + The HRP portfolio weights, sorted by original index. + """ + corr_df, cov_df = corr, cov + + # 1. Calculate distance + distance = distance_corr(corr_df.values) + + # Enforce exact symmetry: a correlation matrix produced by ``cov_to_corr`` or + # by denoising can be asymmetric at the floating-point level, which makes + # ``squareform`` reject it ("Distance matrix must be symmetric"). Averaging + # with the transpose removes that asymmetry without changing the clustering. + distance = (distance + distance.T) / 2.0 + np.fill_diagonal(distance, 0.0) + + dist_condensed = scd.squareform(distance, force="tovector") + + # 2. Cluster + link = sch.linkage(dist_condensed, "single") + + # 3. Quasi-diagonalize + sorted_items_idx = quasi_diagonal(link) + sorted_items_names = corr_df.index[sorted_items_idx].tolist() + + # 4. Recursive bisection + hrp_portfolio = recursive_bisection(cov_df, sorted_items_names) + + return hrp_portfolio.sort_index() diff --git a/src/RiskLabAI/optimization/hyper_parameter_tuning.py b/src/RiskLabAI/optimization/hyper_parameter_tuning.py new file mode 100644 index 0000000..ee87c64 --- /dev/null +++ b/src/RiskLabAI/optimization/hyper_parameter_tuning.py @@ -0,0 +1,197 @@ +""" +Hyperparameter tuning module that integrates with scikit-learn +and the custom PurgedKFold cross-validators. +""" + +import warnings +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaggingClassifier +from sklearn.model_selection import GridSearchCV, RandomizedSearchCV +from sklearn.pipeline import Pipeline + +# Cross-validators are created from the core registry (2.0.0 single source). +from RiskLabAI.core import CROSS_VALIDATORS + + +class SampleWeightedPipeline(Pipeline): + """ + Custom pipeline class to correctly pass `sample_weight` to the + final estimator's `fit` method. + """ + + def fit( + self, + X: pd.DataFrame, + y: pd.Series, + sample_weight: Optional[np.ndarray] = None, + **fit_params, + ) -> "SampleWeightedPipeline": + """ + Fit the pipeline, passing `sample_weight` to the final step. + + Parameters + ---------- + X : pd.DataFrame + Feature data. + y : pd.Series + Labels. + sample_weight : np.ndarray, optional + Sample weights. + **fit_params : Any + Additional fit parameters. + + Returns + ------- + SampleWeightedPipeline + The fitted pipeline. + """ + if sample_weight is not None: + # Add sample_weight to fit_params for the *last step* + step_name = self.steps[-1][0] + fit_params[f"{step_name}__sample_weight"] = sample_weight + + return super().fit(X, y, **fit_params) + + +def clf_hyper_fit( + feature_data: pd.DataFrame, + label: pd.Series, + times: pd.Series, + pipe_clf: Pipeline, + param_grid: dict[str, Any], + validator_type: str = "purgedkfold", + validator_params: Optional[dict[str, Any]] = None, + bagging: Optional[list[Union[int, float]]] = None, + rnd_search_iter: int = 0, + n_jobs: int = -1, + **fit_params, +) -> Union[GridSearchCV, RandomizedSearchCV, Pipeline]: + """ + Perform hyperparameter tuning using a specified cross-validator. + + Parameters + ---------- + feature_data : pd.DataFrame + Feature data (X). + label : pd.Series + Labels (y). + times : pd.Series + Series of event start and end times for purging. + pipe_clf : Pipeline + The scikit-learn pipeline (or `SampleWeightedPipeline`) to tune. + param_grid : dict + Parameter grid for the search. + validator_type : str, default='purgedkfold' + The key for the cross-validator to use + (e.g., 'purgedkfold', 'combinatorialpurged'). + validator_params : dict, optional + Parameters for the cross-validator (e.g., `n_splits`, `embargo`). + bagging : List, optional + If provided, wraps the best estimator in a `BaggingClassifier`. + Format: [n_estimators, max_samples, max_features]. + Default [0, -1, 1.0] from original code seems incorrect, + so changed to None. + rnd_search_iter : int, default=0 + If 0, use `GridSearchCV`. + If > 0, use `RandomizedSearchCV` with this many iterations. + n_jobs : int, default=-1 + Number of parallel jobs for the search. + **fit_params : Any + Additional fit parameters (e.g., `sample_weight`). + + Returns + ------- + Union[GridSearchCV, RandomizedSearchCV, Pipeline] + The fitted grid search object, or a fitted Bagging pipeline. + """ + if bagging is None: + bagging = [0, 0.0, 1.0] # Default to no bagging + + if set(label.unique()) == {0, 1}: + scoring = "f1" # F1-score for meta-labeling + else: + scoring = "neg_log_loss" + + if validator_params is None: + validator_params = { + "times": times, + "n_splits": 5, + "embargo": 0.01, + } + else: + # Ensure 'times' is passed if not already present + if "times" not in validator_params: + validator_params["times"] = times + + # 1. Set up the custom cross-validator + inner_cv = CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **validator_params + ) + + # 2. Set up the hyperparameter search + if rnd_search_iter == 0: + gs = GridSearchCV( + estimator=pipe_clf, + param_grid=param_grid, + scoring=scoring, + cv=inner_cv, + n_jobs=n_jobs, + ) + else: + gs = RandomizedSearchCV( + estimator=pipe_clf, + param_distributions=param_grid, + scoring=scoring, + cv=inner_cv, + n_jobs=n_jobs, + n_iter=rnd_search_iter, + ) + + # 3. Fit the search + gs = gs.fit(feature_data, label, **fit_params) + + # 4. (Optional) Fit bagging classifier on the best model + if bagging[0] > 0: + best_estimator = gs.best_estimator_ + + # Create a new pipeline with the best estimator's steps + bag_pipe = SampleWeightedPipeline(best_estimator.steps) + + bag_clf = BaggingClassifier( + estimator=bag_pipe, + n_estimators=int(bagging[0]), + max_samples=float(bagging[1]) if bagging[1] > 0 else 1.0, + max_features=float(bagging[2]), + n_jobs=n_jobs, + ) + + # Fit the bagging classifier + bag_clf = bag_clf.fit(feature_data, label, **fit_params) + + # Return as a pipeline + return Pipeline([("bag", bag_clf)]) + + # 5. Return the best estimator found + return gs.best_estimator_ + + +class MyPipeline(SampleWeightedPipeline): + """ + Deprecated alias for :class:`SampleWeightedPipeline` (scheduled for removal in 4.0.0). + + Implemented as an explicit subclass rather than via ``deprecated_class`` so + that the scikit-learn ``__init__`` signature is preserved (estimators may + not use ``*args``/``**kwargs`` in their constructor). + """ + + def __init__(self, steps, *, memory=None, verbose=False): + warnings.warn( + "MyPipeline is deprecated and will be removed in 4.0.0; " + "use SampleWeightedPipeline instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(steps, memory=memory, verbose=verbose) diff --git a/src/RiskLabAI/optimization/nco.py b/src/RiskLabAI/optimization/nco.py new file mode 100644 index 0000000..267807d --- /dev/null +++ b/src/RiskLabAI/optimization/nco.py @@ -0,0 +1,130 @@ +""" +Implements the Nested Clustered Optimization (NCO) algorithm. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +from typing import Optional + +import numpy as np +import pandas as pd + +# Import canonical implementations instead of duplicating +from RiskLabAI.cluster.clustering import cluster_k_means_base, covariance_to_correlation + + +def get_optimal_portfolio_weights( + covariance: np.ndarray, mu: Optional[np.ndarray] = None +) -> np.ndarray: + """ + Compute the optimal portfolio weights (Markowitz). + + If `mu` is not provided, computes the Global Minimum Variance (GMV) portfolio. + If `mu` is provided, computes the Mean-Variance Optimization (MVO) portfolio. + + Parameters + ---------- + covariance : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. + + Returns + ------- + np.ndarray + Optimal portfolio weights. + """ + inverse_covariance = np.linalg.inv(covariance) + ones = np.ones(shape=(inverse_covariance.shape[0], 1)) + + if mu is None: + mu = ones # For GMV portfolio + + weights = np.dot(inverse_covariance, mu) + weights /= np.dot(ones.T, weights) # Normalize weights to sum to 1 + return weights + + +def get_optimal_portfolio_weights_nco( + covariance: np.ndarray, + mu: Optional[np.ndarray] = None, + number_clusters: Optional[int] = None, +) -> np.ndarray: + """ + Compute optimal portfolio weights using the NCO algorithm. + + NCO works in three steps: + 1. Cluster assets based on their correlation matrix. + 2. Compute optimal (e.g., GMV) weights *within* each cluster. + 3. Compute optimal weights *between* the clusters (treating each + cluster as a single asset). + 4. Combine the inter-cluster and intra-cluster weights. + + Parameters + ---------- + covariance : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. If None, uses GMV. + number_clusters : int, optional + Maximum number of clusters. If None, defaults to N/2. + + Returns + ------- + np.ndarray + The NCO-optimized portfolio weights. + """ + covariance = pd.DataFrame(covariance) + correlation = covariance_to_correlation(covariance.to_numpy()) + correlation = pd.DataFrame( + correlation, index=covariance.index, columns=covariance.columns + ) + + if mu is not None: + mu = pd.Series(mu.flatten(), index=covariance.index) + + if number_clusters is None: + number_clusters = int(correlation.shape[0] / 2) + + # 1. Cluster assets + _, clusters, _ = cluster_k_means_base( + correlation, max_clusters=number_clusters, iterations=10 + ) + + # 2. Compute intra-cluster weights + weights_intra_cluster = pd.DataFrame( + 0.0, index=covariance.index, columns=clusters.keys() + ) + for i, cluster_assets in clusters.items(): + cov_intra = covariance.loc[cluster_assets, cluster_assets].values + + mu_intra = None + if mu is not None: + mu_intra = mu.loc[cluster_assets].values.reshape(-1, 1) + + weights_intra_cluster.loc[cluster_assets, i] = get_optimal_portfolio_weights( + cov_intra, mu_intra + ).flatten() + + # 3. Compute inter-cluster weights + # Reduce covariance matrix using intra-cluster weights + covariance_inter_cluster = weights_intra_cluster.T.dot( + covariance.dot(weights_intra_cluster) + ) + + mu_inter_cluster = None + if mu is not None: + mu_inter_cluster = weights_intra_cluster.T.dot(mu.values) + mu_inter_cluster = mu_inter_cluster.values.reshape(-1, 1) + + weights_inter_cluster = pd.Series( + get_optimal_portfolio_weights( + covariance_inter_cluster.values, mu_inter_cluster + ).flatten(), + index=covariance_inter_cluster.index, + ) + + # 4. Combine weights + weights_nco = weights_intra_cluster.mul(weights_inter_cluster, axis=1).sum(axis=1) + return weights_nco.values.reshape(-1, 1) diff --git a/src/RiskLabAI/pde/__init__.py b/src/RiskLabAI/pde/__init__.py new file mode 100644 index 0000000..2f5feb6 --- /dev/null +++ b/src/RiskLabAI/pde/__init__.py @@ -0,0 +1,75 @@ +""" +RiskLabAI PDE Solver Module + +Implements a Deep BSDE (Backward Stochastic Differential Equation) +solver for various financial PDEs. + +This sub-package requires PyTorch, which is an optional dependency of +RiskLabAI. The base install stays torch-free: ``import RiskLabAI`` never pulls +in this sub-package (it is loaded lazily). Importing ``RiskLabAI.pde`` without +torch installed raises a clear, actionable error instead of a bare +``ModuleNotFoundError``. +""" + +try: + import torch as _torch # noqa: F401 (presence check only) +except ImportError as _exc: # pragma: no cover - exercised only without torch + raise ImportError( + "RiskLabAI.pde requires PyTorch, which is an optional dependency. " + "Install it with: pip install 'RiskLabAI[pde]' (or: pip install torch)." + ) from _exc + +from .equation import ( + HJBLQ, + BlackScholesBarenblatt, + Equation, + PricingDefaultRisk, + PricingDiffRate, +) +from .model import ( + ISAB, + MAB, + PMA, + SAB, + DeepBSDE, + DeepTimeSetTransformer, + FBSNNNetwork, + Net1, + TimeDependentNetwork, + TimeDependentNetworkMonteCarlo, + TimeNet, + TimeNetForSet, +) +from .solver import ( + FBSDESolver, + FBSNNolver, # deprecated alias (scheduled for removal in 4.0.0) + FBSNNSolver, + initialize_weights, +) + +__all__ = [ + # Equations + "Equation", + "PricingDefaultRisk", + "HJBLQ", + "BlackScholesBarenblatt", + "PricingDiffRate", + # Models + "TimeNet", + "Net1", + "MAB", + "SAB", + "ISAB", + "PMA", + "TimeNetForSet", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "DeepBSDE", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + # Solvers + "initialize_weights", + "FBSDESolver", + "FBSNNSolver", + "FBSNNolver", # deprecated alias (scheduled for removal in 4.0.0) +] diff --git a/src/RiskLabAI/pde/equation.py b/src/RiskLabAI/pde/equation.py new file mode 100644 index 0000000..5426077 --- /dev/null +++ b/src/RiskLabAI/pde/equation.py @@ -0,0 +1,316 @@ +""" +Base class and concrete implementations for Partial Differential Equations +(PDEs) to be solved by the Deep BSDE solver. + +Based on: https://github.com/frankhan91/DeepBSDE +""" + +from typing import Optional, Union + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + + +class Equation: + """ + Base class for defining PDE-related functions. + """ + + def __init__(self, eqn_config: dict): + """ + Initialize the PDE configuration. + + Parameters + ---------- + eqn_config : dict + Dictionary containing PDE parameters: + - 'dim': Dimensionality of the problem (int) + - 'total_time': Total time horizon (float) + - 'num_time_interval': Number of time steps (int) + """ + self.dim: int = eqn_config["dim"] + self.total_time: float = eqn_config["total_time"] + self.num_time_interval: int = eqn_config["num_time_interval"] + self.delta_t: float = self.total_time / self.num_time_interval + self.sqrt_delta_t: float = np.sqrt(self.delta_t) + self.y_init: Optional[float] = None + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + """ + Sample the forward SDE (e.g., the underlying asset path). + + Parameters + ---------- + num_sample : int + Number of sample paths to generate. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - dw_sample: Wiener process increments [num_sample, dim, num_time_interval] + - x_sample: Asset paths [num_sample, dim, num_time_interval+1] + """ + raise NotImplementedError + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + """ + The driver function 'f' of the BSDE, or the generator of the PDE. + Represents the non-linear interest rate and dividend term. + + Parameters + ---------- + t : float + Current time. + x : Tensor + Space coordinates [batch_size, dim]. + y : Tensor + Function value (price) [batch_size, 1]. + z : Tensor + Gradient (delta) [batch_size, dim]. + + Returns + ------- + Tensor + Value of the driver `f` [batch_size, 1]. + """ + raise NotImplementedError + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + """ + The Hamiltonian term H(z) in the PDE, or the volatility-related + term in the BSDE driver. + + Parameters + ---------- + t : float + Current time. + x : Tensor + Space coordinates [batch_size, dim]. + y : Tensor + Function value (price) [batch_size, 1]. + z : Tensor + Gradient (delta) [batch_size, dim]. + + Returns + ------- + Tensor + Value of the Hamiltonian `H(z)` [batch_size, 1]. + """ + raise NotImplementedError + + def terminal(self, t: float, x: Tensor) -> Tensor: + """ + Terminal condition (payoff) of the PDE. + + Parameters + ---------- + t : float + Current time (should be total_time). + x : Tensor + Space coordinates [batch_size, dim]. + + Returns + ------- + Tensor + Terminal payoff value [batch_size, 1]. + """ + raise NotImplementedError + + def sigma_matrix(self, x: Tensor) -> Tensor: + """Helper to get the volatility matrix sigma(x).""" + raise NotImplementedError + + def terminal_for_sample(self, x: Tensor) -> Tensor: + """Terminal condition for a multi-sample path.""" + raise NotImplementedError + + +class PricingDefaultRisk(Equation): + """ + PDE for pricing with default risk. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * 100.0 + self.sigma = 0.2 + self.rate = 0.02 # R + self.delta = 2.0 / 3 + self.gammah = 0.2 + self.gammal = 0.02 + self.mu_bar = 0.02 + self.K = 100 + self.vh = 50.0 + self.vl = 70.0 + self.slope = (self.gammah - self.gammal) / (self.vh - self.vl) + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = (1 + self.mu_bar * self.delta_t) * x_sample[ + :, :, i + ] + (self.sigma_matrix(x_sample[:, :, i]) * dw_sample[:, :, i]) + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + piecewise_linear = ( + nn.ReLU()(nn.ReLU()(y - self.vh) * self.slope + self.gammah - self.gammal) + + self.gammal + ) + return (1 - self.delta) * piecewise_linear + self.rate + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.zeros((x.size()[0], 1), device=x.device) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal(self, t: float, x: Tensor) -> Tensor: + # torch.min(x, 1) returns (values, indices) + return nn.ReLU()(torch.min(x, 1, keepdim=True)[0]) + + def terminal_for_sample(self, x: Tensor) -> Tensor: + return nn.ReLU()(torch.min(x, 2, keepdim=True)[0]) + + +class HJBLQ(Equation): + """ + Hamilton-Jacobi-Bellman (HJB) equation with Linear-Quadratic (LQ) cost. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.zeros(self.dim) + self.sigma = np.sqrt(2.0) + self.lambd = 1.0 + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = x_sample[:, :, i] + self.sigma * dw_sample[:, :, i] + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.zeros((x.size()[0], 1), device=x.device) + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.sum(torch.square(z), dim=1, keepdim=True) / (self.sigma**2) + + def terminal(self, t: float, x: Tensor) -> Tensor: + return torch.log(0.5 * (1 + torch.norm(x, p=2, dim=1, keepdim=True) ** 2)) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> float: + return self.sigma + + def terminal_for_sample(self, x: Tensor) -> Tensor: + # Used by Monte-Carlo solver, needs to match HJB terminal + return torch.log(0.5 * (1 + torch.norm(x, p=2, dim=2, keepdim=True) ** 2)) + + +class BlackScholesBarenblatt(Equation): + """ + Black-Scholes-Barenblatt equation. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * np.array( + [1.0 / (1.0 + i % 2) for i in range(self.dim)] + ) + self.sigma = 0.4 + self.rate = 0.05 # interest rate R + self.mu_bar = 0.0 + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=(num_sample, self.dim, self.num_time_interval)) + * self.sqrt_delta_t + ) + x_sample = np.zeros((num_sample, self.dim, self.num_time_interval + 1)) + x_sample[:, :, 0] = np.ones((num_sample, self.dim)) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = (1 + self.mu_bar * self.delta_t) * x_sample[ + :, :, i + ] + (self.sigma_matrix(x_sample[:, :, i]) * dw_sample[:, :, i]) + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.ones((x.size()[0], 1), device=x.device) * self.rate + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return -1 * torch.sum(z, dim=1, keepdim=True) * self.rate / self.sigma + + def terminal(self, t: float, x: Tensor) -> Tensor: + return torch.sum(x**2, dim=1, keepdim=True) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal_for_sample(self, x: Tensor) -> Tensor: + return torch.sum(x**2, dim=2, keepdim=True) + + +class PricingDiffRate(Equation): + """ + Nonlinear Black-Scholes with different interest rates for + borrowing and lending. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * 100 + self.sigma = 0.2 + self.mu_bar = 0.06 + self.rl = 0.04 + self.rb = 0.06 + self.alpha = 1.0 / self.dim + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + factor = np.exp((self.mu_bar - (self.sigma**2) / 2) * self.delta_t) + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = ( + factor * np.exp(self.sigma * dw_sample[:, :, i]) + ) * x_sample[:, :, i] + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + temp = torch.sum(z, dim=1, keepdim=True) / self.sigma - y + return torch.where(temp > 0, self.rb, self.rl) + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + temp = torch.sum(z, dim=1, keepdim=True) / self.sigma - y + sum_z = torch.sum(z, dim=1, keepdim=True) + return torch.where( + temp > 0, + (self.mu_bar - self.rb) * sum_z / self.sigma, + (self.mu_bar - self.rl) * sum_z / self.sigma, + ) + + def terminal(self, t: float, x: Tensor) -> Tensor: + temp = torch.max(x, 1, keepdim=True)[0] + return nn.ReLU()(temp - 120) - 2 * nn.ReLU()(temp - 150) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal_for_sample(self, x: Tensor) -> Tensor: + temp = torch.max(x, 2, keepdim=True)[0] + return torch.maximum(temp - 120, torch.tensor(0.0, device=x.device)) diff --git a/src/RiskLabAI/pde/model.py b/src/RiskLabAI/pde/model.py new file mode 100644 index 0000000..beef487 --- /dev/null +++ b/src/RiskLabAI/pde/model.py @@ -0,0 +1,372 @@ +""" +Defines the neural network architectures used by the PDE solvers. + +Includes: +- Standard feed-forward networks (Net1, TimeNet) +- Set-Transformer components (MAB, SAB, ISAB, PMA) +- Deep BSDE models (DeepBSDE, FBSNNNetwork, TimeDependentNetwork) + +Set-Transformer code based on: https://github.com/juho-lee/set_transformer +""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import BatchNorm1d, Linear, Module, ReLU, Tanh + +# --- Set Transformer Components --- + + +class MAB(Module): + """Multi-Head Attention Block (MAB).""" + + def __init__( + self, dim_q: int, dim_k: int, dim_v: int, num_heads: int, ln: bool = False + ): + super().__init__() + self.dim_v = dim_v + self.num_heads = num_heads + self.fc_q = Linear(dim_q, dim_v) + self.fc_k = Linear(dim_k, dim_v) + self.fc_v = Linear(dim_k, dim_v) + self.ln0 = nn.LayerNorm(dim_v) if ln else None + self.ln1 = nn.LayerNorm(dim_v) if ln else None + self.fc_o = Linear(dim_v, dim_v) + + def forward(self, q: Tensor, k: Tensor) -> Tensor: + q_fc = self.fc_q(q) + k_fc, v_fc = self.fc_k(k), self.fc_v(k) + + dim_split = self.dim_v // self.num_heads + q_ = torch.cat(q_fc.split(dim_split, 2), 0) + k_ = torch.cat(k_fc.split(dim_split, 2), 0) + v_ = torch.cat(v_fc.split(dim_split, 2), 0) + + attention = torch.softmax(q_.bmm(k_.transpose(1, 2)) / math.sqrt(self.dim_v), 2) + out = torch.cat((q_ + attention.bmm(v_)).split(q.size(0), 0), 2) + + out = self.ln0(out) if self.ln0 is not None else out + out = out + F.relu(self.fc_o(out)) + out = self.ln1(out) if self.ln1 is not None else out + return out + + +class SAB(Module): + """Self-Attention Block (SAB).""" + + def __init__(self, dim_in: int, dim_out: int, num_heads: int, ln: bool = False): + super().__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + return self.mab(x, x) + + +class ISAB(Module): + """Induced Self-Attention Block (ISAB).""" + + def __init__( + self, dim_in: int, dim_out: int, num_heads: int, num_inds: int, ln: bool = False + ): + super().__init__() + self.i = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.i) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + h = self.mab0(self.i.repeat(x.size(0), 1, 1), x) + return self.mab1(x, h) + + +class PMA(Module): + """Pooling Multi-Head Attention (PMA).""" + + def __init__(self, dim: int, num_heads: int, num_seeds: int, ln: bool = False): + super().__init__() + self.s = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.s) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + return self.mab(self.s.repeat(x.size(0), 1, 1), x) + + +class TimeNetForSet(Module): + """ + Time-dependent feature transformation for SetTransformer. + + Applies separate linear layers to time (t) and features (x), + then combines them as exp(t) * x. + """ + + def __init__(self, in_features: int = 1, out_features: int = 64): + super().__init__() + self.feature_layer = Linear(in_features, out_features) + + self.time_layer1 = Linear(1, 10) + self.time_layer2 = Linear(10, 10) + self.time_layer3 = Linear(10, 10) + self.time_layer4 = Linear(10, out_features) + + self.relu_stack = nn.Sequential(ReLU(), ReLU(), ReLU()) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + t_feat = self.relu_stack[0](self.time_layer1(t)) + t_feat = self.relu_stack[1](self.time_layer2(t_feat)) + t_feat = self.relu_stack[2](self.time_layer3(t_feat)) + t_feat = self.time_layer4(t_feat) + + x_feat = self.feature_layer(x) + + return torch.exp(t_feat) * x_feat + + +class DeepTimeSetTransformer(Module): + """ + Full Deep Time Set Transformer model. + """ + + def __init__(self, input_dim: int): + super().__init__() + + # Feature extractor layers + self.layer1 = Linear(input_dim, 32) + self.layer2 = Linear(32, 32) + self.layer3 = Linear(32, 32) + self.layer4 = Linear(32, 32) + self.layer5 = Linear(32, 32) + self.activation = ReLU() + + # Pooling layer + self.regressor = PMA(dim=32, num_heads=4, num_seeds=1) + + # Time-dependent output layers + self.time_layer2 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer3 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer4 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer5 = TimeNetForSet(in_features=32, out_features=1) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + x = self.activation(self.layer1(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer2(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer3(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer4(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer5(x)) + + output = self.regressor(x) + output = torch.squeeze(output) + + output = self.activation(self.time_layer2(t, output)) + output = self.activation(self.time_layer3(t, output)) + output = self.activation(self.time_layer4(t, output)) + output = self.time_layer5(t, output) + + return output + + +# --- Standard Networks --- + + +class TimeNet(Module): + """Simple feed-forward network for time features.""" + + def __init__(self, output_dim: int): + super().__init__() + self.layers = nn.ModuleList( + [ + Linear(4, 100), + Linear(100, 150), + Linear(150, 200), + Linear(200, 300), + Linear(300, 200), + Linear(200, output_dim), + ] + ) + self.tanh_stack = nn.ModuleList([Tanh() for _ in range(6)]) + + def forward(self, x: Tensor) -> Tensor: + for i in range(len(self.layers)): + x = self.tanh_stack[i](self.layers[i](x)) + return x + + +class Net1(Module): + """A simple Linear + BatchNorm layer.""" + + def __init__(self, input_dim: int, output_dim: int): + super().__init__() + self.layer = Linear(input_dim, output_dim) + self.bn = BatchNorm1d(output_dim) # Note: bn is not used in forward + + def forward(self, x: Tensor) -> Tensor: + return self.layer(x) + + +# --- BSDE Solver Networks --- + + +class FBSNNNetwork(Module): + """Feed-forward network for the FBSNN solver.""" + + def __init__(self, layer_sizes: list[int]): + super().__init__() + self.n_layer = len(layer_sizes) - 1 + self.layers = nn.ModuleList([]) + + for i in range(self.n_layer): + self.layers.append(Linear(layer_sizes[i], layer_sizes[i + 1])) + + def forward(self, x: Tensor) -> Tensor: + for i in range(self.n_layer): + x = self.layers[i](x) + if i < self.n_layer - 1: + x = torch.sin(x) # Use sin activation + return x + + +class DeepBSDE(Module): + """Network for the Deep BSDE method (one net per time step).""" + + def __init__(self, layer_sizes: list[int]): + super().__init__() + self.n_layer = len(layer_sizes) - 1 + self.layers = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + + for i in range(self.n_layer): + self.layers.append(Linear(layer_sizes[i], layer_sizes[i + 1], bias=False)) + self.batch_layer.append( + BatchNorm1d(layer_sizes[i], eps=1e-06, momentum=0.01) + ) + + self.batch_layer.append(BatchNorm1d(layer_sizes[-1], eps=1e-06, momentum=0.01)) + self.activation = ReLU() + + def forward(self, x: Tensor) -> Tensor: + # Note: Original code doesn't use batch_layer[0] + for i in range(self.n_layer - 1): + x = self.layers[i](x) + # x = self.batch_layer[i+1](x) # Original code commented this out + x = self.activation(x) + + x = self.layers[-1](x) + # x = self.batch_layer[-1](x) # Original code commented this out + return x + + +class TimeDependentNetwork(Module): + """Time-dependent network for BSDE solver.""" + + def __init__(self, indim: int, layersize: list[int], outdim: int): + super().__init__() + self.n_layer = len(layersize) + self.layers = nn.ModuleList([]) + self.time_layer = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + + # Input layer + self.layers.append(Net1(indim, layersize[0])) + self.time_layer.append(TimeNet(indim)) + self.batch_layer.append(BatchNorm1d(indim, eps=1e-06, momentum=0.01)) + self.batch_layer.append(BatchNorm1d(layersize[0], eps=1e-06, momentum=0.01)) + + # Hidden layers + for i in range(len(layersize) - 1): + self.layers.append(Net1(layersize[i], layersize[i + 1])) + self.time_layer.append(TimeNet(layersize[i])) + self.batch_layer.append( + BatchNorm1d(layersize[i + 1], eps=1e-06, momentum=0.01) + ) + + # Output layer + self.time_layer.append(TimeNet(outdim)) + self.linear = Linear(layersize[-1], outdim) + self.batch_layer.append(BatchNorm1d(outdim, eps=1e-06, momentum=0.01)) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + for i in range(self.n_layer): + # (t, t**2, t**3, exp(t)) + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[i](time_features) + + x = self.batch_layer[i](x) + x = x * (1 + time_weight) + x = self.layers[i](x) + x = torch.sin(x) + + x = self.linear(x) + # x = self.batch_layer[-1](x) # Original code commented this out + return x + + +class TimeDependentNetworkMonteCarlo(Module): + """Time-dependent network for BSDE solver with Monte Carlo gradient.""" + + def __init__(self, indim: int, layersize: list[int], outdim: int, sigma: float): + super().__init__() + self.n_layer = len(layersize) + self.layers = nn.ModuleList([]) + self.time_layer = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + self.sigma = sigma + + # Input layer + self.layers.append(Net1(indim, layersize[0])) + self.time_layer.append(TimeNet(indim)) + self.batch_layer.append(BatchNorm1d(indim)) + self.batch_layer.append(BatchNorm1d(layersize[0])) + + # Hidden layers + for i in range(len(layersize) - 1): + self.layers.append(Net1(layersize[i], layersize[i + 1])) + self.time_layer.append(TimeNet(layersize[i])) + self.batch_layer.append(BatchNorm1d(layersize[i + 1])) + + # Output layer + self.time_layer.append(TimeNet(outdim)) + self.linear = Linear(layersize[-1], outdim) + self.activation = ReLU() + + def forward(self, t: Tensor, x: Tensor, y_mc: Tensor) -> Tensor: + """ + Forward pass. + + Parameters + ---------- + t : Tensor + Time tensor. + x : Tensor + Input features (asset price). + y_mc : Tensor + Monte Carlo estimate of the gradient (Z). + """ + x_prim = x # Store original input + + for i in range(self.n_layer): + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[i](time_features) + + x = x * (1 + time_weight) + x = self.layers[i](x) + x = self.activation(x) + + # Output layer + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[self.n_layer](time_features) + + # Combine network output with MC estimate + # Z = sigma * S * Network(t,S) + (1 - time_weight) * Z_MC + x = self.sigma * x_prim * self.linear(x) + (1 - time_weight) * y_mc + return x diff --git a/src/RiskLabAI/pde/solver.py b/src/RiskLabAI/pde/solver.py new file mode 100644 index 0000000..28003d6 --- /dev/null +++ b/src/RiskLabAI/pde/solver.py @@ -0,0 +1,412 @@ +""" +Implements the main FBSDE (Forward-Backward Stochastic Differential +Equation) solver classes. +""" + +import logging + +import torch +import torch.autograd as autograd +import torch.nn as nn + +from RiskLabAI._deprecation import deprecated_class +from RiskLabAI.pde.equation import Equation +from RiskLabAI.pde.model import * + +logger = logging.getLogger(__name__) + + +def initialize_weights(m: nn.Module) -> None: + """ + Initializes the weights of a Linear layer. + + Parameters + ---------- + m : nn.Module + The module to initialize. + """ + if isinstance(m, nn.Linear): + torch.nn.init.normal_(m.weight, 0, 0.1) + m.bias.data.fill_(0.00) + + +class FBSDESolver: + """ + Solver for FBSDEs using various deep learning methods. + """ + + def __init__( + self, + pde: Equation, + layer_sizes: list[int], + learning_rate: float, + solving_method: str, + device: torch.device, + ): + """ + Initialize the FBSDE Solver. + + Parameters + ---------- + pde : Equation + The PDE to solve (from equation.py). + layer_sizes : List[int] + List of hidden layer sizes for the network. + learning_rate : float + Learning rate for the optimizer. + solving_method : str + Method to use: 'Monte-Carlo', 'Deep-Time-SetTransformer', + 'DTNN', or 'DeepBSDE'. + device : torch.device + The device ('cpu' or 'cuda') to run on. + """ + self.pde = pde + self.layer_size = layer_sizes + self.learning_rate = learning_rate + self.method = solving_method + self.device = device + + # 1. Initialize the correct network model + if solving_method == "Monte-Carlo": + self.solver = TimeDependentNetworkMonteCarlo( + pde.dim, self.layer_size, pde.dim, pde.sigma + ).to(device) + elif solving_method == "Deep-Time-SetTransformer": + self.solver = DeepTimeSetTransformer(1).to(device) # Assumes dim=1 + elif solving_method == "DTNN": + self.solver = TimeDependentNetwork(pde.dim, self.layer_size, pde.dim).to( + device + ) + elif solving_method == "DeepBSDE": + # Create a separate network for each time step + self.solver = nn.ModuleList( + [DeepBSDE(layer_sizes).to(device) for _ in range(pde.num_time_interval)] + ) + else: + raise ValueError(f"Unknown solving_method: {solving_method}") + + # 2. Initialize the optimizer + if solving_method != "DeepBSDE": + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.9, 0.99) + ) + else: + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.9, 0.99) + ) + + def compute_loss( + self, y_path: Tensor, dw_path: Tensor, t: Tensor, init_y: Tensor, init_z: Tensor + ) -> tuple[Tensor, ...]: + """ + Compute the loss for a single batch. + """ + batch_size = y_path.size()[0] + y_terminal = init_y.expand(batch_size, 1) + + coef = torch.ones((batch_size, 1), device=self.device) + dw_coef = torch.zeros((batch_size, 1), device=self.device) + + for z in range(self.pde.num_time_interval): + S0 = y_path[:, :, z] + t0 = t * self.pde.delta_t * z + + if self.method == "DeepBSDE": + if z > 0: + out_z = self.solver[z](S0) + else: + # Use initial gradient Z_0 + out_z = init_z.expand(batch_size, self.pde.dim) + else: + # Other models are time-dependent + out_z = self.solver(t0, S0) # This assumes other models return Z + + samp_dw = dw_path[:, :, z] + + # Get driver components + interest_rate = self.pde.r_u(t0, S0, y_terminal, out_z) + hz = self.pde.h_z(t0, S0, y_terminal, out_z) + + # Update path + y_terminal = ( + y_terminal * (1 + interest_rate * self.pde.delta_t) + + hz * self.pde.delta_t + + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + ) + + # Update coefficients for loss calculation + if z > 0: + dw_coef = ( + dw_coef * (1 + interest_rate * self.pde.delta_t) + + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + + hz * self.pde.delta_t + ) + else: + dw_coef = ( + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + + hz * self.pde.delta_t + ) + + coef = coef * (1 + interest_rate * self.pde.delta_t) + + # 4. Calculate terminal payoff and loss + payoff = self.pde.terminal(t * self.pde.total_time, y_path[:, :, -1]) + + loss = torch.mean(torch.square(payoff - y_terminal)) + + return loss, coef, dw_coef, payoff + + def solve( + self, num_iterations: int, batch_size: int, init_y: float + ) -> tuple[list[float], list[float]]: + """ + Solves the PDE. + + Parameters + ---------- + num_iterations : int + Number of training iterations. + batch_size : int + Batch size for training. + init_y : float + The initial guess for Y_0 (the price at t=0). + + Returns + ------- + Tuple[List[float], List[float]] + - losses: List of losses at each iteration. + - inits: List of Y_0 values at each iteration. + """ + losses = [] + inits = [] + + # Y_0 and Z_0 are trainable parameters for DeepBSDE + y0 = torch.tensor([init_y], device=self.device).requires_grad_(True) + z0 = torch.zeros(1, self.pde.dim, device=self.device).requires_grad_(True) + + if self.method == "DeepBSDE": + init_opt = torch.optim.Adam([y0], lr=0.1, betas=(0.9, 0.99)) + init_grad_opt = torch.optim.Adam([z0], lr=0.001, betas=(0.9, 0.99)) + + # Validation data + dw_val, y_val = self.pde.sample(128) + y_val = torch.tensor(y_val, dtype=torch.float32, device=self.device) + dw_val = torch.tensor(dw_val, dtype=torch.float32, device=self.device) + t_val = torch.ones((128, 1), device=self.device) + + # Set model to train mode + if self.method != "DeepBSDE": + self.solver.train() + else: + for model in self.solver: + model.train() + + for j in range(num_iterations): + logger.info("Iteration %d/%d", j + 1, num_iterations) + + dw_train, y_train = self.pde.sample(batch_size) + y_train = torch.tensor(y_train, dtype=torch.float32, device=self.device) + dw_train = torch.tensor(dw_train, dtype=torch.float32, device=self.device) + t_train = torch.ones((batch_size, 1), device=self.device) + + self.optimizer.zero_grad() + if self.method == "DeepBSDE": + init_opt.zero_grad() + init_grad_opt.zero_grad() + + loss, _, _, _ = self.compute_loss(y_train, dw_train, t_train, y0, z0) + + loss.backward() + + self.optimizer.step() + if self.method == "DeepBSDE": + init_opt.step() + init_grad_opt.step() + + # Validation and logging + with torch.no_grad(): + val_loss, coef, dw_coef, payoff = self.compute_loss( + y_val, dw_val, t_val, y0, z0 + ) + + if self.method != "DeepBSDE": + # For non-DeepBSDE, Y_0 is not a param but computed + # from the loss function's components. + y0_new = torch.mean((payoff - dw_coef) / coef) + inits.append(y0_new.item()) + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0_new.item()) + else: + inits.append(y0.item()) + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0.item()) + + losses.append(val_loss.item()) + + return losses, inits + + +class FBSNNSolver: + """ + Solver for FBSNN (Forward-Backward Stochastic Neural Network). + + This method solves the PDE by learning the solution Y_t directly + at each time step and minimizing the difference between the + predicted Y_{t+1} and the one-step-ahead approximation. + """ + + def __init__( + self, + pde: Equation, + layer_sizes: list[int], + learning_rate: float, + device: torch.device, + ): + """ + Initialize the FBSNN Solver. + + Parameters + ---------- + pde : Equation + The PDE to solve. + layer_sizes : List[int] + Layer sizes for the FBSNNNetwork. + learning_rate : float + Learning rate for the optimizer. + device : torch.device + The device ('cpu' or 'cuda') to run on. + """ + self.pde = pde + self.layer_size = layer_sizes + self.learning_rate = learning_rate + self.device = device + + self.solver = FBSNNNetwork(self.layer_size).to(self.device) + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.99, 0.999) + ) + + def compute_loss( + self, y_path: Tensor, dw_path: Tensor, t: Tensor, init_y: Tensor + ) -> Tensor: + """ + Compute the loss for the FBSNN method. + """ + batch_size = y_path.size()[0] + init_y.expand(batch_size, 1) # Not used, but kept for signature + loss = torch.tensor(0.0, device=self.device) + + for z in range(self.pde.num_time_interval): + S0 = y_path[:, :, z].requires_grad_(True) + S1 = y_path[:, :, z + 1] + + t0 = t * self.pde.delta_t * z + t1 = t * self.pde.delta_t * (z + 1) + + # 1. Get network output Y_t and Y_{t+1} + y_0 = self.solver(torch.cat((t0, S0), dim=1)) + y_1 = self.solver(torch.cat((t1, S1), dim=1)) + + # 2. Calculate the gradient Z_t = dY_t/dS_t * sigma(S_t) + grad_y0 = autograd.grad( + outputs=torch.sum(y_0), inputs=S0, create_graph=True + )[0] + Z = grad_y0 * self.pde.sigma_matrix(S0) + + samp_dw = dw_path[:, :, z] + + # 3. Get driver components + interest_rate = self.pde.r_u(t0, S0, y_0, Z) + hz = self.pde.h_z(t0, S0, y_0, Z) + + # 4. Calculate the one-step approximation Y_hat_{t+1} + y_1_hat = ( + y_0 * (1 + interest_rate * self.pde.delta_t) + + hz * self.pde.delta_t + + torch.sum(Z * samp_dw, dim=1, keepdim=True) + ) + + # 5. Add to total loss + loss += torch.mean(torch.square(y_1_hat - y_1)) + + # 6. Add terminal condition loss + S_T = y_path[:, :, -1] + t_T = t * self.pde.total_time + + payoff = self.pde.terminal(t_T, S_T) + y_T = self.solver(torch.cat((t_T, S_T), dim=1)) + + loss += torch.mean(torch.square(payoff - y_T)) + + return loss + + def solve( + self, + num_iterations: int, + batch_size: int, + init_y: float, # Note: init_y is not used by FBSNN, but kept for API + ) -> tuple[list[float], list[float]]: + """ + Solves the PDE using the FBSNN method. + + Parameters + ---------- + num_iterations : int + Number of training iterations. + batch_size : int + Batch size for training. + init_y : float + Initial value (not used by this solver). + + Returns + ------- + Tuple[List[float], List[float]] + - losses: List of losses at each iteration. + - inits: List of Y_0 values at each iteration. + """ + losses = [] + inits = [] + + self.solver.train() + + # Validation data + dw_val, y_val = self.pde.sample(128) + y_val = torch.tensor(y_val, dtype=torch.float32, device=self.device) + dw_val = torch.tensor(dw_val, dtype=torch.float32, device=self.device) + t_val = torch.ones((128, 1), device=self.device) + + for j in range(num_iterations): + logger.info("Iteration %d/%d", j + 1, num_iterations) + + dw_train, y_train = self.pde.sample(batch_size) + y_train = torch.tensor(y_train, dtype=torch.float32, device=self.device) + dw_train = torch.tensor(dw_train, dtype=torch.float32, device=self.device) + t_train = torch.ones((batch_size, 1), device=self.device) + + self.optimizer.zero_grad() + + # init_y is not used, pass a dummy tensor + dummy_init = torch.tensor(0.0, device=self.device) + + loss = self.compute_loss(y_train, dw_train, t_train, dummy_init) + loss.backward() + self.optimizer.step() + + # Validation and logging + with torch.no_grad(): + val_loss = self.compute_loss(y_val, dw_val, t_val, dummy_init) + losses.append(val_loss.item()) + + # Get Y_0 from the trained network + S0_val = y_val[:, :, 0] + t0_val = t_val * self.pde.delta_t * 0 + y0_val = self.solver(torch.cat((t0_val, S0_val), dim=1)) + + y0_mean = torch.mean(y0_val).item() + inits.append(y0_mean) + + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0_mean) + + return losses, inits + + +# Deprecated alias for the misspelled original name; scheduled for removal in 4.0.0. +FBSNNolver = deprecated_class(FBSNNSolver, "FBSNNolver", removed_in="4.0.0") diff --git a/src/RiskLabAI/utils/__init__.py b/src/RiskLabAI/utils/__init__.py new file mode 100644 index 0000000..2ca8a25 --- /dev/null +++ b/src/RiskLabAI/utils/__init__.py @@ -0,0 +1,108 @@ +""" +RiskLabAI Utilities Package + +This module provides common helper functions used across the library, +including: +- Constants +- EWMA calculation +- Progress bar +- Strategy side determination +- Plotly figure layout helpers +- Publication-quality Matplotlib plotting +""" + +from RiskLabAI._deprecation import warn_deprecated_name + +from .constants import * +from .ewma import ewma +from .momentum_mean_reverting_strategy_sides import determine_strategy_side +from .progress import progress_bar + +# Plotting helpers are imported lazily (PEP 562) so that the base install +# does not require matplotlib/seaborn/plotly (RiskLabAI[plot] extra). +_LAZY_PLOTTING = { + "update_figure_layout": ("update_figure_layout", "update_figure_layout"), + "setup_publication_style": ("publication_plots", "setup_publication_style"), + "apply_plot_style": ("publication_plots", "apply_plot_style"), + "finalize_plot": ("publication_plots", "finalize_plot"), +} + + +# Deprecated non-ASCII constant identifiers -> their ASCII replacements. +# Accessing these via RiskLabAI.utils warns and returns the new value; the +# identifiers are scheduled for removal in 4.0.0 (string values are unchanged regardless). +_DEPRECATED_CONSTANTS = { + "CUMULATIVE_θ": "CUMULATIVE_THETA", + "CUMULATIVE_BUY_θ": "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_θ": "CUMULATIVE_SELL_THETA", +} + + +def __getattr__(name): + if name in _LAZY_PLOTTING: + from importlib import import_module + + module_name, attr = _LAZY_PLOTTING[name] + value = getattr(import_module(f".{module_name}", __name__), attr) + globals()[name] = value + return value + if name in _DEPRECATED_CONSTANTS: + new = _DEPRECATED_CONSTANTS[name] + warn_deprecated_name(name, new, removed_in="4.0.0") + return globals()[new] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +# --- Alias for backward compatibility --- +# The historical `compute_exponential_weighted_moving_average` name now maps to +# the canonical, numba-jitted `ewma` (the former `smoothing_average.py` +# duplicate has been removed). +compute_exponential_weighted_moving_average = ewma + +__all__ = [ + # constants + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + # ewma + "ewma", + "compute_exponential_weighted_moving_average", # Alias + # progress + "progress_bar", + # momentum_mean_reverting_strategy_sides + "determine_strategy_side", + # update_figure_layout + "update_figure_layout", + # publication_plots + "setup_publication_style", + "apply_plot_style", + "finalize_plot", +] diff --git a/src/RiskLabAI/utils/constants.py b/src/RiskLabAI/utils/constants.py new file mode 100644 index 0000000..94366c6 --- /dev/null +++ b/src/RiskLabAI/utils/constants.py @@ -0,0 +1,87 @@ +DATE_TIME = "Date Time" +TIMESTAMP = "Timestamp" +TICK_NUMBER = "Tick Number" +OPEN_PRICE = "Open" +HIGH_PRICE = "High" +LOW_PRICE = "Low" +CLOSE_PRICE = "Close" + +CUMULATIVE_TICKS = "Cumulative Ticks" +CUMULATIVE_DOLLAR = "Cumulative Dollar Value" +THRESHOLD = "Threshold" + +CUMULATIVE_VOLUME = "Cumulative Volume" +CUMULATIVE_BUY_VOLUME = "Cumulative Buy Volume" +CUMULATIVE_SELL_VOLUME = "Cumulative Sell Volume" + +CUMULATIVE_THETA = "Cumulative θ" +CUMULATIVE_BUY_THETA = "Cumulative Buy θ" +CUMULATIVE_SELL_THETA = "Cumulative Sell θ" + +# Deprecated non-ASCII aliases (scheduled for removal in 4.0.0). The string *values* are +# unchanged, so stored data and internal dict keys are unaffected; only the +# Python identifier you import differs. These are intentionally excluded from +# ``__all__`` so ``RiskLabAI.utils`` can warn on access via its ``__getattr__``. +CUMULATIVE_θ = CUMULATIVE_THETA +CUMULATIVE_BUY_θ = CUMULATIVE_BUY_THETA +CUMULATIVE_SELL_θ = CUMULATIVE_SELL_THETA + +EXPECTED_IMBALANCE = "expected_imbalance" +EXPECTED_TICKS_NUMBER = "exp_num_ticks" + +EXPECTED_BUY_IMBALANCE = "exp_imbalance_buy" +EXPECTED_SELL_IMBALANCE = "exp_imbalance_sell" +EXPECTED_BUY_TICKS_PROPORTION = "exp_buy_ticks_proportion" +BUY_TICKS_NUMBER = "buy_ticks_num" + +N_TICKS_ON_BAR_FORMATION = "Number of ticks while bar is formed." + +PREVIOUS_TICK_RULE = "Previous tick rule" +EXPECTED_IMBALANCE_WINDOW = "Expected Imbalance Window" + +PREVIOUS_BARS_N_TICKS_LIST = "List of previous bars number of ticks" +PREVIOUS_TICK_IMBALANCES_LIST = "List of previous tick imbalances" + +PREVIOUS_TICK_IMBALANCES_BUY_LIST = "List of previous (buy) tick imbalances" +PREVIOUS_TICK_IMBALANCES_SELL_LIST = "List of previous (sell) tick imbalances" + +PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST = "List of previous bars buy ticks proportion" + +N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION = "Window size for E[T]" + +# Canonical public names. The deprecated non-ASCII θ aliases above are +# deliberately omitted so that `from .constants import *` does not bind them in +# RiskLabAI.utils (whose __getattr__ then warns on access). Scheduled for removal in 4.0.0. +__all__ = [ + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", +] diff --git a/src/RiskLabAI/utils/ewma.py b/src/RiskLabAI/utils/ewma.py new file mode 100644 index 0000000..ebd002e --- /dev/null +++ b/src/RiskLabAI/utils/ewma.py @@ -0,0 +1,74 @@ +""" +Implements a Numba-jitted Exponentially Weighted Moving Average (EWMA) +that correctly handles the 'adjust=True' logic from pandas. +""" + +import numpy as np + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + + +@jit(nopython=True) +def ewma(array: np.ndarray, window: int) -> np.ndarray: + r""" + Calculate the Exponentially Weighted Moving Average (EWMA) of an array. + + This function mimics the pandas `ewm(span=window, adjust=True).mean()` + method, which is crucial for financial calculations as it does not + lose the first `window` observations. + + The EWMA is calculated using the formula: + .. math:: + EWMA_t = \frac{x_t + (1 - \alpha) x_{t-1} + \ldots + (1 - \alpha)^t x_0} + {1 + (1 - \alpha) + \ldots + (1 - \alpha)^t} + + where: + - \(\alpha = 2 / (window + 1)\) + - The denominator \(\omega_t\) is the sum of weights. + + Parameters + ---------- + array : np.ndarray + Input array (e.g., of tick counts or imbalances). + window : int + The span of the EWMA. + + Returns + ------- + np.ndarray + The array of EWMA values. + """ + length = array.shape[0] + result_ewma_array = np.empty(length, dtype=np.float64) + if length == 0: + return result_ewma_array + + alpha = 2.0 / (float(window) + 1.0) + multiplier = 1.0 - alpha + + # Handle the sum of weights (denominator) + weight_sum = 1.0 + current_weighted_sum = array[0] + result_ewma_array[0] = current_weighted_sum + + for i in range(1, length): + weight_sum += multiplier**i + current_weighted_sum = current_weighted_sum * multiplier + array[i] + result_ewma_array[i] = current_weighted_sum / weight_sum + + return result_ewma_array diff --git a/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py b/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py new file mode 100644 index 0000000..208f23f --- /dev/null +++ b/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py @@ -0,0 +1,70 @@ +""" +Determines strategy side (long/short) based on moving average crossovers. +""" + +import pandas as pd + + +def determine_strategy_side( + prices: pd.Series, + fast_window: int = 20, + slow_window: int = 50, + exponential: bool = False, + mean_reversion: bool = False, +) -> pd.Series: + r""" + Determines the trading side based on moving average crossovers. + + .. math:: + \text{Momentum:} + \begin{cases} + 1 & \text{if } MA_{fast} \geq MA_{slow} \\ + -1 & \text{otherwise} + \end{cases} + + \text{Mean Reversion:} + \begin{cases} + 1 & \text{if } MA_{fast} < MA_{slow} \\ + -1 & \text{otherwise} + \end{cases} + + Parameters + ---------- + prices : pd.Series + Time series of prices. + fast_window : int, default=20 + Window size for the fast moving average. + slow_window : int, default=50 + Window size for the slow moving average. + exponential : bool, default=False + If True, use Exponential Moving Averages (EWMA). + If False, use Simple Moving Averages (SMA). + mean_reversion : bool, default=False + If True, strategy is mean-reverting (short fast > slow). + If False, strategy is momentum-based (long fast > slow). + + Returns + ------- + pd.Series + A Series of sides (1 for long, -1 for short). + """ + if fast_window >= slow_window: + raise ValueError("fast_window must be smaller than slow_window.") + + if exponential: + fast_ma = prices.ewm(span=fast_window, adjust=False, min_periods=1).mean() + slow_ma = prices.ewm(span=slow_window, adjust=False, min_periods=1).mean() + else: + fast_ma = prices.rolling(window=fast_window, min_periods=1).mean() + slow_ma = prices.rolling(window=slow_window, min_periods=1).mean() + + # Create signal: 1 if fast > slow, -1 if fast < slow + # Explicit int64: plain `int` maps to int32 on Windows, which makes the + # returned dtype platform-dependent. + signal = (fast_ma >= slow_ma).astype("int64") * 2 - 1 + + if mean_reversion: + # Invert the signal for mean reversion + return -signal + + return signal diff --git a/src/RiskLabAI/utils/progress.py b/src/RiskLabAI/utils/progress.py new file mode 100644 index 0000000..ff893fd --- /dev/null +++ b/src/RiskLabAI/utils/progress.py @@ -0,0 +1,68 @@ +""" +Provides a simple terminal-style progress bar. +""" + +import sys +import time + + +def progress_bar( + current_progress: int, + total_progress: int, + start_time: float, + bar_length: int = 20, +) -> None: + r""" + Display a terminal-style progress bar. + + Shows completion percentage and estimated time remaining. + + Parameters + ---------- + current_progress : int + Current iteration number (e.g., `i`). + total_progress : int + Total number of items to process. + start_time : float + The time the task started (from `time.time()`). + bar_length : int, default=20 + Length of the progress bar in characters. + """ + if total_progress == 0: + return + + # Check if task is completed + if current_progress == total_progress: + sys.stdout.write( + f"\rCompleted: [{'-' * bar_length}] 100% - Task completed! \n" + ) + else: + percentage = current_progress / total_progress + + # Handle the very first iteration + if percentage == 0: + arrow = "" + else: + # Safely calculate arrow length, ensuring it doesn't go below -1 + arrow_length = int(round(percentage * bar_length) - 1) + arrow = "-" * max(0, arrow_length) + ">" + + spaces = " " * (bar_length - len(arrow)) + elapsed_time_sec = time.time() - start_time + + # Calculate remaining time + if current_progress == 0: + remaining_time_str = "Calculating..." + else: + remaining_time_min = ( + (elapsed_time_sec / current_progress) + * (total_progress - current_progress) + ) / 60 + remaining_time_str = f"{remaining_time_min:.0f} minutes remaining" + + # Assemble and print the bar + sys.stdout.write( + f"\rCompleted: [{arrow + spaces}] {percentage*100:.0f}% - {remaining_time_str}." + ) + + sys.stdout.flush() diff --git a/src/RiskLabAI/utils/publication_plots.py b/src/RiskLabAI/utils/publication_plots.py new file mode 100644 index 0000000..c6c9a84 --- /dev/null +++ b/src/RiskLabAI/utils/publication_plots.py @@ -0,0 +1,192 @@ +""" +Utilities for creating publication-quality plots with Matplotlib +and Seaborn, using Times New Roman font and high DPI. + +Provides 6 themes and a configuration-based saving function. +""" + +import logging +import os +from typing import Any, Optional + +import matplotlib.figure as fig # For type hinting +import matplotlib.pyplot as plt +import seaborn as sns + +logger = logging.getLogger(__name__) + +# [THEMES dictionary remains the same] +THEMES: dict[str, dict[str, Any]] = { + "light": { + "figure.facecolor": "#FFFFFF", + "axes.facecolor": "#FFFFFF", + "text.color": "#000000", + "axes.labelcolor": "#000000", + "axes.edgecolor": "#000000", + "xtick.color": "#000000", + "ytick.color": "#000000", + "grid.color": "#CCCCCC", + "legend.facecolor": "#FFFFFF", + "legend.edgecolor": "#B0B0B0", + }, + "medium": { + "figure.facecolor": "#B0B0B0", # A more solid, medium grey + "axes.facecolor": "#B0B0B0", + "text.color": "#FFFFFF", # White text (like the dark theme) + "axes.labelcolor": "#FFFFFF", + "axes.edgecolor": "#FFFFFF", + "xtick.color": "#FFFFFF", + "ytick.color": "#FFFFFF", + "grid.color": "#E0E0E0", # Lighter grid lines on medium bg + "legend.facecolor": "#B0B0B0", + "legend.edgecolor": "#FFFFFF", + }, + "dark": { + "figure.facecolor": "#2E2E2E", + "axes.facecolor": "#2E2E2E", + "text.color": "#F0F0F0", + "axes.labelcolor": "#F0F0F0", + "axes.edgecolor": "#F0F0F0", + "xtick.color": "#F0F0F0", + "ytick.color": "#F0F0F0", + "grid.color": "#6A6A6A", + "legend.facecolor": "#2E2E2E", + "legend.edgecolor": "#F0F0F0", + }, +} + +# --- MODULE-LEVEL CONFIGURATION --- +# This dictionary will store the settings from setup_publication_style +_CONFIG = {"save_plots": False, "save_dir": "figs"} + + +# --- UPDATED FUNCTION --- +def setup_publication_style( + theme: str = "light", + quality: int = 300, + save_plots: bool = False, # <-- New parameter + save_dir: str = "figs", # <-- New parameter +) -> None: + """ + Sets the global Matplotlib rcParams and saving configuration. + + Call this function once at the beginning of your notebook. + + Parameters + ---------- + theme : str, optional + The theme to apply. Defaults to 'light'. + quality : int, optional + The DPI for the figures. Defaults to 300. + save_plots : bool, optional + Global switch to enable/disable saving plots. Defaults to False. + save_dir : str, optional + The directory to save figures in. Defaults to 'figs'. + """ + + # [All the theme parsing and styling code remains the same] + # ... (omitted for brevity) ... + is_transparent = False + base_theme_name = theme + if theme.endswith("-transparent"): + is_transparent = True + base_theme_name = theme.replace("-transparent", "") + if base_theme_name not in THEMES: + base_theme_name = "light" + params = THEMES[base_theme_name].copy() + common_params = { + "font.size": 12, + "axes.labelsize": 12, + "axes.titlesize": 14, + "axes.titleweight": "bold", + "xtick.labelsize": 12, + "ytick.labelsize": 12, + "legend.fontsize": 12, + "legend.title_fontsize": 13, + "figure.dpi": quality, + "savefig.dpi": quality, + "axes.grid": True, + "grid.linestyle": "--", + "grid.alpha": 0.7, + "axes.linewidth": 1.2, + } + params.update(common_params) + if is_transparent: + params["figure.facecolor"] = (0, 0, 0, 0) + params["axes.facecolor"] = (0, 0, 0, 0) + params["savefig.transparent"] = True + params["legend.facecolor"] = (0, 0, 0, 0) + else: + params["savefig.transparent"] = False + try: + plt.rc("font", family="Times New Roman") + except Exception: + logger.warning("Times New Roman not found. Defaulting to serif.") + plt.rc("font", family="serif") + plt.rcParams.update(params) + sns_style = "darkgrid" if base_theme_name == "dark" else "whitegrid" + sns.set_style(sns_style, rc=params) + + # --- Store saving configuration --- + _CONFIG["save_plots"] = save_plots + _CONFIG["save_dir"] = save_dir + + logger.info( + "Matplotlib style updated. Theme: '%s', Quality: %s DPI.", theme, quality + ) + if save_plots: + logger.info("Plot saving enabled. Saving to: '%s'", save_dir) + else: + logger.info("Plot saving disabled.") + + +# [apply_plot_style function remains exactly the same] +def apply_plot_style( + ax: plt.Axes, + title: str, + xlabel: str, + ylabel: str, + legend_title: Optional[str] = None, +) -> None: + ax.set_title(title) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + if ax.get_legend() and legend_title is not None: + ax.legend(title=legend_title) + + +# --- UPDATED FUNCTION --- +def finalize_plot(fig: fig.Figure, filename: str) -> None: + """ + Shows the plot and saves it *if* saving was enabled in + setup_publication_style. + + Parameters + ---------- + fig : plt.Figure + The figure object to save. + filename : str + The name of the file (e.g., 'model_performance.png'). + This is required, but only used if saving is enabled. + """ + + # --- 1. Save the figure if global switch is on --- + if _CONFIG["save_plots"]: + save_dir = _CONFIG["save_dir"] + + # Create the directory if it doesn't exist + os.makedirs(save_dir, exist_ok=True) + + # Construct the full path + full_path = os.path.join(save_dir, filename) + + # Save the figure + fig.savefig(full_path, bbox_inches="tight") + + logger.info("Figure saved to: %s", full_path) + + # --- 2. Always show the plot --- + plt.show() + + # --- 3. Close the figure object --- + plt.close(fig) diff --git a/src/RiskLabAI/utils/update_figure_layout.py b/src/RiskLabAI/utils/update_figure_layout.py new file mode 100644 index 0000000..b4357e3 --- /dev/null +++ b/src/RiskLabAI/utils/update_figure_layout.py @@ -0,0 +1,56 @@ +""" +A helper function to apply a consistent, dark-themed layout +to Plotly figures. +""" + +import plotly.graph_objects as go + + +def update_figure_layout( + fig: go.Figure, + title: str, + xaxis_title: str, + yaxis_title: str, + legend_x: float = 1.0, + legend_y: float = 1.0, +) -> go.Figure: + """ + Apply a standardized dark-theme layout to a Plotly figure. + + This function modifies the figure in-place. + + Parameters + ---------- + fig : go.Figure + The Plotly figure object to update. + title : str + The title for the chart. + xaxis_title : str + The title for the x-axis. + yaxis_title : str + The title for the y-axis. + legend_x : float, default=1.0 + The x-position of the legend. + legend_y : float, default=1.0 + The y-position of the legend. + + Returns + ------- + go.Figure + The modified figure. + """ + fig.update_layout( + title=title, + xaxis_title=xaxis_title, + yaxis_title=yaxis_title, + template="plotly_dark", + plot_bgcolor="rgba(0,0,0,0)", # Transparent background + paper_bgcolor="rgba(0,0,0,0)", # Transparent background + legend=dict( + x=legend_x, + y=legend_y, + xanchor="auto", + yanchor="auto", + ), + ) + return fig diff --git a/test/backtest/test_technical_indicators.py b/test/backtest/test_technical_indicators.py new file mode 100644 index 0000000..3b81506 --- /dev/null +++ b/test/backtest/test_technical_indicators.py @@ -0,0 +1,119 @@ +import builtins + +import numpy as np +import pandas as pd +import pytest + +from RiskLabAI.backtest.backtest_overfitting_simulation import ( + _average_directional_index, + _average_true_range, + _commodity_channel_index, + _ichimoku_lines, + _relative_strength_index, + _stochastic_oscillator, + _wilder_change_average, + financial_features_backtest_overfitting_simulation, +) + + +def test_wilder_indicators_match_hand_calculations(): + prices = pd.Series([1.0, 2.0, 3.0, 2.0, 4.0]) + np.testing.assert_allclose( + _relative_strength_index(prices, 2).to_numpy()[2:], + [100.0, 50.0, 100.0 * 1.25 / 1.5], + rtol=0.0, + atol=1e-12, + ) + + true_range_prices = pd.Series([1.0, 3.0, 2.0, 6.0]) + np.testing.assert_allclose( + _average_true_range(true_range_prices, 2).to_numpy()[2:], + [1.5, 2.75], + rtol=0.0, + atol=1e-12, + ) + + directional_prices = pd.Series([1.0, 2.0, 3.0, 2.0, 1.0, 2.0, 3.0]) + np.testing.assert_allclose( + _average_directional_index(directional_prices, 2).to_numpy()[3:], + [50.0, 50.0, 37.5, 50.0], + rtol=0.0, + atol=1e-12, + ) + + +def test_rolling_indicators_match_hand_calculations(): + prices = pd.Series([1.0, 2.0, 3.0, 2.0, 4.0]) + np.testing.assert_allclose( + _commodity_channel_index(prices.iloc[:4], 3).to_numpy()[2:], + [100.0, -50.0], + rtol=0.0, + atol=1e-12, + ) + + oscillator, signal = _stochastic_oscillator(prices, 3, 3) + np.testing.assert_allclose( + oscillator.to_numpy()[2:], [100.0, 0.0, 100.0], rtol=0.0, atol=1e-12 + ) + np.testing.assert_allclose( + signal.to_numpy()[4:], [200.0 / 3.0], rtol=0.0, atol=1e-12 + ) + + conversion, base, span_a, span_b = _ichimoku_lines( + pd.Series(np.arange(1.0, 7.0)), 2, 3, 4 + ) + np.testing.assert_allclose(conversion.to_numpy()[1:], [1.5, 2.5, 3.5, 4.5, 5.5]) + np.testing.assert_allclose(base.to_numpy()[2:], [2.0, 3.0, 4.0, 5.0]) + np.testing.assert_allclose(span_a.to_numpy()[2:], [2.25, 3.25, 4.25, 5.25]) + np.testing.assert_allclose(span_b.to_numpy()[3:], [2.5, 3.5, 4.5]) + + +def test_constant_prices_produce_neutral_mature_indicators(): + prices = pd.Series(np.ones(60)) + assert _relative_strength_index(prices).iloc[-1] == 50.0 + assert _average_true_range(prices).iloc[-1] == 0.0 + assert _average_directional_index(prices).iloc[-1] == 0.0 + assert _commodity_channel_index(prices).iloc[-1] == 0.0 + oscillator, signal = _stochastic_oscillator(prices) + assert oscillator.iloc[-1] == 50.0 + assert signal.iloc[-1] == 50.0 + + +@pytest.mark.parametrize("window", [0, -1, True, 1.5]) +def test_wilder_average_rejects_invalid_windows(window): + with pytest.raises(ValueError, match="positive integer"): + _wilder_change_average(np.array([0.0, 1.0, 2.0]), window) + + +def test_financial_feature_generation_is_self_contained(monkeypatch): + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "ta" or name.startswith("ta."): + raise AssertionError("unexpected external technical-analysis import") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + generator = np.random.default_rng(7) + prices = pd.Series( + 100.0 * np.exp(np.cumsum(generator.normal(0.0, 0.01, 300))), + index=pd.date_range("2024-01-01", periods=300), + ) + features = financial_features_backtest_overfitting_simulation(prices) + + assert not features.empty + assert features.index.isin(prices.index).all() + assert np.isfinite(features.dropna().to_numpy()).all() + assert { + "ADX", + "RSI", + "CCI", + "Stochastic", + "ROC", + "ATR", + "Kumo Breakout", + "TK Position", + "Price Kumo Position", + "Cloud Thickness", + "Momentum Confirmation", + }.issubset(features.columns) diff --git a/test/backtest/validation/test_cross_validator_controller.py b/test/backtest/validation/test_cross_validator_controller.py index 306fe2d..72b6a64 100644 --- a/test/backtest/validation/test_cross_validator_controller.py +++ b/test/backtest/validation/test_cross_validator_controller.py @@ -55,6 +55,6 @@ def test_controller_public_attribute(dummy_args): def test_controller_is_deprecated(): """The controller is deprecated in 2.0.0 (delegates to the core registry).""" - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): controller = CrossValidatorController(validator_type="kfold", n_splits=5) assert isinstance(controller.get_validator(), KFold) diff --git a/test/backtest/validation/test_cross_validator_factory.py b/test/backtest/validation/test_cross_validator_factory.py index 7008cd7..68bd331 100644 --- a/test/backtest/validation/test_cross_validator_factory.py +++ b/test/backtest/validation/test_cross_validator_factory.py @@ -91,6 +91,6 @@ def test_factory_invalid_type(): def test_factory_is_deprecated(): """The factory is deprecated in 2.0.0 (delegates to the core registry).""" - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): cv = CrossValidatorFactory.create_cross_validator("kfold", n_splits=5) assert isinstance(cv, KFold) diff --git a/test/causal_factor_analysis/test_cfi_book_graph_examples.py b/test/causal_factor_analysis/test_cfi_book_graph_examples.py new file mode 100644 index 0000000..4536415 --- /dev/null +++ b/test/causal_factor_analysis/test_cfi_book_graph_examples.py @@ -0,0 +1,100 @@ +"""Hand-verifiable graph tests for the 2023 book examples.""" + +from __future__ import annotations + +from RiskLabAI.causal_factor_analysis import ( + CausalDAG, + causal_role_evidence, + check_backdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) + + +def _dag(nodes, edges, observed=None): + node_tuple = tuple(nodes) + return CausalDAG( + node_tuple, + tuple(edges), + node_tuple if observed is None else tuple(observed), + ) + + +def test_figures_1_2_and_13_require_the_common_cause_for_backdoor_adjustment(): + graph = _dag(("X", "Y", "Z"), (("Z", "X"), ("Z", "Y"))) + assert d_separation(graph, ("X",), ("Y",)).separated is False + assert d_separation(graph, ("X",), ("Y",), ("Z",)).separated is True + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("Z",),) + role = causal_role_evidence(graph, "X", "Y", "Z") + assert role.common_cause_paths == (("X", "Z", "Y"),) + + +def test_figure_3_frontdoor_mediator_identifies_when_confounder_is_latent(): + graph = _dag( + ("M", "U", "X", "Y"), + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("M", "X", "Y"), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == () + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + + +def test_figure_4_instrument_satisfies_both_named_graph_profiles(): + graph = _dag( + ("U", "W", "X", "Y"), + (("W", "X"), ("X", "Y"), ("U", "X"), ("U", "Y")), + ) + result = check_instrument(graph, "W", "X", "Y") + assert result.pearl_graphical is True + assert result.cfi_simple is True + + +def test_figure_8_hypothetical_factor_mechanism_has_backdoor_and_frontdoor_sets(): + edges = ( + ("MOM", "HML"), + ("MOM", "PC"), + ("HML", "OI"), + ("OI", "PC"), + ) + observed_graph = _dag(("HML", "MOM", "OI", "PC"), edges) + assert minimal_backdoor_adjustment_sets(observed_graph, "HML", "PC") == (("MOM",),) + + latent_graph = _dag( + ("HML", "MOM", "OI", "PC"), + edges, + observed=("HML", "OI", "PC"), + ) + assert minimal_backdoor_adjustment_sets(latent_graph, "HML", "PC") == () + assert minimal_frontdoor_adjustment_sets(latent_graph, "HML", "PC") == (("OI",),) + + +def test_figures_16_to_18_conditioning_on_collider_opens_the_path(): + graph = _dag(("X", "Y", "Z"), (("X", "Z"), ("Y", "Z"))) + assert d_separation(graph, ("X",), ("Y",)).separated is True + conditioned = d_separation(graph, ("X",), ("Y",), ("Z",)) + assert conditioned.separated is False + assert tuple(path.nodes for path in conditioned.paths if path.is_open) == ( + ("X", "Z", "Y"), + ) + + +def test_figure_19_conditioning_on_chain_mediator_blocks_total_effect_path(): + graph = _dag(("X", "Y", "Z"), (("X", "Z"), ("Z", "Y"))) + assert d_separation(graph, ("X",), ("Y",)).separated is False + assert d_separation(graph, ("X",), ("Y",), ("Z",)).separated is True + role = causal_role_evidence(graph, "X", "Y", "Z") + assert role.mediator_on_directed_paths == (("X", "Z", "Y"),) + + +def test_figure_20_empty_backdoor_set_is_valid_but_conditioning_mediator_is_not(): + graph = _dag( + ("W", "X", "Y", "Z"), + (("X", "Z"), ("W", "Z"), ("Z", "Y"), ("W", "Y")), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == ((),) + assert check_backdoor_adjustment_set(graph, "X", "Y", ()).admissible is True + conditioned = check_backdoor_adjustment_set(graph, "X", "Y", ("Z",)) + assert conditioned.admissible is False + assert conditioned.descendants_in_set == ("Z",) diff --git a/test/causal_factor_analysis/test_factor_mirage.py b/test/causal_factor_analysis/test_factor_mirage.py new file mode 100644 index 0000000..ef32f86 --- /dev/null +++ b/test/causal_factor_analysis/test_factor_mirage.py @@ -0,0 +1,405 @@ +"""Tests for the published factor-mirage analytical formulas.""" + +from __future__ import annotations + +import math +import subprocess +import sys +from dataclasses import FrozenInstanceError +from fractions import Fraction +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + ColliderCoefficients, + ColliderDiagnostics, + StrategyPerformance, + collider_factor_return, + collider_forecast_return, + collider_model_diagnostics, + collider_overcontrolled_coefficients, + confounder_factor_return, + confounder_forecast_return, + confounder_undercontrolled_coefficient, +) + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [ + (1.0, 3.0, 1.0), + (-0.7, 1.2, -0.4), + (0.0, -2.0, 3.0), + (2.5, 0.0, -8.0), + ], +) +def test_confounder_coefficient_matches_equation_2(beta, gamma, delta): + expected = beta + gamma * delta / (1.0 + delta**2) + actual = confounder_undercontrolled_coefficient(beta, gamma, delta) + assert actual == pytest.approx(expected, rel=1e-14, abs=1e-14) + assert actual - beta == pytest.approx( + gamma * delta / (1.0 + delta**2), rel=1e-14, abs=1e-14 + ) + + +def test_confounder_zero_links_remove_omitted_variable_bias(): + assert confounder_undercontrolled_coefficient(2.0, 0.0, 4.0) == 2.0 + assert confounder_undercontrolled_coefficient(2.0, 4.0, 0.0) == 2.0 + + +def test_mirage_confounder_example_matches_equations_5_to_9(): + coefficient = confounder_undercontrolled_coefficient(1.0, 3.0, 1.0) + factor = confounder_factor_return(1.0, 1.0, 1.0, 3.0, 1.0) + forecast = confounder_forecast_return(1.0, 3.0, 1.0) + + assert coefficient == 2.5 + assert factor == StrategyPerformance(correct=16.0, misspecified=6.25) + assert forecast.correct == pytest.approx(17.0, abs=1e-14) + assert forecast.misspecified == pytest.approx(12.5, abs=1e-14) + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [(1.0, 3.0, 1.0), (-0.5, 0.75, 2.0), (2.0, -1.0, -0.25)], +) +def test_static_confounder_forecast_penalty_identity(beta, gamma, delta): + result = confounder_forecast_return(beta, gamma, delta) + expected_gap = gamma**2 / (1.0 + delta**2) + assert result.correct - result.misspecified == pytest.approx( + expected_gap, rel=1e-12, abs=1e-14 + ) + assert result.correct >= 0.0 + assert result.misspecified >= 0.0 + + +def test_confounder_shift_can_create_systematic_losses(): + factor = confounder_factor_return( + 1.0, + 1.0, + 1.0, + 3.0, + 1.0, + delta_realized=-1.0, + ) + forecast = confounder_forecast_return(1.0, 3.0, 1.0, delta_realized=-1.0) + + assert factor == StrategyPerformance(correct=16.0, misspecified=-1.25) + assert forecast.correct == pytest.approx(5.0, abs=1e-14) + assert forecast.misspecified == pytest.approx(-2.5, abs=1e-14) + + +def test_confounder_shift_loss_has_exact_zero_boundaries(): + # beta=1, gamma=3 has roots (-3 +/- sqrt(5))/2. + root = (-3.0 + math.sqrt(5.0)) / 2.0 + factor = confounder_factor_return(2.0, 0.5, 1.0, 3.0, root, delta_realized=1.0) + forecast = confounder_forecast_return(1.0, 3.0, root, delta_realized=1.0) + assert factor.misspecified == pytest.approx(0.0, abs=2e-15) + assert forecast.misspecified == pytest.approx(0.0, abs=2e-15) + assert ( + confounder_factor_return( + 0.0, 0.5, 1.0, 3.0, 1.0, delta_realized=-1.0 + ).misspecified + == 0.0 + ) + + +def test_omitted_realized_delta_is_the_constant_parameter_case(): + static_factor = confounder_factor_return(0.7, -0.2, 0.8, 1.1, -0.6) + shifted_factor = confounder_factor_return( + 0.7, -0.2, 0.8, 1.1, -0.6, delta_realized=-0.6 + ) + static_forecast = confounder_forecast_return(0.8, 1.1, -0.6) + shifted_forecast = confounder_forecast_return(0.8, 1.1, -0.6, delta_realized=-0.6) + assert shifted_factor == static_factor + assert shifted_forecast == static_forecast + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [(0.5, 1.0, 0.8), (1.0, -2.0, 0.25), (-0.7, 0.4, -1.2)], +) +def test_collider_coefficients_match_equations_11_and_12(beta, gamma, delta): + result = collider_overcontrolled_coefficients(beta, gamma, delta) + assert result.beta_hat == pytest.approx( + (beta - delta * gamma) / (1.0 + gamma**2), rel=1e-14, abs=1e-14 + ) + assert result.theta_hat == pytest.approx( + gamma / (1.0 + gamma**2), rel=1e-14, abs=1e-14 + ) + + +def test_collider_unbiased_condition_has_the_correct_sign(): + beta, gamma = 0.7, 1.5 + unbiased = collider_overcontrolled_coefficients(beta, gamma, -beta * gamma) + delta_zero = collider_overcontrolled_coefficients(beta, gamma, 0.0) + gamma_zero = collider_overcontrolled_coefficients(beta, 0.0, 4.0) + + assert unbiased.beta_hat == pytest.approx(beta, rel=1e-14, abs=1e-14) + assert unbiased.theta_hat != 0.0 + assert delta_zero.beta_hat != pytest.approx(beta) + assert gamma_zero == ColliderCoefficients(beta_hat=beta, theta_hat=0.0) + + +def test_conditioning_back_over_x_recovers_the_causal_expectation(): + beta, gamma, delta, x = 0.8, 1.2, -0.4, 2.5 + coefficients = collider_overcontrolled_coefficients(beta, gamma, delta) + expected_proxy = (beta * gamma + delta) * x + conditional_prediction = ( + coefficients.beta_hat * x + coefficients.theta_hat * expected_proxy + ) + assert conditional_prediction == pytest.approx(beta * x, rel=1e-14, abs=1e-14) + + +def test_mirage_collider_factor_example_is_negative(): + result = collider_factor_return(1.0, 1.0, 1.0, 1.0, 3.0) + assert result == StrategyPerformance(correct=1.0, misspecified=-0.5) + + +def test_mirage_collider_forecast_example_is_negative(): + result = collider_forecast_return(1.0, 1.0, 2.0) + assert result == StrategyPerformance(correct=1.0, misspecified=-0.5) + + +def test_collider_loss_boundaries_are_exact(): + assert collider_forecast_return(0.0, 2.0, 3.0).misspecified == 0.0 + assert collider_forecast_return(2.0, 1.0, 2.0).misspecified == 0.0 + assert collider_factor_return(0.0, 4.0, 1.0, 2.0, 3.0).misspecified == 0.0 + assert collider_factor_return(1.0, 2.0, 0.0, 2.0, 3.0).misspecified == 0.0 + + +def test_zero_collider_link_makes_both_strategies_equal(): + factor = collider_factor_return(1.25, -8.0, 0.6, 0.0, 3.0) + forecast = collider_forecast_return(0.6, 0.0, 3.0) + assert factor.misspecified == pytest.approx(factor.correct, abs=1e-15) + assert forecast.misspecified == pytest.approx(forecast.correct, abs=1e-15) + + +def test_collider_diagnostics_match_appendix_d(): + result = collider_model_diagnostics(1.0, 1.0, 2.0, 5) + assert result.residual_variance == 0.5 + assert result.outcome_variance == pytest.approx(2.0, abs=1e-14) + assert result.correct_r_squared == 0.5 + assert result.overcontrolled_r_squared == 0.75 + assert result.correct_adjusted_r_squared == pytest.approx(1.0 / 3.0) + assert result.overcontrolled_adjusted_r_squared == 0.5 + assert result.correct_beta_variance == 0.2 + assert result.overcontrolled_beta_variance == pytest.approx(11.0 / 20.0) + assert result.collider_coefficient_variance == 0.05 + assert result.correct_beta_t_statistic == pytest.approx(math.sqrt(5.0)) + assert result.overcontrolled_beta_t_statistic == pytest.approx( + -math.sqrt(5.0 / 11.0) + ) + assert result.collider_t_statistic == pytest.approx(math.sqrt(5.0)) + assert result.adjusted_r_squared_prefers_overcontrolled is True + assert result.absolute_beta_t_prefers_overcontrolled is False + + +def test_ordinary_r_squared_is_strict_only_for_nonzero_gamma(): + equal = collider_model_diagnostics(0.8, 0.0, 10.0, 50) + larger = collider_model_diagnostics(0.8, -0.2, 10.0, 50) + assert equal.overcontrolled_r_squared == equal.correct_r_squared + assert larger.overcontrolled_r_squared > larger.correct_r_squared + + +def test_small_nonzero_links_do_not_cancel_out_of_r_squared(): + beta_only = collider_model_diagnostics(1e-8, 0.0, 0.0, 20) + gamma_only = collider_model_diagnostics(0.0, 1e-8, 0.0, 20) + assert beta_only.correct_r_squared == pytest.approx(1e-16, rel=1e-15) + assert gamma_only.overcontrolled_r_squared == pytest.approx(1e-16, rel=1e-15) + + +def test_adjusted_r_squared_threshold_is_strict_and_sign_invariant(): + sample_size = 7 + threshold = 1.0 / math.sqrt(sample_size - 3) + at_boundary = collider_model_diagnostics(1.0, threshold, 0.0, sample_size) + below = collider_model_diagnostics(1.0, 0.49, 0.0, sample_size) + above = collider_model_diagnostics(1.0, -0.51, 0.0, sample_size) + + assert at_boundary.correct_adjusted_r_squared == pytest.approx( + at_boundary.overcontrolled_adjusted_r_squared, abs=1e-15 + ) + assert at_boundary.adjusted_r_squared_prefers_overcontrolled is False + assert below.adjusted_r_squared_prefers_overcontrolled is False + assert above.adjusted_r_squared_prefers_overcontrolled is True + + +@pytest.mark.parametrize( + "beta,gamma,delta,n_observations", + [(0.3, 0.8, -0.2, 20), (-1.0, 1.5, 0.4, 100), (0.0, -2.0, 3.0, 8)], +) +def test_expected_t_statistics_match_appendix_d(beta, gamma, delta, n_observations): + result = collider_model_diagnostics(beta, gamma, delta, n_observations) + denominator = math.sqrt((beta * gamma + delta) ** 2 + gamma**2 + 1.0) + expected_overcontrolled = ( + math.sqrt(n_observations) * (beta - delta * gamma) / denominator + ) + assert result.correct_beta_t_statistic == pytest.approx( + math.sqrt(n_observations) * beta, rel=1e-13, abs=1e-14 + ) + assert result.overcontrolled_beta_t_statistic == pytest.approx( + expected_overcontrolled, rel=1e-13, abs=1e-14 + ) + assert result.collider_t_statistic == pytest.approx( + math.sqrt(n_observations) * gamma, rel=1e-13, abs=1e-14 + ) + assert result.absolute_beta_t_prefers_overcontrolled is ( + abs(beta - delta * gamma) / denominator > abs(beta) + ) + + +@pytest.mark.parametrize("sample_size", [True, 4.0, 3, 0, -4]) +def test_invalid_sample_sizes_are_rejected(sample_size): + match = "integer" if isinstance(sample_size, (bool, float)) else "greater than 3" + with pytest.raises(ValueError, match=match): + collider_model_diagnostics(1.0, 1.0, 1.0, sample_size) + + +def test_numpy_integer_sample_size_is_accepted(): + result = collider_model_diagnostics(1.0, 1.0, 1.0, np.int64(5)) + assert isinstance(result, ColliderDiagnostics) + + +@pytest.mark.parametrize( + "invalid", + [True, 1.0 + 2.0j, np.nan, np.inf, -np.inf, [1.0], np.array([1.0])], +) +def test_invalid_real_scalars_are_rejected(invalid): + calls = ( + lambda: confounder_undercontrolled_coefficient(invalid, 1.0, 1.0), + lambda: confounder_factor_return(invalid, 1.0, 1.0, 1.0, 1.0), + lambda: confounder_forecast_return(invalid, 1.0, 1.0), + lambda: collider_overcontrolled_coefficients(invalid, 1.0, 1.0), + lambda: collider_factor_return(invalid, 1.0, 1.0, 1.0, 1.0), + lambda: collider_forecast_return(invalid, 1.0, 1.0), + lambda: collider_model_diagnostics(invalid, 1.0, 1.0, 10), + ) + for call in calls: + with pytest.raises(ValueError, match="real scalar|finite"): + call() + + +def test_extreme_finite_scaling_avoids_spurious_overflow(): + coefficient = confounder_undercontrolled_coefficient(0.0, 1e308, 1e308) + factor = confounder_factor_return(1e200, 0.0, 1e-200, 0.0, 1.0) + collider = collider_overcontrolled_coefficients(1.0, 1e308, 1.0) + + assert coefficient == pytest.approx(1.0, rel=1e-15) + assert factor.correct == pytest.approx(1.0, rel=1e-15) + assert factor.misspecified == pytest.approx(1.0, rel=1e-15) + assert collider.beta_hat == pytest.approx(-1e-308, rel=1e-15, abs=0.0) + assert collider.theta_hat == pytest.approx(1e-308, rel=1e-15, abs=0.0) + + +def test_extreme_collider_scaling_preserves_representable_results(): + coefficients = collider_overcontrolled_coefficients(1e308, 1e163, 0.0) + diagnostics = collider_model_diagnostics(1e154, 1e200, 0.0, 4) + assert coefficients.beta_hat == pytest.approx(1e-18, rel=1e-14, abs=0.0) + assert diagnostics.overcontrolled_beta_variance == pytest.approx( + 2.5e-93, rel=1e-14, abs=0.0 + ) + + +def test_confounder_composites_preserve_latent_scaled_coefficients(): + maximum = sys.float_info.max + finite_large = confounder_factor_return(1e-308, 0.0, maximum, maximum, 1.0) + latent_factor = confounder_factor_return(1e308, 0.0, 0.0, 1e-200, 1e200) + latent_forecast = confounder_forecast_return(0.0, 1e-200, 1.0, delta_realized=1e308) + + assert finite_large.correct == pytest.approx(3.2317006071310996, rel=1e-15) + assert finite_large.misspecified == pytest.approx(7.271326366044974, rel=1e-15) + assert latent_factor.misspecified == pytest.approx(1e-184, rel=1e-14) + assert latent_forecast.misspecified == pytest.approx(5e-93, rel=1e-14) + + +def test_collider_composites_preserve_latent_scaled_coefficients(): + latent_factor = collider_factor_return(1e208, 0.0, 1e-54, 1e200, 0.0) + latent_forecast = collider_forecast_return(1e75, 1e200, 0.0) + tiny_prefactor = collider_factor_return(1e-308, 1e308, 1e-308, 1.0, 0.0) + + assert latent_factor.correct == pytest.approx(1e308, rel=1e-14) + assert latent_factor.misspecified == pytest.approx(1e-92, rel=1e-14) + assert latent_forecast.correct == pytest.approx(1e150, rel=1e-14) + assert latent_forecast.misspecified == pytest.approx(1e-250, rel=1e-14) + assert tiny_prefactor.misspecified == pytest.approx(5e-309, rel=1e-14) + + +def test_expected_t_statistic_preserves_latent_scaled_ratio(): + diagnostics = collider_model_diagnostics(0.0, 1e-200, 1e-200, 10**200) + assert diagnostics.overcontrolled_beta_t_statistic == pytest.approx( + -1e-300, rel=1e-14, abs=0.0 + ) + assert diagnostics.absolute_beta_t_prefers_overcontrolled is True + + +def test_loss_sign_is_preserved_at_ill_conditioned_boundaries(): + confounder = confounder_undercontrolled_coefficient(1.0, 1e10, -1e-10) + collider = collider_forecast_return(1.0, -1e-10, -1e10) + assert confounder == pytest.approx(-3.642219731549774e-17, rel=1e-14) + assert collider.misspecified == pytest.approx(-3.643219731549774e-17, rel=1e-14) + assert confounder < 0.0 + assert collider.misspecified < 0.0 + + +def test_adjusted_r_squared_boolean_uses_the_exact_strict_inequality(): + gamma = 1.0 / math.sqrt(3.0) + diagnostics = collider_model_diagnostics(0.0, gamma, 0.0, 6) + expected = Fraction.from_float(gamma) ** 2 * 3 > 1 + assert diagnostics.adjusted_r_squared_prefers_overcontrolled is expected + + +def test_unrepresentable_result_raises_stable_value_error(): + with pytest.raises(ValueError, match="cannot be represented"): + collider_forecast_return(1e308, 0.0, 0.0) + + +def test_result_records_are_frozen(): + performance = collider_forecast_return(1.0, 1.0, 2.0) + coefficients = collider_overcontrolled_coefficients(1.0, 1.0, 2.0) + diagnostics = collider_model_diagnostics(1.0, 1.0, 2.0, 5) + with pytest.raises(FrozenInstanceError): + performance.correct = 0.0 + with pytest.raises(FrozenInstanceError): + coefficients.beta_hat = 0.0 + with pytest.raises(FrozenInstanceError): + diagnostics.correct_r_squared = 0.0 + + +def test_public_import_is_isolated_and_source_conflicted_api_is_absent(tmp_path): + program = """ +import sys +import RiskLabAI + +module = RiskLabAI.causal_factor_analysis +required = { + 'confounder_undercontrolled_coefficient', + 'confounder_factor_return', + 'confounder_forecast_return', + 'collider_overcontrolled_coefficients', + 'collider_factor_return', + 'collider_forecast_return', + 'collider_model_diagnostics', +} +missing = required.difference(module.__all__) +if missing: + raise SystemExit(f'missing public names: {sorted(missing)}') +for forbidden in ('collider_shift_return', 'collider_p_value'): + if hasattr(module, forbidden): + raise SystemExit(f'source-conflicted or unsupported API exposed: {forbidden}') + +blocked = { + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +} +loaded = blocked.intersection(sys.modules) +if loaded: + raise SystemExit(f'unexpected imports: {sorted(loaded)}') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_graph_identification.py b/test/causal_factor_analysis/test_graph_identification.py new file mode 100644 index 0000000..b8fafd5 --- /dev/null +++ b/test/causal_factor_analysis/test_graph_identification.py @@ -0,0 +1,962 @@ +"""Source-locked tests for clean graphical identification evidence.""" + +from __future__ import annotations + +import ast +import itertools +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from RiskLabAI.causal_factor_analysis.graph_identification import ( + BackdoorAdjustmentEvidence, + CausalDAG, + DSeparationEvidence, + FrontdoorAdjustmentEvidence, + InstrumentEvidence, + NodeRoleEvidence, + PathEvidence, + causal_role_evidence, + check_backdoor_adjustment_set, + check_frontdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) + +PUBLIC_NAMES = { + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", +} + + +def _dag(edges, *, observed=None, extra=()): + nodes = sorted(set(extra) | {node for edge in edges for node in edge}) + if observed is None: + observed = nodes + return CausalDAG(nodes, edges, observed) + + +@pytest.mark.parametrize( + "conditioned,separated", + [((), False), (("U",), True)], +) +def test_fork_d_separation(conditioned, separated): + graph = _dag((("U", "X"), ("U", "Y"))) + evidence = d_separation(graph, ("X",), ("Y",), conditioned) + assert evidence.separated is separated + assert evidence.paths[0].nodes == ("X", "U", "Y") + + +@pytest.mark.parametrize( + "conditioned,separated", + [((), False), (("M",), True)], +) +def test_chain_d_separation(conditioned, separated): + graph = _dag((("X", "M"), ("M", "Y"))) + assert d_separation(graph, ("X",), ("Y",), conditioned).separated is separated + + +def test_collider_and_conditioned_descendant_reverse_path_status(): + graph = _dag((("X", "C"), ("Y", "C"), ("C", "D"))) + closed = d_separation(graph, ("X",), ("Y",)) + opened_at_collider = d_separation(graph, ("X",), ("Y",), ("C",)) + opened_at_descendant = d_separation(graph, ("X",), ("Y",), ("D",)) + + assert closed.separated + assert closed.paths[0].blocking_colliders == ("C",) + assert not opened_at_collider.separated + assert opened_at_collider.paths[0].activated_colliders == ("C",) + assert not opened_at_descendant.separated + assert opened_at_descendant.paths[0].activated_colliders == ("C",) + + +def test_collider_is_path_relative_not_a_global_node_label(): + graph = _dag((("A", "X"), ("A", "C"), ("B", "C"), ("B", "Y"))) + empty = d_separation(graph, ("X",), ("Y",)) + selected = d_separation(graph, ("X",), ("Y",), ("C",)) + roles = causal_role_evidence(graph, "X", "Y", "C") + + assert empty.separated + assert not selected.separated + assert roles.collider_on_paths == (("X", "A", "C", "B", "Y"),) + assert roles.backdoor_noncollider_on_paths == () + + +def test_d_separation_is_symmetric_with_oriented_path_evidence(): + graph = _dag((("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y"))) + forward = d_separation(graph, ("X",), ("Y",), ("U", "M")) + reverse = d_separation(graph, ("Y",), ("X",), ("U", "M")) + assert forward.separated == reverse.separated + assert {path.nodes for path in forward.paths} == { + tuple(reversed(path.nodes)) for path in reverse.paths + } + + +@pytest.mark.parametrize( + "edges,expected", + [ + (("X", "Y"), ((),)), + (("Y", "X"), ()), + ((), ((),)), + ], +) +def test_empty_or_missing_backdoor_sets_are_distinguished(edges, expected): + graph = _dag( + tuple((edges[index], edges[index + 1]) for index in range(0, len(edges), 2)), + extra=("X", "Y"), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == expected + + +@pytest.mark.parametrize( + "observed,expected", + [ + (("U", "X", "Y", "Z"), (("U",), ("Z",))), + (("X", "Y", "Z"), (("Z",),)), + (("U", "X", "Y"), (("U",),)), + (("X", "Y"), ()), + ], +) +def test_backdoor_search_respects_observability(observed, expected): + graph = _dag( + (("U", "Z"), ("Z", "X"), ("U", "Y"), ("X", "Y")), + observed=observed, + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == expected + + +def test_backdoor_returns_all_inclusion_minimal_sets_not_only_smallest(): + graph = _dag( + ( + ("A", "X"), + ("A", "B"), + ("B", "Y"), + ("A", "C"), + ("C", "Y"), + ("X", "Y"), + ) + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == ( + ("A",), + ("B", "C"), + ) + + +def test_indegree_two_node_can_be_the_required_backdoor_blocker(): + graph = _dag((("A", "C"), ("B", "C"), ("C", "X"), ("C", "Y"), ("X", "Y"))) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("C",),) + + +def test_conditioning_on_a_collider_can_invalidate_an_empty_backdoor_set(): + graph = _dag((("A", "X"), ("A", "C"), ("B", "C"), ("B", "Y"), ("X", "Y"))) + assert check_backdoor_adjustment_set(graph, "X", "Y", ()).admissible + selected = check_backdoor_adjustment_set(graph, "X", "Y", ("C",)) + assert not selected.admissible + assert selected.open_backdoor_paths + + +def test_descendant_control_is_reported_and_rejected(): + graph = _dag((("X", "Y"), ("X", "D"))) + evidence = check_backdoor_adjustment_set(graph, "X", "Y", ("D",)) + assert evidence.descendants_in_set == ("D",) + assert not evidence.admissible + + +def test_classic_frontdoor_set_is_graphically_admissible_but_needs_positivity(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "M", "Y"), + ) + evidence = check_frontdoor_adjustment_set(graph, "X", "Y", ("M",)) + assert evidence.graphically_admissible + assert evidence.positivity_required + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == () + + +def test_frontdoor_checker_accepts_redundant_superset_but_search_removes_it(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "M", "N", "Y"), + extra=("N",), + ) + assert check_frontdoor_adjustment_set( + graph, "X", "Y", ("M", "N") + ).graphically_admissible + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + + +@pytest.mark.parametrize( + "extra_edge,failed_field", + [ + (("X", "Y"), "unintercepted_directed_paths"), + (("V", "M"), "open_treatment_mediator_backdoor_paths"), + (("W", "Y"), "open_mediator_outcome_backdoor_paths_given_treatment"), + ], +) +def test_each_frontdoor_criterion_fails_independently(extra_edge, failed_field): + edges = [("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")] + if extra_edge == ("V", "M"): + edges.append(("V", "X")) + if extra_edge == ("W", "Y"): + edges.append(("W", "M")) + edges.append(extra_edge) + observed = tuple( + node for node in {item for edge in edges for item in edge} if node != "U" + ) + graph = _dag(tuple(edges), observed=observed) + evidence = check_frontdoor_adjustment_set(graph, "X", "Y", ("M",)) + assert not evidence.graphically_admissible + assert getattr(evidence, failed_field) + + +def test_parallel_and_serial_mediator_sets_have_correct_minimality(): + parallel = _dag( + ( + ("U", "X"), + ("U", "Y"), + ("X", "M1"), + ("M1", "Y"), + ("X", "M2"), + ("M2", "Y"), + ), + observed=("X", "M1", "M2", "Y"), + ) + serial = _dag( + (("U", "X"), ("U", "Y"), ("X", "M1"), ("M1", "M2"), ("M2", "Y")), + observed=("X", "M1", "M2", "Y"), + ) + assert minimal_frontdoor_adjustment_sets(parallel, "X", "Y") == (("M1", "M2"),) + assert minimal_frontdoor_adjustment_sets(serial, "X", "Y") == ( + ("M1",), + ("M2",), + ) + + +def test_latent_or_irrelevant_frontdoor_set_is_not_returned(): + hidden = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "Y"), + ) + unrelated = _dag((), observed=("X", "Y", "M"), extra=("X", "Y", "M")) + assert not check_frontdoor_adjustment_set( + hidden, "X", "Y", ("M",) + ).graphically_admissible + assert minimal_frontdoor_adjustment_sets(hidden, "X", "Y") == () + assert minimal_frontdoor_adjustment_sets(unrelated, "X", "Y") == () + + +def test_pearl_and_cfi_instrument_profiles_are_not_collapsed(): + graph = _dag((("U", "Z"), ("U", "X"), ("X", "Y"))) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_relevance + assert evidence.pearl_exclusion_exogeneity + assert evidence.pearl_graphical + assert not evidence.cfi_direct_relevance + assert not evidence.cfi_simple + + +def test_simple_direct_instrument_satisfies_both_profiles(): + graph = _dag((("Z", "X"), ("X", "Y"))) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_graphical + assert evidence.cfi_direct_relevance + assert evidence.cfi_full_mediation + assert evidence.cfi_exogeneity + assert evidence.cfi_simple + + +def test_cfi_simple_profile_is_unconditional_when_pearl_controls_are_supplied(): + graph = _dag((("Z", "X"), ("X", "Y")), extra=("W",)) + evidence = check_instrument(graph, "Z", "X", "Y", ("W",)) + assert evidence.pearl_graphical + assert evidence.cfi_simple + + +def test_conditional_instrument_blocks_observed_common_cause_for_pearl_only(): + graph = _dag((("W", "Z"), ("W", "Y"), ("Z", "X"), ("X", "Y"))) + unconditional = check_instrument(graph, "Z", "X", "Y") + conditional = check_instrument(graph, "Z", "X", "Y", ("W",)) + assert not unconditional.pearl_graphical + assert conditional.pearl_graphical + assert not conditional.cfi_simple + + +@pytest.mark.parametrize( + "edges,conditioned,failed", + [ + ((("Z", "X"), ("X", "Y"), ("Z", "Y")), (), "exclusion"), + ((("U", "Z"), ("U", "Y"), ("Z", "X"), ("X", "Y")), (), "exogeneity"), + ( + (("Z", "C"), ("U", "C"), ("U", "Y"), ("Z", "X"), ("X", "Y")), + ("C",), + "collider", + ), + ], +) +def test_instrument_exclusion_and_exogeneity_failures(edges, conditioned, failed): + graph = _dag(edges) + evidence = check_instrument(graph, "Z", "X", "Y", conditioned) + assert not evidence.pearl_graphical, failed + + +def test_treatment_descendant_conditioning_invalidates_pearl_profile(): + graph = _dag((("Z", "X"), ("X", "Y"), ("X", "D"))) + evidence = check_instrument(graph, "Z", "X", "Y", ("D",)) + assert not evidence.conditioning_unaffected_by_treatment + assert not evidence.pearl_graphical + + +def test_post_treatment_proxy_is_not_a_pearl_instrument(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "Y"), ("X", "Z")), + observed=("X", "Y", "Z"), + ) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_relevance + assert not evidence.pearl_exclusion_exogeneity + assert not evidence.pearl_graphical + + +def test_node_role_evidence_is_nonexclusive_and_path_witnessed(): + graph = _dag( + ( + ("X", "M"), + ("M", "Y"), + ("A", "M"), + ("A", "X"), + ("B", "M"), + ("B", "Y"), + ) + ) + evidence = causal_role_evidence(graph, "X", "Y", "M") + assert evidence.mediator_on_directed_paths + assert evidence.collider_on_paths + assert evidence.noncollider_on_paths + assert evidence.descendant_of_treatment + assert evidence.ancestor_of_outcome + + +def test_common_cause_and_remote_backdoor_witnesses_are_distinct(): + graph = _dag((("U", "Z"), ("Z", "X"), ("U", "Y"), ("X", "Y"))) + upstream = causal_role_evidence(graph, "X", "Y", "U") + downstream = causal_role_evidence(graph, "X", "Y", "Z") + assert upstream.common_cause_paths + assert upstream.backdoor_noncollider_on_paths + assert downstream.backdoor_noncollider_on_paths + assert downstream.common_cause_paths == () + + +def test_common_cause_witness_requires_a_pathwise_directed_fork(): + graph = _dag((("U", "X"), ("U", "Y"), ("X", "D"), ("U", "A"), ("A", "D"))) + evidence = causal_role_evidence(graph, "X", "Y", "U") + assert evidence.common_cause_paths == (("X", "U", "Y"),) + assert ("X", "D", "A", "U", "Y") in evidence.noncollider_on_paths + + +def test_causal_dag_normalizes_without_mutating_callers_and_is_frozen(): + nodes = ["Y", "X", "U"] + edges = [["U", "Y"], ["U", "X"]] + observed = ["Y", "X"] + graph = CausalDAG(nodes, edges, observed) + nodes.append("LATER") + edges[0][0] = "CHANGED" + observed.clear() + assert graph.nodes == ("U", "X", "Y") + assert graph.directed_edges == (("U", "X"), ("U", "Y")) + assert graph.observed_nodes == ("X", "Y") + assert not hasattr(graph, "__dict__") + with pytest.raises(AttributeError): + graph.nodes = () + + +def test_causal_dag_namedtuple_helpers_cannot_bypass_validation(): + graph = _dag((("X", "Y"),)) + with pytest.raises(ValueError): + graph._replace(nodes=("ONLY",)) + with pytest.raises(ValueError): + CausalDAG._make((("X", "Y"), (("X", "Y"), ("Y", "X")), ("X", "Y"))) + with pytest.raises(TypeError): + CausalDAG._make((graph.nodes, graph.directed_edges)) + + +@pytest.mark.parametrize( + "args,error", + [ + (("XYZ", (), ()), TypeError), + ((("X", 1), (), ("X",)), TypeError), + ((("X", ""), (), ("X",)), ValueError), + ((("X", "X"), (), ("X",)), ValueError), + ((("X", "Y"), (("X",),), ("X", "Y")), ValueError), + ((("X", "Y"), ({"X", "Y"},), ("X", "Y")), TypeError), + ((("X", "Y"), (("X", "Z"),), ("X", "Y")), ValueError), + ((("X", "Y"), (("X", "X"),), ("X", "Y")), ValueError), + ((("X", "Y"), (("X", "Y"), ("Y", "X")), ("X", "Y")), ValueError), + ], +) +def test_malformed_dag_fails_closed(args, error): + with pytest.raises(error): + CausalDAG(*args) + + +def test_query_validation_fails_closed_without_scalar_string_iteration(): + graph = _dag((("X", "Y"),), extra=("Z",)) + with pytest.raises(TypeError): + d_separation(graph, "X", ("Y",)) + with pytest.raises(ValueError): + d_separation(graph, ("X",), ("Y",), ("UNKNOWN",)) + with pytest.raises(ValueError): + d_separation(graph, ("X",), ("Y",), ("X",)) + with pytest.raises(ValueError): + check_backdoor_adjustment_set(graph, "X", "X") + with pytest.raises(ValueError): + check_instrument(graph, "X", "X", "Y") + + +def test_known_latent_and_endpoint_sets_are_substantive_negatives(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), observed=("X", "Y", "M") + ) + latent = check_backdoor_adjustment_set(graph, "X", "Y", ("U",)) + endpoint = check_backdoor_adjustment_set(graph, "X", "Y", ("X",)) + frontdoor = check_frontdoor_adjustment_set(graph, "X", "Y", ("U",)) + assert not latent.all_observed and not latent.admissible + assert endpoint.endpoints_in_set == ("X",) and not endpoint.admissible + assert not frontdoor.all_observed and not frontdoor.graphically_admissible + + +def _oracle_children(nodes, edges): + result = {node: set() for node in nodes} + for source, target in edges: + result[source].add(target) + return result + + +def _oracle_descendants(nodes, edges, start): + children = _oracle_children(nodes, edges) + result = set() + pending = list(children[start]) + while pending: + node = pending.pop() + if node not in result: + result.add(node) + pending.extend(children[node]) + return result + + +def _oracle_simple_paths(nodes, edges, start, finish): + adjacency = {node: set() for node in nodes} + for source, target in edges: + adjacency[source].add(target) + adjacency[target].add(source) + result = [] + + def visit(node, path): + if node == finish: + result.append(tuple(path)) + return + for neighbor in sorted(adjacency[node]): + if neighbor not in path: + visit(neighbor, path + [neighbor]) + + visit(start, [start]) + return result + + +def _oracle_path_open(nodes, edges, path, conditioned): + edge_set = set(edges) + conditioned = set(conditioned) + for index in range(1, len(path) - 1): + previous, node, following = path[index - 1 : index + 2] + collider = (previous, node) in edge_set and (following, node) in edge_set + if collider: + family = {node} | _oracle_descendants(nodes, edges, node) + if not family.intersection(conditioned): + return False + elif node in conditioned: + return False + return True + + +def _oracle_d_separated(nodes, edges, left, right, conditioned): + return not any( + _oracle_path_open(nodes, edges, path, conditioned) + for path in _oracle_simple_paths(nodes, edges, left, right) + ) + + +def _oracle_directed_paths(nodes, edges, start, finish): + edge_set = set(edges) + return tuple( + sorted( + ( + path + for path in _oracle_simple_paths(nodes, edges, start, finish) + if all( + (path[index], path[index + 1]) in edge_set + for index in range(len(path) - 1) + ) + ), + key=lambda path: (len(path), path), + ) + ) + + +def _oracle_frontdoor_admissible(nodes, edges, treatment, outcome, mediators): + mediators = tuple(sorted(mediators)) + directed = _oracle_directed_paths(nodes, edges, treatment, outcome) + if not mediators or not directed: + return False + if any(not set(path[1:-1]).intersection(mediators) for path in directed): + return False + treatment_graph = tuple(edge for edge in edges if edge[0] != treatment) + if any( + not _oracle_d_separated(nodes, treatment_graph, treatment, node, ()) + for node in mediators + ): + return False + mediator_graph = tuple(edge for edge in edges if edge[0] not in mediators) + return all( + _oracle_d_separated(nodes, mediator_graph, node, outcome, (treatment,)) + for node in mediators + ) + + +def _oracle_role(nodes, edges, treatment, outcome, node): + edge_set = set(edges) + paths = tuple( + sorted( + _oracle_simple_paths(nodes, edges, treatment, outcome), + key=lambda path: (len(path), path), + ) + ) + colliders = [] + noncolliders = [] + mediators = [] + backdoors = [] + common_causes = [] + for path in paths: + index = path.index(node) if node in path[1:-1] else None + if index is None: + continue + previous, following = path[index - 1], path[index + 1] + is_collider = (previous, node) in edge_set and (following, node) in edge_set + if is_collider: + colliders.append(path) + else: + noncolliders.append(path) + if all( + (path[position], path[position + 1]) in edge_set + for position in range(len(path) - 1) + ): + mediators.append(path) + if (path[1], treatment) in edge_set and not is_collider: + backdoors.append(path) + directed_to_treatment = all( + (path[position], path[position - 1]) in edge_set + for position in range(index, 0, -1) + ) + directed_to_outcome = all( + (path[position], path[position + 1]) in edge_set + for position in range(index, len(path) - 1) + ) + if not is_collider and directed_to_treatment and directed_to_outcome: + common_causes.append(path) + return { + "collider_on_paths": tuple(colliders), + "noncollider_on_paths": tuple(noncolliders), + "mediator_on_directed_paths": tuple(mediators), + "backdoor_noncollider_on_paths": tuple(backdoors), + "common_cause_paths": tuple(common_causes), + "descendant_of_treatment": node in _oracle_descendants(nodes, edges, treatment), + "ancestor_of_outcome": outcome in _oracle_descendants(nodes, edges, node), + } + + +def _oracle_is_acyclic(nodes, edges): + descendants = _oracle_children(nodes, edges) + indegree = {node: 0 for node in nodes} + for _, target in edges: + indegree[target] += 1 + pending = [node for node in nodes if indegree[node] == 0] + visited = 0 + while pending: + node = pending.pop() + visited += 1 + for child in descendants[node]: + indegree[child] -= 1 + if indegree[child] == 0: + pending.append(child) + return visited == len(nodes) + + +def _all_four_node_dags(): + nodes = ("A", "B", "C", "D") + pairs = tuple(itertools.combinations(nodes, 2)) + for states in itertools.product((0, 1, 2), repeat=len(pairs)): + edges = [] + for state, (first, second) in zip(states, pairs): + if state == 1: + edges.append((first, second)) + elif state == 2: + edges.append((second, first)) + edges = tuple(edges) + if _oracle_is_acyclic(nodes, edges): + yield nodes, edges + + +def test_all_543_four_node_dags_match_independent_d_separation_oracle(): + count = 0 + for nodes, edges in _all_four_node_dags(): + count += 1 + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for left, right in itertools.permutations(nodes, 2): + remaining = tuple(node for node in nodes if node not in {left, right}) + for size in range(len(remaining) + 1): + for conditioned in itertools.combinations(remaining, size): + expected = _oracle_d_separated( + nodes, edges, left, right, conditioned + ) + actual = d_separation( + graph, (left,), (right,), conditioned + ).separated + assert actual is expected + assert count == 543 + + +def test_all_four_node_backdoor_sets_match_independent_powerset_oracle(): + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, edges, nodes) + for treatment, outcome in itertools.permutations(nodes, 2): + descendants = _oracle_descendants(nodes, edges, treatment) + candidates = tuple( + node + for node in nodes + if node not in {treatment, outcome} and node not in descendants + ) + mutilated = tuple(edge for edge in edges if edge[0] != treatment) + valid = [] + for size in range(len(candidates) + 1): + for candidate in itertools.combinations(candidates, size): + if _oracle_d_separated( + nodes, mutilated, treatment, outcome, candidate + ): + valid.append(candidate) + expected = tuple( + candidate + for candidate in sorted(valid, key=lambda item: (len(item), item)) + if not any(set(other) < set(candidate) for other in valid) + ) + assert ( + minimal_backdoor_adjustment_sets(graph, treatment, outcome) == expected + ) + + +def test_all_four_node_frontdoor_sets_match_independent_oracle(): + queries = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for treatment, outcome in itertools.permutations(nodes, 2): + queries += 1 + candidates = tuple( + node for node in nodes if node not in {treatment, outcome} + ) + valid = [] + for size in range(1, len(candidates) + 1): + for mediators in itertools.combinations(candidates, size): + expected = _oracle_frontdoor_admissible( + nodes, edges, treatment, outcome, mediators + ) + actual = check_frontdoor_adjustment_set( + graph, treatment, outcome, mediators + ).graphically_admissible + assert actual is expected + if expected: + valid.append(mediators) + expected_minimal = tuple( + candidate + for candidate in sorted(valid, key=lambda item: (len(item), item)) + if not any(set(other) < set(candidate) for other in valid) + ) + assert ( + minimal_frontdoor_adjustment_sets(graph, treatment, outcome) + == expected_minimal + ) + assert queries == 6_516 + + +def test_all_four_node_instrument_profiles_match_independent_oracles(): + assessments = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + edge_set = set(edges) + for treatment, outcome in itertools.permutations(nodes, 2): + for instrument in ( + node for node in nodes if node not in {treatment, outcome} + ): + control = next( + node + for node in nodes + if node not in {instrument, treatment, outcome} + ) + for conditioned in ((), (control,)): + assessments += 1 + unaffected = not set(conditioned).intersection( + _oracle_descendants(nodes, edges, treatment) + ) + pearl_relevance = not _oracle_d_separated( + nodes, edges, instrument, treatment, conditioned + ) + intervention_graph = tuple( + edge for edge in edges if edge[1] != treatment + ) + pearl_exclusion = _oracle_d_separated( + nodes, + intervention_graph, + instrument, + outcome, + conditioned, + ) + directed = _oracle_directed_paths(nodes, edges, instrument, outcome) + cfi_direct = (instrument, treatment) in edge_set + cfi_full_mediation = bool(directed) and all( + treatment in path[1:-1] for path in directed + ) + instrument_graph = tuple( + edge for edge in edges if edge[0] != instrument + ) + cfi_exogeneity = _oracle_d_separated( + nodes, instrument_graph, instrument, outcome, () + ) + evidence = check_instrument( + graph, + instrument, + treatment, + outcome, + conditioned, + ) + assert evidence.conditioning_unaffected_by_treatment is unaffected + assert evidence.pearl_relevance is pearl_relevance + assert evidence.pearl_exclusion_exogeneity is pearl_exclusion + assert evidence.pearl_graphical is ( + unaffected and pearl_relevance and pearl_exclusion + ) + assert evidence.cfi_direct_relevance is cfi_direct + assert evidence.cfi_full_mediation is cfi_full_mediation + assert evidence.cfi_exogeneity is cfi_exogeneity + assert evidence.cfi_simple is ( + cfi_direct and cfi_full_mediation and cfi_exogeneity + ) + assert assessments == 26_064 + + +def test_all_four_node_role_evidence_matches_path_oracle(): + assessments = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for treatment, outcome in itertools.permutations(nodes, 2): + for node in nodes: + if node in {treatment, outcome}: + continue + assessments += 1 + expected = _oracle_role(nodes, edges, treatment, outcome, node) + actual = causal_role_evidence(graph, treatment, outcome, node) + for field, value in expected.items(): + assert getattr(actual, field) == value + assert assessments == 13_032 + + +def test_complete_path_evidence_fails_closed_at_explicit_budget(): + graph = _dag((("X", "A"), ("A", "Y"), ("X", "B"), ("B", "Y"))) + assert len(d_separation(graph, ("X",), ("Y",), max_paths=2).paths) == 2 + with pytest.raises(ValueError, match="max_paths"): + d_separation(graph, ("X",), ("Y",), max_paths=1) + with pytest.raises(ValueError, match="max_paths"): + causal_role_evidence(graph, "X", "Y", "A", max_paths=1) + + +def test_path_state_budget_bounds_dense_dead_end_search(): + lobe = tuple(f"A{index:02d}" for index in range(6)) + edges = [("X", "Y")] + edges.extend(("X", node) for node in lobe) + edges.extend( + (left, right) for index, left in enumerate(lobe) for right in lobe[index + 1 :] + ) + graph = _dag(tuple(edges)) + with pytest.raises(ValueError, match="max_path_states"): + d_separation( + graph, + ("X",), + ("Y",), + max_paths=1, + max_path_states=50, + ) + evidence = check_frontdoor_adjustment_set( + graph, + "X", + "Y", + (), + max_paths=1, + max_path_states=2, + ) + assert not evidence.graphically_admissible + + +def test_minimal_search_budget_fails_closed_without_partial_results(): + graph = _dag((("Y", "X"),), extra=("A", "B")) + with pytest.raises(ValueError, match="max_candidates"): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=1) + + frontdoor_graph = _dag((("X", "Y"),), extra=("A", "B")) + with pytest.raises(ValueError, match="max_candidates"): + minimal_frontdoor_adjustment_sets(frontdoor_graph, "X", "Y", max_candidates=1) + + +def test_large_candidate_space_hits_budget_without_recursion_failure(): + irrelevant = tuple(f"N{index:04d}" for index in range(1_100)) + graph = _dag((("Y", "X"),), extra=irrelevant) + with pytest.raises(ValueError, match="max_candidates"): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=2_000) + + +def test_minimal_search_prunes_supersets_of_many_singleton_solutions(): + chain = tuple(f"A{index:02d}" for index in range(20)) + edges = [(chain[index], chain[index - 1]) for index in range(1, 20)] + edges.extend(((chain[0], "X"), (chain[-1], "Y"))) + graph = _dag(tuple(edges)) + assert minimal_backdoor_adjustment_sets( + graph, "X", "Y", max_candidates=100 + ) == tuple((node,) for node in chain) + + +def test_empty_backdoor_set_short_circuits_candidate_search_exactly(): + irrelevant = tuple(f"N{index:02d}" for index in range(20)) + graph = _dag((("X", "Y"),), extra=irrelevant) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=1) == ((),) + + +@pytest.mark.parametrize("value,error", [(True, TypeError), (0, ValueError)]) +def test_resource_budgets_require_positive_integers_or_none(value, error): + graph = _dag((("X", "Y"),), extra=("Z",)) + with pytest.raises(error): + d_separation(graph, ("X",), ("Y",), max_paths=value) + with pytest.raises(error): + d_separation(graph, ("X",), ("Y",), max_path_states=value) + with pytest.raises(error): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=value) + + +def test_protocol_stage_two_graph_tuples_construct_the_same_dag(): + from RiskLabAI.causal_factor_analysis.protocol import CausalDiscoveryStage + + stage = CausalDiscoveryStage( + method_labels=("PC",), + graph_id="resolved-dag-v1", + graph_nodes=("Y", "X", "U"), + directed_edges=(("U", "Y"), ("X", "Y"), ("U", "X")), + graph_kind="DAG", + ambiguous_edges=(), + assumptions=("Graph accepted after external causal review.",), + ) + graph = CausalDAG( + stage.graph_nodes, + stage.directed_edges, + stage.graph_nodes, + ) + assert graph.nodes == ("U", "X", "Y") + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("U",),) + + +def test_public_records_and_exports_are_exact_and_frozen(): + import RiskLabAI.causal_factor_analysis.graph_identification as module + + assert set(module.__all__) == PUBLIC_NAMES + graph = _dag((("X", "Y"),), extra=("Z",)) + records = ( + d_separation(graph, ("X",), ("Y",)), + check_backdoor_adjustment_set(graph, "X", "Y"), + check_frontdoor_adjustment_set(graph, "X", "Y", ("Z",)), + check_instrument(_dag((("Z", "X"), ("X", "Y"))), "Z", "X", "Y"), + causal_role_evidence(_dag((("X", "Z"), ("Z", "Y"))), "X", "Y", "Z"), + ) + assert isinstance(records[0], DSeparationEvidence) + assert isinstance(records[0].paths[0], PathEvidence) + assert isinstance(records[1], BackdoorAdjustmentEvidence) + assert isinstance(records[2], FrontdoorAdjustmentEvidence) + assert isinstance(records[3], InstrumentEvidence) + assert isinstance(records[4], NodeRoleEvidence) + for record in records: + assert not hasattr(record, "__dict__") + with pytest.raises(AttributeError): + records[0].separated = False + + +def test_module_is_python39_and_standard_library_only(): + module_path = ( + Path(__file__).parents[2] + / "src" + / "RiskLabAI" + / "causal_factor_analysis" + / "graph_identification.py" + ) + source = module_path.read_text(encoding="utf-8") + tree = ast.parse(source, feature_version=(3, 9)) + imported_roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported_roots.add(node.module.split(".")[0]) + assert imported_roots <= { + "__future__", + "collections", + "dataclasses", + "itertools", + "typing", + } + + +def test_package_import_does_not_load_legacy_or_private_namespaces(tmp_path): + public_names = repr(sorted(PUBLIC_NAMES)) + program = f""" +import sys +import RiskLabAI.causal_factor_analysis as package +import RiskLabAI.causal_factor_analysis.graph_identification as module +expected = set({public_names}) +assert expected <= set(package.__all__) +assert expected == set(module.__all__) +blocked = ( + 'RiskLabAI.causal', + 'RiskLabAI.backtest', + 'RiskLabAI.optimization', + 'RiskLabAI.risk_control', +) +assert not any(name == root or name.startswith(root + '.') for name in sys.modules for root in blocked) +""" + environment = os.environ.copy() + environment["PYTHONDONTWRITEBYTECODE"] = "1" + subprocess.run( + [sys.executable, "-B", "-c", program], + cwd=tmp_path, + env=environment, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_minimum_variance.py b/test/causal_factor_analysis/test_minimum_variance.py new file mode 100644 index 0000000..454d775 --- /dev/null +++ b/test/causal_factor_analysis/test_minimum_variance.py @@ -0,0 +1,450 @@ +"""Source-locked tests for target-exposure minimum-variance allocation.""" + +from __future__ import annotations + +import subprocess +import sys +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import minimum_variance_factor_weights + + +def _assert_factor_constraints_within_roundoff(exposures, weights, targets): + """Check componentwise residuals against a float64 backward-error bound.""" + constraint_matrix = exposures.T + achieved = constraint_matrix @ weights + residuals = np.abs(achieved - targets) + roundoff_scales = np.abs(constraint_matrix) @ np.abs(weights) + np.abs(targets) + tolerances = ( + 8.0 * constraint_matrix.shape[1] * np.finfo(np.float64).eps * roundoff_scales + ) + np.testing.assert_array_less(residuals, np.nextafter(tolerances, np.inf)) + + +def test_eq4_solution_satisfies_exposure_and_kkt_conditions(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array( + [ + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + ] + ) + targets = np.array([0.0, 1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + expected_weights = np.array([-2.0, 5.0, 2.0]) / 7.0 + expected_lagrange = np.array([4.0, -20.0]) / 7.0 + np.testing.assert_allclose(weights, expected_weights, rtol=0.0, atol=1e-14) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-14) + np.testing.assert_allclose( + 2.0 * covariance @ weights + exposures @ expected_lagrange, + np.zeros(3), + rtol=0.0, + atol=1e-14, + ) + assert weights @ covariance @ weights == pytest.approx( + 10.0 / 7.0, rel=1e-14, abs=1e-14 + ) + + +def test_eq4_solution_is_strict_minimum_over_feasible_perturbations(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array( + [ + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + ] + ) + targets = np.array([0.0, 1.0]) + weights = minimum_variance_factor_weights(covariance, exposures, targets) + null_direction = np.array([-1.0, -1.0, 1.0]) + np.testing.assert_allclose( + exposures.T @ null_direction, np.zeros(2), rtol=0.0, atol=0.0 + ) + optimum = weights @ covariance @ weights + + for scale in (-2.0, -0.5, 0.5, 2.0): + candidate = weights + scale * null_direction + np.testing.assert_allclose( + exposures.T @ candidate, targets, rtol=0.0, atol=1e-14 + ) + assert candidate @ covariance @ candidate > optimum + + +def test_eq5_square_solution_is_covariance_independent(): + exposures = np.array([[1.0, 1.0], [1.5, 1.0]]) + targets = np.array([0.0, 1.0]) + expected = np.linalg.solve(exposures.T, targets) + covariance_matrices = ( + np.array([[1.0, 0.2], [0.2, 2.0]]), + np.array([[5.0, -1.0], [-1.0, 0.5]]), + ) + + np.testing.assert_allclose(expected, np.array([3.0, -2.0]), rtol=0.0, atol=1e-14) + for covariance in covariance_matrices: + weights = minimum_variance_factor_weights(covariance, exposures, targets) + np.testing.assert_allclose(weights, expected, rtol=0.0, atol=1e-13) + + +def test_eq5_square_solution_does_not_use_qr(monkeypatch): + def fail_if_called(*_args, **_kwargs): + raise AssertionError("square exposure systems must use the Equation (5) solve") + + monkeypatch.setattr(np.linalg, "qr", fail_if_called) + exposures = np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = np.array([1.0, 0.0]) + + weights = minimum_variance_factor_weights(np.eye(2), exposures, targets) + + np.testing.assert_allclose( + weights, np.linalg.solve(exposures.T, targets), rtol=5e-8, atol=0.0 + ) + _assert_factor_constraints_within_roundoff(exposures, weights, targets) + + +def test_eq6_single_factor_solution(): + covariance = np.array([[2.0, 0.5], [0.5, 1.0]]) + exposures = np.array([[1.0], [1.0]]) + targets = np.array([1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([0.25, 0.75]), rtol=0.0, atol=1e-14) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-14) + assert weights @ covariance @ weights == pytest.approx( + 7.0 / 8.0, rel=1e-14, abs=1e-14 + ) + + +def test_one_asset_one_factor_solution(): + weights = minimum_variance_factor_weights( + np.array([[2.5]]), np.array([[-4.0]]), np.array([3.0]) + ) + np.testing.assert_allclose(weights, np.array([-0.75]), rtol=0.0, atol=0.0) + + +def test_near_collinear_square_constraints_remain_feasible(): + covariance = np.eye(2) + exposures = np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = np.array([1.0, 0.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + direct_solution = np.linalg.solve(exposures.T, targets) + + np.testing.assert_allclose(weights, direct_solution, rtol=5e-8, atol=0.0) + _assert_factor_constraints_within_roundoff(exposures, weights, targets) + + +def test_extreme_finite_exposure_scale_does_not_overflow(): + covariance = np.array([[1.0]]) + exposures = np.array([[1e308]]) + targets = np.array([1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([1e-308]), rtol=1e-15, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_extreme_factor_unit_differences_do_not_create_false_rank_deficiency(): + covariance = np.eye(2) + exposures = np.diag([1e-200, 1e200]) + targets = np.array([1e-200, 1e200]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.ones(2), rtol=0.0, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_feasible_target_normalization_does_not_overflow(): + covariance = np.eye(2) + exposures = np.array([[1e-308], [1e-308]]) + targets = np.array([2.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([1e308, 1e308]), rtol=1e-15, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_unrepresentable_nonzero_scaled_target_is_rejected(): + smallest_positive = np.nextafter(0.0, 1.0) + with pytest.raises(ValueError, match="represented.*numerical precision"): + minimum_variance_factor_weights( + np.array([[1.0]]), + np.array([[1e308]]), + np.array([smallest_positive]), + ) + + +def test_representable_tiny_target_is_not_treated_as_zero(): + weights = minimum_variance_factor_weights( + np.array([[1.0]]), np.array([[1.0]]), np.array([1e-20]) + ) + np.testing.assert_allclose(weights, np.array([1e-20]), rtol=1e-15, atol=0.0) + + +@pytest.mark.parametrize("factor_scale", [1.0, 1e20]) +def test_numerically_unresolved_constraints_are_rejected_independently_of_units( + factor_scale, +): + exposures = factor_scale * np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = factor_scale * np.array([1e-20, 0.0]) + + with pytest.raises(ValueError, match="resolved.*numerical precision"): + minimum_variance_factor_weights(np.eye(2), exposures, targets) + + +def test_extreme_finite_covariance_scale_does_not_overflow_when_symmetrized(): + weights = minimum_variance_factor_weights( + np.array([[1e308]]), np.array([[1.0]]), np.array([1.0]) + ) + np.testing.assert_allclose(weights, np.array([1.0]), rtol=0.0, atol=0.0) + + +def test_smallest_positive_covariance_is_normalized_before_symmetrization(): + smallest_positive = np.nextafter(0.0, 1.0) + weights = minimum_variance_factor_weights( + np.array([[smallest_positive]]), np.array([[1.0]]), np.array([1.0]) + ) + np.testing.assert_allclose(weights, np.array([1.0]), rtol=0.0, atol=0.0) + + +def test_zero_targets_return_zero_weights(): + covariance = np.diag([1.0, 2.0, 3.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + weights = minimum_variance_factor_weights(covariance, exposures, np.zeros(2)) + np.testing.assert_array_equal(weights, np.zeros(3)) + + +@pytest.mark.parametrize("covariance_scale", [1e-12, 1.0, 1e12]) +def test_covariance_scale_does_not_change_weights(covariance_scale): + covariance = covariance_scale * np.array( + [[2.0, 0.25, 0.1], [0.25, 1.5, -0.2], [0.1, -0.2, 1.0]] + ) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.2, -0.4]) + + baseline = minimum_variance_factor_weights( + covariance / covariance_scale, exposures, targets + ) + scaled = minimum_variance_factor_weights(covariance, exposures, targets) + np.testing.assert_allclose(scaled, baseline, rtol=1e-12, atol=1e-12) + + +@pytest.mark.parametrize("target_scale", [-3.0, 0.25, 4.0]) +def test_weights_are_homogeneous_in_targets(target_scale): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.5, -0.25]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + scaled = minimum_variance_factor_weights( + covariance, exposures, target_scale * targets + ) + np.testing.assert_allclose(scaled, target_scale * baseline, rtol=1e-12, atol=1e-13) + + +def test_factor_unit_changes_leave_weights_unchanged(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.5, -0.25]) + factor_scales = np.array([100.0, 0.01]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + rescaled = minimum_variance_factor_weights( + covariance, + exposures * factor_scales, + targets * factor_scales, + ) + np.testing.assert_allclose(rescaled, baseline, rtol=1e-12, atol=1e-13) + + +def test_asset_and_factor_permutations_are_equivariant(): + covariance = np.array([[2.0, 0.2, 0.1], [0.2, 1.5, -0.1], [0.1, -0.1, 1.0]]) + exposures = np.array([[1.0, 0.0], [0.2, 1.0], [1.0, -0.5]]) + targets = np.array([0.3, -0.2]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + asset_order = np.array([2, 0, 1]) + factor_order = np.array([1, 0]) + permuted = minimum_variance_factor_weights( + covariance[np.ix_(asset_order, asset_order)], + exposures[asset_order][:, factor_order], + targets[factor_order], + ) + restored = np.empty_like(permuted) + restored[asset_order] = permuted + np.testing.assert_allclose(restored, baseline, rtol=1e-12, atol=1e-13) + + +@pytest.mark.parametrize("covariance_scale", [1e-200, 1.0, 1e200]) +def test_tiny_covariance_asymmetry_is_symmetrized(covariance_scale): + covariance = covariance_scale * np.array([[2.0, 0.3 + 5e-13], [0.3 - 5e-13, 1.0]]) + exposures = np.array([[1.0], [1.0]]) + targets = np.array([1.0]) + symmetric = (covariance + covariance.T) / 2.0 + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + expected = minimum_variance_factor_weights(symmetric, exposures, targets) + np.testing.assert_allclose(weights, expected, rtol=0.0, atol=1e-14) + + +@pytest.mark.parametrize("covariance_scale", [1e-200, 1.0, 1e200]) +def test_material_covariance_asymmetry_is_rejected(covariance_scale): + covariance = covariance_scale * np.array([[2.0, 0.3 + 1e-5], [0.3 - 1e-5, 1.0]]) + with pytest.raises(ValueError, match="symmetric"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +def test_asymmetry_is_measured_relative_to_covariance_scale(): + covariance = np.array([[1e-20, 4e-13], [-4e-13, 1e-20]]) + with pytest.raises(ValueError, match="symmetric"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +@pytest.mark.parametrize( + "covariance", + [ + np.array([[1.0, 1.0], [1.0, 1.0]]), + np.array([[1.0, 2.0], [2.0, 1.0]]), + ], +) +def test_non_positive_definite_covariance_is_rejected(covariance): + with pytest.raises(ValueError, match="positive definite"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +def test_rank_deficient_exposures_are_rejected(): + exposures = np.array([[1.0, 2.0], [2.0, 4.0], [3.0, 6.0]]) + with pytest.raises(ValueError, match="full column rank"): + minimum_variance_factor_weights(np.eye(3), exposures, np.array([1.0, 2.0])) + + +@pytest.mark.parametrize( + "covariance, exposures, targets, match", + [ + (np.ones(2), np.ones((2, 1)), np.ones(1), "2-D square"), + (np.ones((2, 3)), np.ones((2, 1)), np.ones(1), "square"), + (np.empty((0, 0)), np.empty((0, 1)), np.ones(1), "non-empty square"), + (np.eye(2), np.ones(2), np.ones(1), "2-D"), + (np.eye(2), np.ones((3, 1)), np.ones(1), "rows"), + (np.eye(2), np.empty((2, 0)), np.empty(0), "at least one"), + (np.eye(2), np.ones((2, 3)), np.ones(3), "more columns than rows"), + (np.eye(2), np.ones((2, 1)), np.array([[1.0]]), "1-D"), + (np.eye(2), np.ones((2, 1)), np.ones(2), "length"), + ], +) +def test_invalid_shapes_are_rejected(covariance, exposures, targets, match): + with pytest.raises(ValueError, match=match): + minimum_variance_factor_weights(covariance, exposures, targets) + + +@pytest.mark.parametrize("input_name", ["covariance", "exposures", "targets"]) +def test_nonfinite_inputs_are_rejected(input_name): + covariance = np.eye(2) + exposures = np.ones((2, 1)) + targets = np.ones(1) + if input_name == "covariance": + covariance[0, 0] = np.nan + elif input_name == "exposures": + exposures[0, 0] = np.inf + else: + targets[0] = -np.inf + + with pytest.raises(ValueError, match="finite"): + minimum_variance_factor_weights(covariance, exposures, targets) + + +def test_values_outside_float64_range_raise_stable_value_error(): + with pytest.raises(ValueError, match="numeric array"): + minimum_variance_factor_weights([[10**400]], [[1.0]], [1.0]) + + +@pytest.mark.parametrize("input_name", ["covariance", "exposures", "targets"]) +def test_complex_inputs_are_rejected(input_name): + covariance = np.eye(2).astype(complex) + exposures = np.ones((2, 1), dtype=complex) + targets = np.ones(1, dtype=complex) + if input_name != "covariance": + covariance = covariance.real + if input_name != "exposures": + exposures = exposures.real + if input_name != "targets": + targets = targets.real + + with pytest.raises(ValueError, match="real-valued"): + minimum_variance_factor_weights(covariance, exposures, targets) + + +def test_inputs_are_not_mutated_and_output_is_fresh_float64(): + covariance = np.array([[2, 1], [1, 3]], dtype=np.int64) + exposures = np.array([[1], [2]], dtype=np.int64) + targets = np.array([1], dtype=np.int64) + originals = (covariance.copy(), exposures.copy(), targets.copy()) + + first = minimum_variance_factor_weights(covariance, exposures, targets) + second = minimum_variance_factor_weights(covariance, exposures, targets) + + for actual, original in zip((covariance, exposures, targets), originals): + np.testing.assert_array_equal(actual, original) + assert first.dtype == np.float64 + assert first.shape == (2,) + assert np.all(np.isfinite(first)) + assert not np.shares_memory(first, second) + + +def test_explicit_matrix_inverse_is_not_used(monkeypatch): + def fail_if_called(*_args, **_kwargs): + raise AssertionError("explicit matrix inverse must not be used") + + monkeypatch.setattr(np.linalg, "inv", fail_if_called) + weights = minimum_variance_factor_weights(np.eye(2), np.ones((2, 1)), np.ones(1)) + np.testing.assert_allclose(weights, np.array([0.5, 0.5]), rtol=0.0, atol=1e-14) + + +def test_final_factor_exposure_feasibility_is_checked(monkeypatch): + original_qr = np.linalg.qr + + def zero_orthonormal_basis(*args, **kwargs): + basis, triangular_factor = original_qr(*args, **kwargs) + return np.zeros_like(basis), triangular_factor + + monkeypatch.setattr(np.linalg, "qr", zero_orthonormal_basis) + with pytest.raises(ValueError, match="constraints.*numerical precision"): + minimum_variance_factor_weights(np.eye(2), np.ones((2, 1)), np.ones(1)) + + +def test_public_import_is_isolated_from_other_model_namespaces(tmp_path): + program = """ +import sys +import RiskLabAI + +if 'causal_factor_analysis' not in RiskLabAI.__all__: + raise SystemExit('root public surface omits causal_factor_analysis') +module = RiskLabAI.causal_factor_analysis +if not hasattr(module, 'minimum_variance_factor_weights'): + raise SystemExit('root lazy import did not resolve the optimizer') + +blocked = { + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +} +loaded = blocked.intersection(sys.modules) +if loaded: + raise SystemExit(f'unexpected imports: {sorted(loaded)}') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_protocol.py b/test/causal_factor_analysis/test_protocol.py new file mode 100644 index 0000000..5064f0f --- /dev/null +++ b/test/causal_factor_analysis/test_protocol.py @@ -0,0 +1,1165 @@ +"""Source-locked tests for the seven-stage causal-factor protocol.""" + +from __future__ import annotations + +import math +import subprocess +import sys +from dataclasses import FrozenInstanceError, replace +import pytest + +from RiskLabAI.causal_factor_analysis.protocol import ( + BacktestStage, + CausalAdjustmentSetStage, + CausalDiscoveryStage, + CausalExplanatoryAndPredictivePowerStage, + CausalFactorProtocolReport, + CausalPortfolioConstructionStage, + EventHorizon, + FoldEvidence, + MultipleTestingAdjustmentsStage, + ValidationEvidence, + VariableSelectionStage, + validate_causal_factor_protocol, +) + +PUBLIC_NAMES = { + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol", +} + + +def _horizons(): + return ( + EventHorizon("o1", 0.0, 2.0), + EventHorizon("o2", 1.0, 3.0), + EventHorizon("o3", 4.0, 5.0), + EventHorizon("o4", 6.0, 7.0), + EventHorizon("o5", 8.0, 9.0), + EventHorizon("o6", 10.0, 11.0), + ) + + +def _fold(fold_id, path_id, train_ids, test_ids, refit_id): + return FoldEvidence( + fold_id=fold_id, + path_id=path_id, + train_ids=train_ids, + test_ids=test_ids, + fit_ids=train_ids, + prediction_ids=test_ids, + refit_id=refit_id, + refit_components=( + "VARIABLE_SELECTION", + "CAUSAL_ESTIMATION", + "PORTFOLIO_CONSTRUCTION", + ), + ) + + +def _validation(method_labels=("PURGED_CROSS_VALIDATION",)): + return ValidationEvidence( + method_labels=method_labels, + event_horizons=_horizons(), + folds=( + _fold("f1", "p1", ("o3", "o4", "o5", "o6"), ("o1", "o2"), "r1"), + _fold("f2", "p1", ("o1", "o2", "o5", "o6"), ("o3", "o4"), "r2"), + _fold("f3", "p1", ("o1", "o2", "o3", "o4"), ("o5", "o6"), "r3"), + ), + embargo=1.0, + ) + + +def _cpcv_validation(): + return ValidationEvidence( + method_labels=("COMBINATORIAL_PURGED_CROSS_VALIDATION",), + event_horizons=_horizons(), + folds=( + _fold("p1-f1", "p1", ("o3", "o4", "o5", "o6"), ("o1", "o2"), "p1-r1"), + _fold("p1-f2", "p1", ("o1", "o2", "o5", "o6"), ("o3", "o4"), "p1-r2"), + _fold("p1-f3", "p1", ("o1", "o2", "o3", "o4"), ("o5", "o6"), "p1-r3"), + _fold("p2-f1", "p2", ("o4", "o5", "o6"), ("o1", "o3"), "p2-r1"), + _fold("p2-f2", "p2", ("o3", "o4", "o6"), ("o2", "o5"), "p2-r2"), + _fold("p2-f3", "p2", ("o1", "o2", "o3"), ("o4", "o6"), "p2-r3"), + ), + embargo=1.0, + ) + + +def _valid_report(): + trial_ids = ("strategy-a", "strategy-b", "strategy-c") + family_id = "preregistered-family" + selection_validation = _validation() + evaluation_validation = _validation() + backtest_validation = _cpcv_validation() + + return CausalFactorProtocolReport( + trial_family_id=family_id, + declared_trial_ids=trial_ids, + variable_selection=VariableSelectionStage( + purpose="RISK_PREMIA_HARVESTING", + selected_variables=("X", "Z", "I", "M", "C", "D"), + method_labels=( + "MUTUAL_INFORMATION", + "SHAPLEY_VALUES", + "MEAN_DECREASE_IMPURITY", + "PERMUTATION_FEATURE_IMPORTANCE", + ), + overlapping_returns=True, + strong_time_dependence=True, + validation=selection_validation, + ), + causal_discovery=CausalDiscoveryStage( + method_labels=( + "PC", + "ECONOMIC_REASONING", + "DOMAIN_EXPERTISE", + "OBSERVED_OUTCOMES", + ), + graph_id="resolved-dag", + graph_nodes=("X", "Y", "Z", "I", "M", "C", "D"), + directed_edges=( + ("Z", "X"), + ("Z", "Y"), + ("I", "X"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + graph_kind="DAG", + ambiguous_edges=(), + assumptions=( + "The recorded variables are causally sufficient for the target effect.", + "Edge directions use temporal and economic restrictions.", + ), + ), + causal_adjustment_set=CausalAdjustmentSetStage( + treatment="X", + outcome="Y", + method_label="BACKDOOR_ADJUSTMENT", + identified=True, + admissible_adjustment_sets=(("Z",),), + selected_adjustment_set=("Z",), + confounders=("Z",), + descendants=("M", "C", "D"), + mediators=("M",), + colliders=("C",), + instruments=("I",), + control_justifications=(("Z", "Common cause of factor and return."),), + open_backdoor_paths=(), + ), + causal_explanatory_and_predictive_power=( + CausalExplanatoryAndPredictivePowerStage( + task_types=("RETURN_SIZE",), + estimator_label="fold-refit causal regression", + explanatory_metric_labels=("R_SQUARED",), + predictive_metric_labels=( + "MEAN_SQUARED_ERROR", + "SPEARMAN_CORRELATION", + ), + explanatory_evidence_id="explanatory-evidence", + predictive_evidence_id="predictive-evidence", + naive_benchmark_id="training-mean-benchmark", + validation=evaluation_validation, + ) + ), + causal_portfolio_construction=CausalPortfolioConstructionStage( + method_labels=( + "POSITION_SIZING", + "EXPOSURE_CONTROL", + "ECONOMIC_RATIONALE", + "FRAGILITY_STRESS_TEST", + "TRANSACTION_COST_OPTIMIZATION", + "TRANSFER_COEFFICIENT", + ), + causal_exposures=("X",), + controlled_unintended_exposures=("C", "D"), + cost_model_id="cost-model-v1", + constraint_set_id="constraints-v1", + economic_rationale="Positions target the identified X-to-Y effect.", + fragility_scenarios=("weaken X-to-Y", "perturb Z-to-X"), + transfer_coefficient=0.82, + ), + backtest=BacktestStage( + method_labels=("COMBINATORIAL_PURGED_CROSS_VALIDATION",), + trial_family_id=family_id, + declared_trial_ids=trial_ids, + validation=backtest_validation, + ), + multiple_testing_adjustments=MultipleTestingAdjustmentsStage( + method_labels=("HOLM", "DEFLATED_SHARPE_RATIO"), + trial_family_id=family_id, + declared_trial_ids=trial_ids, + family_partitions=( + ("primary", ("strategy-a", "strategy-b")), + ("robustness", ("strategy-c",)), + ), + alpha=0.05, + backtests_independent=False, + p_value_estimator="dependence-adjusted-estimator", + time_dependence_model="purged temporal folds", + p_value_inputs_assume_independence=False, + sharpe_variance=0.04, + effective_trials=2.0, + sample_length=252, + skewness=0.1, + kurtosis=3.2, + selection_bias_evidence_id="selection-bias-evidence", + ), + ) + + +def _validate_with(report, **changes): + return validate_causal_factor_protocol(replace(report, **changes)) + + +def test_protocol_01_success_has_exact_stage_order_and_provenance(): + report = validate_causal_factor_protocol(_valid_report()) + + assert report.stage_names == ( + "Variable Selection", + "Causal Discovery", + "Causal Adjustment Set", + "Causal Explanatory and Predictive Power", + "Causal Portfolio Construction", + "Backtest", + "Multiple Testing Adjustments", + ) + assert len(report.stages) == 7 + assert report.source_oracle == "PROTOCOL-01" + assert report.source_locator == ( + "PDF pages 18-22 (journal pages 27-31), Exhibit 10" + ) + + +def test_valid_records_are_frozen_and_sequence_inputs_are_tuple_backed(): + stage = VariableSelectionStage( + purpose="CAUSAL_ATTRIBUTION", + selected_variables=["X"], + method_labels=["MUTUAL_INFORMATION"], + overlapping_returns=False, + strong_time_dependence=False, + validation=_validation(), + ) + assert stage.selected_variables == ("X",) + assert stage.method_labels == ("MUTUAL_INFORMATION",) + with pytest.raises(FrozenInstanceError): + stage.purpose = "RISK_PREMIA_HARVESTING" + + +def test_wrong_stage_record_type_is_rejected(): + report = _valid_report() + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=report.variable_selection) + + +def test_boundary_touching_horizons_are_not_overlapping(): + report = _valid_report() + horizons = tuple( + EventHorizon(f"b{index}", float(index), float(index + 1)) for index in range(6) + ) + validation = replace( + report.variable_selection.validation, + event_horizons=horizons, + folds=( + _fold("b1", "bp", ("b2", "b3", "b4", "b5"), ("b0", "b1"), "br1"), + _fold("b2", "bp", ("b0", "b1", "b4", "b5"), ("b2", "b3"), "br2"), + _fold("b3", "bp", ("b0", "b1", "b2", "b3"), ("b4", "b5"), "br3"), + ), + embargo=0.0, + ) + selection = replace( + report.variable_selection, + overlapping_returns=False, + strong_time_dependence=False, + validation=validation, + ) + evaluation = replace( + report.causal_explanatory_and_predictive_power, + validation=validation, + ) + validate_causal_factor_protocol( + replace( + report, + variable_selection=selection, + causal_explanatory_and_predictive_power=evaluation, + ) + ) + + +def test_overlap_declaration_and_purging_are_required(): + report = _valid_report() + selection = replace(report.variable_selection, overlapping_returns=False) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_cpcv_is_a_source_supported_purged_validation_method(): + report = _valid_report() + cpcv = _cpcv_validation() + selection = replace(report.variable_selection, validation=cpcv) + evaluation = replace( + report.causal_explanatory_and_predictive_power, + validation=cpcv, + ) + validate_causal_factor_protocol( + replace( + report, + variable_selection=selection, + causal_explanatory_and_predictive_power=evaluation, + ) + ) + + validation = replace( + report.variable_selection.validation, + method_labels=("WALK_FORWARD",), + ) + selection = replace(report.variable_selection, validation=validation) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_strong_time_dependence_requires_positive_embargo(): + report = _valid_report() + validation = replace(report.variable_selection.validation, embargo=0.0) + selection = replace(report.variable_selection, validation=validation) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_in_sample_stage1_methods_do_not_require_temporal_folds_without_guards(): + report = _valid_report() + selection = replace( + report.variable_selection, + method_labels=("MUTUAL_INFORMATION", "SHAPLEY_VALUES"), + overlapping_returns=False, + strong_time_dependence=False, + validation=None, + ) + validate_causal_factor_protocol(replace(report, variable_selection=selection)) + + with pytest.raises(ValueError, match="Stage 1"): + validate_causal_factor_protocol( + replace( + report, + variable_selection=replace( + selection, + method_labels=("PERMUTATION_FEATURE_IMPORTANCE",), + ), + ) + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("train_ids", ("o3", "o4", "missing")), + ("test_ids", ("o1", "o3")), + ("fit_ids", ("o1",)), + ("fit_ids", ("o3",)), + ("prediction_ids", ("o1",)), + ("refit_id", ""), + ("refit_components", ("CAUSAL_ESTIMATION",)), + ], +) +def test_fold_membership_and_training_only_refit_evidence(field, value): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + bad_fold = replace(validation.folds[0], **{field: value}) + bad_validation = replace(validation, folds=(bad_fold,) + validation.folds[1:]) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=bad_validation, + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_refit_id_cannot_represent_different_training_sets(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + second = replace(validation.folds[1], refit_id=validation.folds[0].refit_id) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace( + validation, folds=(validation.folds[0], second, validation.folds[2]) + ), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_temporal_overlap_and_embargo_leakage_are_rejected(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + overlapping_fold = _fold("bad", "p2", ("o2", "o3"), ("o1",), "bad-refit") + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace(validation, folds=(overlapping_fold,)), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + horizons = tuple( + EventHorizon(item.observation_id, item.start, item.end) + for item in validation.event_horizons + ) + ( + EventHorizon("embargoed", 3.5, 3.8), + ) + embargo_fold = _fold("bad2", "p3", ("embargoed", "o3"), ("o1", "o2"), "bad2-refit") + bad_validation = replace(validation, event_horizons=horizons, folds=(embargo_fold,)) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=bad_validation, + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +@pytest.mark.parametrize( + "changes", + [ + {"graph_kind": "CPDAG"}, + {"ambiguous_edges": (("X", "Y"),)}, + {"directed_edges": (("X", "X"),)}, + {"directed_edges": (("X", "M"), ("M", "X"))}, + {"assumptions": ()}, + ], +) +def test_discovery_requires_explicit_resolved_acyclic_dag(changes): + report = _valid_report() + discovery = replace(report.causal_discovery, **changes) + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=discovery) + + +def test_discovery_graph_contains_every_selected_variable(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + graph_nodes=tuple( + node for node in report.causal_discovery.graph_nodes if node != "D" + ), + directed_edges=tuple( + edge for edge in report.causal_discovery.directed_edges if "D" not in edge + ), + ) + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=discovery) + + +@pytest.mark.parametrize("forbidden", ["C", "M", "D"]) +def test_ordinary_controls_exclude_colliders_mediators_and_descendants(forbidden): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=((forbidden,),), + selected_adjustment_set=(forbidden,), + control_justifications=((forbidden, "Not admissible as an ordinary control."),), + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +@pytest.mark.parametrize("inadmissible", ["C", "M", "D", "I"]) +def test_every_declared_admissible_adjustment_set_is_graph_valid(inadmissible): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=(("Z",), (inadmissible,)), + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_declared_descendants_must_exactly_match_the_graph(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + descendants=("M", "C"), + ) + + with pytest.raises(ValueError, match="exactly match the graph"): + _validate_with(report, causal_adjustment_set=adjustment) + + +@pytest.mark.parametrize( + "changes", + [ + {"identified": False}, + {"open_backdoor_paths": ("X<-Z->Y",)}, + {"control_justifications": ()}, + {"selected_adjustment_set": ("I",)}, + ], +) +def test_backdoor_adjustment_fails_closed(changes): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, **changes) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_frontdoor_mediator_is_a_mechanism_not_an_ordinary_control(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="FRONT_DOOR_ADJUSTMENT", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + control_justifications=(), + frontdoor_criteria_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=adjustment) + ) + + discovery = replace( + report.causal_discovery, + directed_edges=tuple( + edge + for edge in report.causal_discovery.directed_edges + if edge != ("X", "Y") + ), + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_adjustment_set=replace( + adjustment, + admissible_adjustment_sets=(("M",),), + selected_adjustment_set=("M",), + control_justifications=(("M", "ordinary control"),), + ), + ) + ) + + +def test_instrumental_variables_requires_all_identification_premises(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + control_justifications=(), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + validate_causal_factor_protocol(replace(report, causal_adjustment_set=adjustment)) + for field in ( + "instrument_relevance_satisfied", + "instrument_exclusion_satisfied", + "instrument_exogeneity_satisfied", + ): + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_adjustment_set=replace(adjustment, **{field: False}), + ) + ) + + discovery = replace( + report.causal_discovery, + directed_edges=tuple( + edge + for edge in report.causal_discovery.directed_edges + if edge != ("I", "X") + ), + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_instrumental_variable_can_be_conditionally_exogenous(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("Z", "I"), + ("I", "X"), + ("Z", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_instrument_cannot_also_be_an_ordinary_control(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=(("I",),), + selected_adjustment_set=("I",), + control_justifications=(("I", "Not an ordinary control."),), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_selected_controls_cannot_block_instrument_relevance(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("I", "Z"), + ("Z", "X"), + ("D", "X"), + ("D", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=(("D", "Z"),), + selected_adjustment_set=("D", "Z"), + confounders=("D",), + descendants=("M", "C"), + control_justifications=( + ("D", "Blocks the graph-implied backdoor path."), + ("Z", "Declared conditional control."), + ), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_declared_instruments_are_graph_valid_for_every_adjustment_method(): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, instruments=("D",)) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_graph_implied_confounders_cannot_be_omitted_or_used_as_instruments(): + report = _valid_report() + omitted = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + confounders=(), + control_justifications=(), + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol(replace(report, causal_adjustment_set=omitted)) + + invalid_instrument = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + instruments=("Z",), + control_justifications=(), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=invalid_instrument) + ) + + +@pytest.mark.parametrize( + "field", + [ + "frontdoor_criteria_satisfied", + "instrument_relevance_satisfied", + "instrument_exclusion_satisfied", + "instrument_exogeneity_satisfied", + ], +) +def test_adjustment_method_specific_evidence_is_inapplicable_by_default(field): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, **{field: []}) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_graph_valid_backdoor_blocker_need_not_be_the_remote_common_cause(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("Z", "I"), + ("I", "X"), + ("Z", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=(("I",),), + selected_adjustment_set=("I",), + instruments=(), + control_justifications=(("I", "Blocks the graph-implied backdoor path."),), + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +@pytest.mark.parametrize( + "changes", + [ + {"predictive_metric_labels": ("LOG_LOSS",)}, + {"naive_benchmark_id": ""}, + {"predictive_evidence_id": "explanatory-evidence"}, + {"validation": replace(_validation(), method_labels=("WALK_FORWARD",))}, + ], +) +def test_stage4_metrics_baseline_separation_and_purging(changes): + report = _valid_report() + stage = replace(report.causal_explanatory_and_predictive_power, **changes) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_stage4_refits_variable_selection_inside_each_training_fold(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + first_fold = replace( + validation.folds[0], + refit_components=("CAUSAL_ESTIMATION",), + ) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace( + validation, + folds=(first_fold,) + validation.folds[1:], + ), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_purged_cross_validation_covers_every_out_of_sample_observation(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace(validation, folds=validation.folds[:2]), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_one_source_supported_stage4_aspect_is_sufficient(): + report = _valid_report() + stage = replace( + report.causal_explanatory_and_predictive_power, + explanatory_metric_labels=(), + explanatory_evidence_id=None, + ) + validate_causal_factor_protocol( + replace(report, causal_explanatory_and_predictive_power=stage) + ) + + +def test_complementary_stage4_tasks_and_multiclass_options_are_supported(): + report = _valid_report() + stage = replace( + report.causal_explanatory_and_predictive_power, + task_types=("PROBABILITY", "RANKING", "RETURN_SIZE"), + explanatory_metric_labels=("BRIER_SCORE", "R_SQUARED"), + predictive_metric_labels=( + "LOG_LOSS", + "ROC_CURVE", + "CLASSIFICATION_ACCURACY", + "MEAN_SQUARED_ERROR", + ), + multiclass_encoding="ONE_VS_REST", + averaging_method="MACRO", + ) + validate_causal_factor_protocol( + replace(report, causal_explanatory_and_predictive_power=stage) + ) + + +@pytest.mark.parametrize( + "changes", + [ + {"cost_model_id": ""}, + {"constraint_set_id": ""}, + {"economic_rationale": ""}, + {"fragility_scenarios": ()}, + {"transfer_coefficient": math.inf}, + {"transfer_coefficient": -1.01}, + {"transfer_coefficient": 1.01}, + {"causal_exposures": ()}, + { + "causal_exposures": ("X", "D"), + "controlled_unintended_exposures": ("C",), + }, + {"controlled_unintended_exposures": ("C", "D", "NOT_IN_GRAPH")}, + ], +) +def test_portfolio_construction_requires_complete_evidence(changes): + report = _valid_report() + stage = replace(report.causal_portfolio_construction, **changes) + with pytest.raises(ValueError, match="Stage 5"): + _validate_with(report, causal_portfolio_construction=stage) + + +def test_monte_carlo_requires_an_explicit_dgp(): + report = _valid_report() + stage = replace( + report.backtest, + method_labels=("MONTE_CARLO",), + validation=None, + monte_carlo_dgp="structural market simulator v1", + ) + validate_causal_factor_protocol(replace(report, backtest=stage)) + with pytest.raises(ValueError, match="Stage 6"): + validate_causal_factor_protocol( + replace(report, backtest=replace(stage, monte_carlo_dgp="")) + ) + with pytest.raises(ValueError, match="Stage 6"): + validate_causal_factor_protocol( + replace(report, backtest=replace(stage, validation="invalid")) + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with( + report, + backtest=replace(report.backtest, monte_carlo_dgp=[]), + ) + + +def test_backtest_labels_and_cpcv_path_contributions_fail_closed(): + report = _valid_report() + with pytest.raises(ValueError, match="Stage 6"): + _validate_with( + report, + backtest=replace(report.backtest, method_labels=("PBO",)), + ) + + validation = report.backtest.validation + duplicate = replace( + validation.folds[1], + path_id=validation.folds[0].path_id, + test_ids=("o1",), + prediction_ids=("o1",), + ) + stage = replace( + report.backtest, + validation=replace(validation, folds=(validation.folds[0], duplicate)), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=stage) + + incomplete = replace( + report.backtest, + validation=replace( + report.backtest.validation, + folds=(report.backtest.validation.folds[0],), + ), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=incomplete) + + +def test_cpcv_paths_must_encode_distinct_test_partitions(): + report = _valid_report() + first_path = report.backtest.validation.folds[:3] + copied_path = tuple( + replace( + fold, + fold_id=f"copy-{fold.fold_id}", + path_id="copied-path", + refit_id=f"copy-{fold.refit_id}", + ) + for fold in first_path + ) + backtest = replace( + report.backtest, + validation=replace( + report.backtest.validation, + folds=first_path + copied_path, + ), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +def test_walk_forward_cannot_train_on_future_observations(): + report = _valid_report() + stage = replace( + report.backtest, + method_labels=("WALK_FORWARD",), + validation=_validation(("WALK_FORWARD",)), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=stage) + + +def test_walk_forward_cannot_duplicate_out_of_sample_predictions(): + report = _valid_report() + first = _fold("wf1", "wf-path-1", ("o1", "o2"), ("o3", "o4"), "wf-r1") + second = replace( + first, + fold_id="wf2", + path_id="wf-path-2", + refit_id="wf-r2", + ) + validation = ValidationEvidence( + method_labels=("WALK_FORWARD",), + event_horizons=_horizons(), + folds=(first, second), + embargo=1.0, + ) + backtest = replace( + report.backtest, + method_labels=("WALK_FORWARD",), + validation=validation, + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +def test_strong_time_dependence_requires_a_backtest_embargo(): + report = _valid_report() + backtest = replace( + report.backtest, + validation=replace(report.backtest.validation, embargo=0.0), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +@pytest.mark.parametrize( + "stage_name,changes", + [ + ("backtest", {"trial_family_id": "other-family"}), + ("backtest", {"declared_trial_ids": ("strategy-a", "fabricated")}), + ("multiple_testing_adjustments", {"trial_family_id": "other-family"}), + ("multiple_testing_adjustments", {"declared_trial_ids": ("strategy-a",)}), + ], +) +def test_trial_family_identity_is_exact_across_stages(stage_name, changes): + report = _valid_report() + stage = replace(getattr(report, stage_name), **changes) + with pytest.raises(ValueError, match="Stage [67]"): + _validate_with(report, **{stage_name: stage}) + + +@pytest.mark.parametrize( + "changes", + [ + {"method_labels": ("BHY",)}, + {"alpha": True}, + {"alpha": 1.0}, + {"backtests_independent": True}, + {"family_partitions": (("primary", ("strategy-a",)),)}, + {"p_value_estimator": ""}, + {"time_dependence_model": ""}, + {"p_value_inputs_assume_independence": True}, + {"p_value_inputs_assume_independence": None}, + {"sharpe_variance": 0.0}, + {"effective_trials": 0.1}, + {"effective_trials": 3.0}, + {"sample_length": 1}, + {"skewness": math.nan}, + {"skewness": 3.0, "kurtosis": 2.0}, + {"kurtosis": -100.0}, + {"selection_bias_evidence_id": ""}, + ], +) +def test_multiple_testing_inputs_and_methods_fail_closed(changes): + report = _valid_report() + stage = replace(report.multiple_testing_adjustments, **changes) + with pytest.raises(ValueError, match="Stage 7"): + _validate_with(report, multiple_testing_adjustments=stage) + + +def test_non_iid_p_values_and_dsr_only_evidence_are_supported(): + report = _valid_report() + dependence_adjusted = replace( + report.multiple_testing_adjustments, + p_value_estimator="non-IID block bootstrap", + time_dependence_model="serial-dependence block bootstrap", + p_value_inputs_assume_independence=False, + ) + _validate_with(report, multiple_testing_adjustments=dependence_adjusted) + + dsr_only = replace( + report.multiple_testing_adjustments, + method_labels=("DEFLATED_SHARPE_RATIO",), + p_value_estimator=None, + time_dependence_model=None, + p_value_inputs_assume_independence=None, + ) + _validate_with(report, multiple_testing_adjustments=dsr_only) + + p_value_only = replace( + report.multiple_testing_adjustments, + method_labels=("HOLM",), + sharpe_variance=None, + effective_trials=None, + sample_length=None, + skewness=None, + kurtosis=None, + selection_bias_evidence_id=None, + ) + _validate_with(report, multiple_testing_adjustments=p_value_only) + + +@pytest.mark.parametrize( + "stage_name,stage", + [ + ( + "variable_selection", + replace(_valid_report().variable_selection, purpose=[]), + ), + ( + "causal_discovery", + replace(_valid_report().causal_discovery, method_labels=(["PC"],)), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, method_label=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, treatment=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, outcome=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, colliders=None), + ), + ( + "causal_explanatory_and_predictive_power", + replace( + _valid_report().causal_explanatory_and_predictive_power, + task_types=(["RETURN_SIZE"],), + ), + ), + ( + "causal_explanatory_and_predictive_power", + replace( + _valid_report().causal_explanatory_and_predictive_power, + averaging_method=[], + ), + ), + ( + "variable_selection", + replace( + _valid_report().variable_selection, + validation=replace( + _valid_report().variable_selection.validation, + method_labels=(["PURGED_CROSS_VALIDATION"],), + ), + ), + ), + ], +) +def test_malformed_labels_fail_with_value_error(stage_name, stage): + with pytest.raises(ValueError): + _validate_with(_valid_report(), **{stage_name: stage}) + + +def test_module_and_package_exports_are_exact_and_isolated(tmp_path): + program = f""" +import sys +import RiskLabAI.causal_factor_analysis as package +import RiskLabAI.causal_factor_analysis.protocol as module + +expected = {sorted(PUBLIC_NAMES)!r} +if sorted(module.__all__) != expected: + raise SystemExit('protocol public surface mismatch') +if not set(expected).issubset(package.__all__): + raise SystemExit('package public surface omits protocol names') + +blocked = {{ + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +}} +if blocked.intersection(sys.modules): + raise SystemExit('private-dependent namespace loaded') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_specification_diagnostics.py b/test/causal_factor_analysis/test_specification_diagnostics.py new file mode 100644 index 0000000..35c0249 --- /dev/null +++ b/test/causal_factor_analysis/test_specification_diagnostics.py @@ -0,0 +1,259 @@ +"""Analytical and source-example tests for the book's specification experiments.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + PopulationRegressionDiagnostics, + RegressionDiagnostics, + SpecificationExperimentResult, + SpecificationPopulationResult, + collider_population_diagnostics, + collider_specification_experiment, + confounder_undercontrolled_coefficient, + confounded_mediator_population_diagnostics, + confounded_mediator_specification_experiment, + fork_population_diagnostics, + fork_specification_experiment, +) + + +def _independent_population_regression( + predictor_covariance, + predictor_outcome_covariance, + outcome_variance, +): + covariance = np.asarray(predictor_covariance, dtype=float) + cross_covariance = np.asarray(predictor_outcome_covariance, dtype=float) + slopes = np.linalg.solve(covariance, cross_covariance) + r_squared = float(cross_covariance @ slopes / outcome_variance) + return tuple(float(value) for value in slopes), r_squared + + +def test_equation_16_and_appendix_equations_43_to_53_close_analytically(): + beta, gamma, delta = 1.25, -0.75, 2.0 + covariance_v_x = 1.0 + variance_x = 1.0 + delta**2 + conditional_v_coefficient = covariance_v_x / variance_x + conditional_v_variance = 1.0 - covariance_v_x**2 / variance_x + assert conditional_v_coefficient == 1.0 / (1.0 + delta**2) + assert conditional_v_variance == delta**2 / (1.0 + delta**2) + expected = beta + gamma * delta / (1.0 + delta**2) + assert confounder_undercontrolled_coefficient(beta, gamma, delta) == pytest.approx( + expected, rel=1.0e-15, abs=1.0e-15 + ) + + +def test_appendix_equations_54_to_61_give_exact_conditional_gaussian_moments(): + covariance_xy = np.eye(2) + covariance_xy_z = np.array([[1.0], [1.0]]) + variance_z = np.array([[3.0]]) + conditional_mean_coefficient = covariance_xy_z @ np.linalg.inv(variance_z) + conditional_covariance = ( + covariance_xy - conditional_mean_coefficient @ covariance_xy_z.T + ) + np.testing.assert_allclose( + conditional_mean_coefficient, + np.array([[1.0 / 3.0], [1.0 / 3.0]]), + rtol=0.0, + atol=1.0e-15, + ) + np.testing.assert_allclose( + conditional_covariance, + np.array([[2.0 / 3.0, -1.0 / 3.0], [-1.0 / 3.0, 2.0 / 3.0]]), + rtol=0.0, + atol=1.0e-15, + ) + + +def test_fork_population_targets_follow_from_covariance_oracle(): + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((2.0,),), + (1.0,), + 2.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((2.0, 1.0), (1.0, 1.0)), + (1.0, 1.0), + 2.0, + ) + result = fork_population_diagnostics() + assert result == SpecificationPopulationResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=PopulationRegressionDiagnostics( + ("intercept", "X"), + (0.0, *unconditioned_slopes), + unconditioned_r2, + ), + conditioned=PopulationRegressionDiagnostics( + ("intercept", "X", "Z"), + (0.0, *conditioned_slopes), + conditioned_r2, + ), + ) + + +def test_collider_population_targets_and_equation_31_are_exact(): + conditional_covariance = 0.0 - 1.0 * (1.0 / 3.0) * 1.0 + assert conditional_covariance == pytest.approx(-1.0 / 3.0) + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((1.0,),), + (0.0,), + 1.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((1.0, 1.0), (1.0, 3.0)), + (0.0, 1.0), + 1.0, + ) + result = collider_population_diagnostics() + assert result.unconditioned.coefficients == (0.0, *unconditioned_slopes) + assert result.unconditioned.r_squared == unconditioned_r2 + assert result.conditioned.coefficients == (0.0, *conditioned_slopes) + assert result.conditioned.r_squared == conditioned_r2 + assert result.conditioned.coefficients[1:] == (-0.5, 0.5) + + +def test_confounded_mediator_population_targets_follow_from_covariance_oracle(): + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((1.0,),), + (1.0,), + 7.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((1.0, 1.0), (1.0, 3.0)), + (1.0, 4.0), + 7.0, + ) + result = confounded_mediator_population_diagnostics() + assert result.unconditioned.coefficients == (0.0, *unconditioned_slopes) + assert result.unconditioned.r_squared == unconditioned_r2 + assert result.conditioned.coefficients == (0.0, *conditioned_slopes) + assert result.conditioned.r_squared == conditioned_r2 + assert result.unconditioned.r_squared == 1.0 / 7.0 + assert result.conditioned.r_squared == 11.0 / 14.0 + + +@pytest.mark.parametrize( + "function,expected_unconditioned,expected_conditioned,expected_r_squared", + [ + ( + fork_specification_experiment, + (0.008987566774752393, 0.4963512366313372), + (0.0053778294681722984, 0.0007202300104839902, 0.9957125101775881), + (0.2470257605084264, 0.49480398566467254), + ), + ( + collider_specification_experiment, + (-0.022119251900918794, 0.00150355284008128), + (-0.013760210010524129, -0.4962958394242535, 0.4988282874959081), + (2.2405222220855947e-06, 0.49918868068004596), + ), + ( + confounded_mediator_specification_experiment, + (-0.022225744034636837, 1.0055025493825809), + (0.002741745742420054, -0.4813665679581204, 1.4899423082054921), + (0.14399915797065876, 0.7840010726224431), + ), + ], +) +def test_seed_zero_source_experiments_reproduce_reported_specification_patterns( + function, + expected_unconditioned, + expected_conditioned, + expected_r_squared, +): + result = function() + assert result.unconditioned.coefficients == pytest.approx( + expected_unconditioned, rel=1.0e-12, abs=1.0e-12 + ) + assert result.conditioned.coefficients == pytest.approx( + expected_conditioned, rel=1.0e-12, abs=1.0e-12 + ) + assert result.unconditioned.r_squared == pytest.approx( + expected_r_squared[0], rel=1.0e-12, abs=1.0e-12 + ) + assert result.conditioned.r_squared == pytest.approx( + expected_r_squared[1], rel=1.0e-12, abs=1.0e-12 + ) + assert result.unconditioned.n_observations == 5_000 + assert result.conditioned.n_observations == 5_000 + + +def test_seeded_experiments_are_reproducible_and_seed_sensitive(): + first = fork_specification_experiment(sample_size=200, seed=17) + second = fork_specification_experiment(sample_size=200, seed=17) + different = fork_specification_experiment(sample_size=200, seed=18) + assert first == second + assert first != different + + +@pytest.mark.parametrize( + "function", + [ + fork_specification_experiment, + collider_specification_experiment, + confounded_mediator_specification_experiment, + ], +) +@pytest.mark.parametrize( + "sample_size,seed", + [ + (3, 0), + (True, 0), + (4.5, 0), + (1_000_001, 0), + (10, True), + (10, -1), + (10, 2**32), + ], +) +def test_experiments_reject_invalid_resource_and_seed_inputs( + function, + sample_size, + seed, +): + with pytest.raises(ValueError): + function(sample_size=sample_size, seed=seed) + + +def test_public_diagnostic_records_are_immutable_tuple_records(): + records = ( + fork_population_diagnostics(), + fork_population_diagnostics().conditioned, + fork_specification_experiment(sample_size=20, seed=3), + fork_specification_experiment(sample_size=20, seed=3).conditioned, + ) + assert isinstance(records[0], SpecificationPopulationResult) + assert isinstance(records[1], PopulationRegressionDiagnostics) + assert isinstance(records[2], SpecificationExperimentResult) + assert isinstance(records[3], RegressionDiagnostics) + for record in records: + assert not hasattr(record, "__dict__") + with pytest.raises(AttributeError): + record.structure = "changed" + + +def test_module_is_python_39_grammar_and_uses_only_numpy_plus_standard_library(): + source_path = ( + Path(__file__).resolve().parents[2] + / "src" + / "RiskLabAI" + / "causal_factor_analysis" + / "specification_diagnostics.py" + ) + source = source_path.read_text(encoding="utf-8") + tree = ast.parse(source, feature_version=(3, 9)) + roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + roots.add(node.module.split(".", 1)[0]) + assert roots == {"__future__", "math", "numbers", "typing", "numpy"} diff --git a/test/causal_factor_analysis/test_treatment_effects.py b/test/causal_factor_analysis/test_treatment_effects.py new file mode 100644 index 0000000..f971fe1 --- /dev/null +++ b/test/causal_factor_analysis/test_treatment_effects.py @@ -0,0 +1,178 @@ +"""Exact tests for the 2023 book's treatment-effect estimands.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + DifferenceInDifferencesEstimate, + TreatmentEffectDecomposition, + average_treatment_effect, + backdoor_adjusted_average_treatment_effect, + backdoor_adjusted_expectation, + difference_in_differences, + frontdoor_adjusted_probability, + linear_instrumental_variable_effect, + randomized_mean_difference, + treatment_effect_decomposition, +) + + +def test_equation_2_average_treatment_effect_is_exact_mean_contrast(): + assert average_treatment_effect(7.25, 2.0) == 5.25 + assert average_treatment_effect(-2, -5) == 3.0 + + +def test_equation_3_decomposition_closes_exactly(): + result = treatment_effect_decomposition( + observed_treated_mean=10.0, + observed_control_mean=4.0, + counterfactual_control_mean_for_treated=6.0, + ) + assert result == TreatmentEffectDecomposition( + observed_difference=6.0, + average_treatment_effect_on_treated=4.0, + sample_selection_bias=2.0, + ) + assert result.observed_difference == ( + result.average_treatment_effect_on_treated + result.sample_selection_bias + ) + + +def test_equations_4_to_6_randomized_difference_uses_both_groups(): + treated = np.array([4.0, 6.0, 8.0]) + control = np.array([1.0, 3.0, 5.0]) + treated_before = treated.copy() + control_before = control.copy() + assert randomized_mean_difference(treated, control) == 3.0 + np.testing.assert_array_equal(treated, treated_before) + np.testing.assert_array_equal(control, control_before) + + +def test_two_group_two_period_difference_in_differences_is_exact(): + result = difference_in_differences( + treated_before=10.0, + treated_after=15.0, + control_before=20.0, + control_after=22.0, + ) + assert result == DifferenceInDifferencesEstimate( + treated_change=5.0, + control_change=2.0, + estimate=3.0, + ) + + +def test_equation_8_backdoor_standardization_matches_hand_sum(): + result = backdoor_adjusted_expectation( + conditional_outcome_means=(0.1, 0.8), + adjustment_probabilities=(0.75, 0.25), + ) + assert result == pytest.approx(0.1 * 0.75 + 0.8 * 0.25, abs=1.0e-15) + + +def test_equation_7_backdoor_ate_matches_stratum_contrasts(): + result = backdoor_adjusted_average_treatment_effect( + treated_conditional_means=(0.4, 0.9), + control_conditional_means=(0.1, 0.2), + adjustment_probabilities=(0.75, 0.25), + ) + assert result == pytest.approx( + (0.4 - 0.1) * 0.75 + (0.9 - 0.2) * 0.25, + abs=1.0e-15, + ) + + +def test_equation_9_frontdoor_probability_matches_nested_hand_sum(): + result = frontdoor_adjusted_probability( + mediator_probabilities_given_treatment=(0.25, 0.75), + outcome_probabilities_given_mediator_and_treatment=( + (0.1, 0.5), + (0.4, 0.8), + ), + treatment_probabilities=(0.6, 0.4), + ) + expected = 0.25 * (0.1 * 0.6 + 0.5 * 0.4) + 0.75 * (0.4 * 0.6 + 0.8 * 0.4) + assert result == pytest.approx(expected, abs=1.0e-15) + + +def test_linear_instrumental_variable_ratio_is_exact(): + assert linear_instrumental_variable_effect(0.9, 0.3) == pytest.approx(3.0) + assert linear_instrumental_variable_effect(-2.0, 0.5) == -4.0 + + +@pytest.mark.parametrize( + "function,args", + [ + (average_treatment_effect, (True, 0.0)), + (average_treatment_effect, (math.inf, 0.0)), + (treatment_effect_decomposition, (1.0, 0.0, math.nan)), + (randomized_mean_difference, ((), (1.0,))), + (randomized_mean_difference, (((1.0, 2.0),), (1.0,))), + (randomized_mean_difference, (("1",), (1.0,))), + (difference_in_differences, (0.0, 1.0, 0.0, math.inf)), + (backdoor_adjusted_expectation, ((1.0,), (0.4, 0.6))), + (backdoor_adjusted_expectation, ((1.0, 2.0), (0.4, 0.4))), + ( + backdoor_adjusted_average_treatment_effect, + ((1.0,), (0.0, 1.0), (1.0,)), + ), + ( + frontdoor_adjusted_probability, + ((1.0,), ((0.2, 0.3),), (1.0,)), + ), + ( + frontdoor_adjusted_probability, + ((1.0,), ((1.2,),), (1.0,)), + ), + (linear_instrumental_variable_effect, (1.0, 0.0)), + ], +) +def test_estimands_reject_malformed_or_undefined_inputs(function, args): + with pytest.raises(ValueError): + function(*args) + + +def test_probability_tolerance_accepts_only_roundoff_scale_error(): + assert backdoor_adjusted_expectation( + (2.0, 4.0), + (0.5, 0.5 + 5.0e-13), + ) == pytest.approx(3.0 + 2.0e-12, abs=1.0e-14) + with pytest.raises(ValueError, match="sum to one"): + backdoor_adjusted_expectation((2.0, 4.0), (0.5, 0.50001)) + + +@pytest.mark.parametrize( + "function,args", + [ + (average_treatment_effect, (np.finfo(float).max, -np.finfo(float).max)), + ( + treatment_effect_decomposition, + (np.finfo(float).max, -np.finfo(float).max, 0.0), + ), + ( + difference_in_differences, + (-np.finfo(float).max, np.finfo(float).max, 0.0, 0.0), + ), + ( + backdoor_adjusted_average_treatment_effect, + ((np.finfo(float).max,), (-np.finfo(float).max,), (1.0,)), + ), + ], +) +def test_finite_inputs_that_overflow_the_estimand_fail_closed(function, args): + with np.errstate(over="ignore", invalid="ignore"): + with pytest.raises(ValueError): + function(*args) + + +def test_public_result_records_are_tuple_immutable_without_instance_dicts(): + treatment = treatment_effect_decomposition(3.0, 1.0, 2.0) + did = difference_in_differences(1.0, 3.0, 4.0, 5.0) + for result in (treatment, did): + assert not hasattr(result, "__dict__") + with pytest.raises(AttributeError): + result.estimate = 0.0 diff --git a/test/data/denoise/test_denoising.py b/test/data/denoise/test_denoising.py index cbcf59e..378d190 100644 --- a/test/data/denoise/test_denoising.py +++ b/test/data/denoise/test_denoising.py @@ -6,10 +6,12 @@ import pandas as pd import pytest +import RiskLabAI.data.denoise as denoise from RiskLabAI.data.denoise import ( corr_to_cov, cov_to_corr, denoise_cov, + denoised_corr2, marcenko_pastur_pdf, optimal_portfolio, ) @@ -71,6 +73,82 @@ def test_cov_corr_conversion(): assert np.allclose(cov, cov_new) +def test_denoise_public_exports_are_bound(): + """Every advertised denoising name must be bound on the package.""" + assert all(hasattr(denoise, name) for name in denoise.__all__) + + +def test_denoised_corr2_hand_verifiable_rotation(): + """Targeted shrinkage matches an exact two-factor calculation.""" + eigenvalues = np.diag([3.0, 1.0]) + eigenvectors = np.array([[1.0, 1.0], [1.0, -1.0]]) / np.sqrt(2.0) + + fully_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.0) + half_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.5) + unshrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=1.0) + + assert np.allclose(fully_shrunk, [[2.0, 1.5], [1.5, 2.0]]) + assert np.allclose(half_shrunk, [[2.0, 1.25], [1.25, 2.0]]) + assert np.allclose(unshrunk, [[2.0, 1.0], [1.0, 2.0]]) + + +def test_denoised_corr2_preserves_diagonal_trace_and_interpolates(): + """Shrinkage changes only the noise off-diagonal contribution.""" + eigenvalues = np.diag([4.0, 2.0, 0.5]) + eigenvectors, _ = np.linalg.qr( + np.array([[1.0, 2.0, 3.0], [2.0, -1.0, 1.0], [1.0, 1.0, -1.0]]) + ) + original = eigenvectors @ eigenvalues @ eigenvectors.T + + fully_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.0) + partly_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.25) + unshrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=1.0) + + assert np.allclose(unshrunk, original) + assert np.allclose(np.diag(fully_shrunk), np.diag(original)) + assert np.isclose(np.trace(fully_shrunk), np.trace(original)) + assert np.allclose(partly_shrunk, 0.75 * fully_shrunk + 0.25 * unshrunk) + + +def test_denoised_corr2_factor_count_boundaries(): + """The all-noise and all-signal partitions have exact meanings.""" + eigenvalues = np.diag([3.0, 1.0]) + eigenvectors = np.array([[1.0, 1.0], [1.0, -1.0]]) / np.sqrt(2.0) + original = eigenvectors @ eigenvalues @ eigenvectors.T + + assert np.allclose( + denoised_corr2(eigenvalues, eigenvectors, 0, alpha=0.0), + np.diag(np.diag(original)), + ) + assert np.allclose( + denoised_corr2(eigenvalues, eigenvectors, 2, alpha=0.0), original + ) + + +@pytest.mark.parametrize("alpha", [-0.01, 1.01, np.inf, np.nan]) +def test_denoised_corr2_rejects_invalid_alpha(alpha): + eigenvalues = np.diag([2.0, 1.0]) + eigenvectors = np.eye(2) + + with pytest.raises(ValueError, match="alpha"): + denoised_corr2(eigenvalues, eigenvectors, 1, alpha=alpha) + + +@pytest.mark.parametrize("num_factors", [-1, 3]) +def test_denoised_corr2_rejects_invalid_factor_count(num_factors): + with pytest.raises(ValueError, match="num_factors"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(2), num_factors) + + +def test_denoised_corr2_rejects_incompatible_inputs(): + with pytest.raises(ValueError, match="diagonal"): + denoised_corr2(np.array([[2.0, 0.1], [0.0, 1.0]]), np.eye(2), 1) + with pytest.raises(ValueError, match="match eigenvalues"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(3), 1) + with pytest.raises(TypeError, match="num_factors"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(2), 1.0) + + def test_denoise_cov(noisy_cov_matrix): """Test the end-to-end denoising function.""" cov, q = noisy_cov_matrix diff --git a/test/data/synthetic_data/test_synthetic_controlled_environment.py b/test/data/synthetic_data/test_synthetic_controlled_environment.py index 3329cd6..154df59 100644 --- a/test/data/synthetic_data/test_synthetic_controlled_environment.py +++ b/test/data/synthetic_data/test_synthetic_controlled_environment.py @@ -58,6 +58,7 @@ def test_align_params_length(): def test_generate_prices_from_regimes(sample_regimes): """Test the single-path price generation.""" + pytest.importorskip("quantecon", reason="requires RiskLabAI[synth]") regimes, tm = sample_regimes n_steps = 100 @@ -76,6 +77,7 @@ def test_generate_prices_from_regimes(sample_regimes): def test_parallel_generate_prices(sample_regimes): """Test the parallel price generation.""" + pytest.importorskip("quantecon", reason="requires RiskLabAI[synth]") regimes, tm = sample_regimes n_steps = 50 n_paths = 4 diff --git a/test/features/feature_importance/test_feature_importance.py b/test/features/feature_importance/test_feature_importance.py index 9d1c10e..b2cc3f3 100644 --- a/test/features/feature_importance/test_feature_importance.py +++ b/test/features/feature_importance/test_feature_importance.py @@ -45,7 +45,7 @@ def mock_data(): def test_controller_is_deprecated(mock_data): """The controller is deprecated in 2.0.0 (delegates to the core registry).""" _, _, classifier, _ = mock_data - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): FeatureImportanceController("MDI", classifier=classifier) diff --git a/test/optimization/test_hyper_parameter_tuning.py b/test/optimization/test_hyper_parameter_tuning.py index a92955c..89bf583 100644 --- a/test/optimization/test_hyper_parameter_tuning.py +++ b/test/optimization/test_hyper_parameter_tuning.py @@ -53,7 +53,7 @@ def fit(self, X, y, **kwargs): def test_my_pipeline_deprecated_alias_warns(): """MyPipeline is a deprecated subclass of SampleWeightedPipeline.""" assert issubclass(MyPipeline, SampleWeightedPipeline) - with pytest.warns(DeprecationWarning, match="MyPipeline.*2.1.0"): + with pytest.warns(DeprecationWarning, match="MyPipeline.*4.0.0"): pipe = MyPipeline([("clf", LogisticRegression())]) assert isinstance(pipe, SampleWeightedPipeline) diff --git a/test/test_deprecation.py b/test/test_deprecation.py index 5e8ab67..fa6c5da 100644 --- a/test/test_deprecation.py +++ b/test/test_deprecation.py @@ -20,9 +20,9 @@ def new_func(a, b=2): """Add two numbers.""" return a + b - old_func = deprecated_alias(new_func, "old_func", removed_in="2.1.0") + old_func = deprecated_alias(new_func, "old_func", removed_in="4.0.0") - with pytest.warns(DeprecationWarning, match="old_func.*2.1.0.*new_func"): + with pytest.warns(DeprecationWarning, match="old_func.*4.0.0.*new_func"): result = old_func(3, b=4) assert result == 7 @@ -34,12 +34,12 @@ class New: def __init__(self, x): self.x = x - Old = deprecated_class(New, "Old", removed_in="2.1.0") + Old = deprecated_class(New, "Old", removed_in="4.0.0") assert issubclass(Old, New) assert Old is not New - with pytest.warns(DeprecationWarning, match="Old.*2.1.0.*New"): + with pytest.warns(DeprecationWarning, match="Old.*4.0.0.*New"): obj = Old(5) assert isinstance(obj, New) @@ -47,15 +47,15 @@ def __init__(self, x): def test_warn_deprecated_name_emits_warning(): - with pytest.warns(DeprecationWarning, match="OLD_NAME.*2.1.0.*NEW_NAME"): - warn_deprecated_name("OLD_NAME", "NEW_NAME", removed_in="2.1.0") + with pytest.warns(DeprecationWarning, match="OLD_NAME.*4.0.0.*NEW_NAME"): + warn_deprecated_name("OLD_NAME", "NEW_NAME", removed_in="4.0.0") def test_deprecated_theta_constant_alias_warns(): """The non-ASCII θ constants warn (and return the ASCII value) via utils.""" import RiskLabAI.utils as utils - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): value = utils.CUMULATIVE_θ # Identifier changed; the string value is intentionally unchanged.