diff --git a/HISTORY.rst b/HISTORY.rst index caf3d557..5b2f4322 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,7 @@ History 0.8.x (2024-xx-xx) ------------------ +* Add `** predict_params` in fit and predict method for Mapie Regression * Update the ts-changepoint notebook with the tutorial * Change import related to conformity scores into ts-changepoint notebook * Replace `assert np.array_equal` by `np.testing.assert_array_equal` in Mapie unit tests diff --git a/mapie/estimator/classifier.py b/mapie/estimator/classifier.py index 0c7fa16c..c0e41fa5 100644 --- a/mapie/estimator/classifier.py +++ b/mapie/estimator/classifier.py @@ -447,7 +447,7 @@ def predict( self, X: ArrayLike, agg_scores: Optional[str] = None, - **predict_params + **predict_params, ) -> NDArray: """ Predict target from X. It also computes the prediction per train sample diff --git a/mapie/estimator/regressor.py b/mapie/estimator/regressor.py index da6596c3..a200586c 100644 --- a/mapie/estimator/regressor.py +++ b/mapie/estimator/regressor.py @@ -233,6 +233,7 @@ def _predict_oof_estimator( estimator: RegressorMixin, X: ArrayLike, val_index: ArrayLike, + **predict_params ) -> Tuple[NDArray, ArrayLike]: """ Perform predictions on a single out-of-fold model on a validation set. @@ -248,6 +249,9 @@ def _predict_oof_estimator( val_index: ArrayLike of shape (n_samples_val) Validation data indices. + **predict_params : dict + Additional predict parameters. + Returns ------- Tuple[NDArray, ArrayLike] @@ -255,7 +259,7 @@ def _predict_oof_estimator( """ X_val = _safe_indexing(X, val_index) if _num_samples(X_val) > 0: - y_pred = estimator.predict(X_val) + y_pred = estimator.predict(X_val, **predict_params) else: y_pred = np.array([]) return y_pred, val_index @@ -306,7 +310,7 @@ def _aggregate_with_mask( else: raise ValueError("The value of self.agg_function is not correct") - def _pred_multi(self, X: ArrayLike) -> NDArray: + def _pred_multi(self, X: ArrayLike, **predict_params) -> NDArray: """ Return a prediction per train sample for each test sample, by aggregation with matrix ``k_``. @@ -316,12 +320,15 @@ def _pred_multi(self, X: ArrayLike) -> NDArray: X: ArrayLike of shape (n_samples_test, n_features) Input data + **predict_params : dict + Additional predict parameters. + Returns ------- NDArray of shape (n_samples_test, n_samples_train) """ y_pred_multi = np.column_stack( - [e.predict(X) for e in self.estimators_] + [e.predict(X, **predict_params) for e in self.estimators_] ) # At this point, y_pred_multi is of shape # (n_samples_test, n_estimators_). The method @@ -334,7 +341,8 @@ def predict_calib( self, X: ArrayLike, y: Optional[ArrayLike] = None, - groups: Optional[ArrayLike] = None + groups: Optional[ArrayLike] = None, + **predict_params ) -> NDArray: """ Perform predictions on X : the calibration set. @@ -355,6 +363,9 @@ def predict_calib( By default ``None``. + **predict_params : dict + Additional predict parameters. + Returns ------- NDArray of shape (n_samples_test, 1) @@ -371,7 +382,7 @@ def predict_calib( cv = cast(BaseCrossValidator, self.cv) outputs = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)( delayed(self._predict_oof_estimator)( - estimator, X, calib_index, + estimator, X, calib_index, **predict_params ) for (_, calib_index), estimator in zip( cv.split(X, y, groups), @@ -404,7 +415,7 @@ def fit( y: ArrayLike, sample_weight: Optional[ArrayLike] = None, groups: Optional[ArrayLike] = None, - **fit_params, + **fit_params ) -> EnsembleRegressor: """ Fit the base estimator under the ``single_estimator_`` attribute. @@ -526,6 +537,9 @@ def predict( predictions (3 arrays). If ``False`` the method return the simple predictions only. + **predict_params : dict + Additional predict parameters. + Returns ------- Tuple[NDArray, NDArray, NDArray] @@ -535,7 +549,7 @@ def predict( """ check_is_fitted(self, self.fit_attributes) - y_pred = self.single_estimator_.predict(X) + y_pred = self.single_estimator_.predict(X, **predict_params) if not return_multi_pred and not ensemble: return y_pred @@ -543,7 +557,7 @@ def predict( y_pred_multi_low = y_pred[:, np.newaxis] y_pred_multi_up = y_pred[:, np.newaxis] else: - y_pred_multi = self._pred_multi(X) + y_pred_multi = self._pred_multi(X, **predict_params) if self.method == "minmax": y_pred_multi_low = np.min(y_pred_multi, axis=1, keepdims=True) diff --git a/mapie/regression/quantile_regression.py b/mapie/regression/quantile_regression.py index 2635b026..e30646ab 100644 --- a/mapie/regression/quantile_regression.py +++ b/mapie/regression/quantile_regression.py @@ -649,6 +649,7 @@ def predict( optimize_beta: bool = False, allow_infinite_bounds: bool = False, symmetry: Optional[bool] = True, + **predict_params, ) -> Union[NDArray, Tuple[NDArray, NDArray]]: """ Predict target on new samples with confidence intervals. @@ -676,6 +677,9 @@ def predict( each residuals separatly or to use the maximum of the two combined. + predict_params : dict + Additional predict parameters. + Returns ------- Union[NDArray, Tuple[NDArray, NDArray]] @@ -699,7 +703,7 @@ def predict( dtype=float, ) for i, est in enumerate(self.estimators_): - y_preds[i] = est.predict(X) + y_preds[i] = est.predict(X, **predict_params) check_lower_upper_bounds(y_preds[0], y_preds[1], y_preds[2]) if symmetry: quantile = np.full( diff --git a/mapie/regression/regression.py b/mapie/regression/regression.py index aeb68b5b..aa6656e8 100644 --- a/mapie/regression/regression.py +++ b/mapie/regression/regression.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Iterable, Optional, Tuple, Union, cast +from typing import Any, Iterable, Optional, Tuple, Union, cast import numpy as np from sklearn.base import BaseEstimator, RegressorMixin @@ -19,7 +19,8 @@ from mapie.utils import (check_alpha, check_alpha_and_n_samples, check_cv, check_estimator_fit_predict, check_n_features_in, check_n_jobs, check_null_weight, - check_verbose, get_effective_calibration_samples) + check_verbose, get_effective_calibration_samples, + check_predict_params) class MapieRegressor(BaseEstimator, RegressorMixin): @@ -471,7 +472,7 @@ def fit( y: ArrayLike, sample_weight: Optional[ArrayLike] = None, groups: Optional[ArrayLike] = None, - **fit_params, + **kwargs: Any ) -> MapieRegressor: """ Fit estimator and compute conformity scores used for @@ -504,14 +505,21 @@ def fit( train/test set. By default ``None``. - **fit_params : dict - Additional fit parameters. + kwargs : dict + Additional fit and predict parameters. Returns ------- MapieRegressor The model itself. """ + fit_params = kwargs.pop('fit_params', {}) + predict_params = kwargs.pop('predict_params', {}) + if len(predict_params) > 0: + self._predict_params = True + else: + self._predict_params = False + # Checks (estimator, self.conformity_score_function_, @@ -538,7 +546,9 @@ def fit( ) # Predict on calibration data - y_pred = self.estimator_.predict_calib(X, y=y, groups=groups) + y_pred = self.estimator_.predict_calib( + X, y=y, groups=groups, **predict_params + ) # Compute the conformity scores (manage jk-ab case) self.conformity_scores_ = \ @@ -555,6 +565,7 @@ def predict( alpha: Optional[Union[float, Iterable[float]]] = None, optimize_beta: bool = False, allow_infinite_bounds: bool = False, + **predict_params ) -> Union[NDArray, Tuple[NDArray, NDArray]]: """ Predict target on new samples with confidence intervals. @@ -604,6 +615,9 @@ def predict( By default ``False``. + predict_params : dict + Additional predict parameters. + Returns ------- Union[NDArray, Tuple[NDArray, NDArray]] @@ -614,6 +628,8 @@ def predict( - [:, 1, :]: Upper bound of the prediction interval. """ # Checks + if hasattr(self, '_predict_params'): + check_predict_params(self._predict_params, predict_params, self.cv) check_is_fitted(self, self.fit_attributes) self._check_ensemble(ensemble) alpha = cast(Optional[NDArray], check_alpha(alpha)) @@ -621,7 +637,7 @@ def predict( # If alpha is None, predict the target without confidence intervals if alpha is None: y_pred = self.estimator_.predict( - X, ensemble, return_multi_pred=False + X, ensemble, return_multi_pred=False, **predict_params ) return np.array(y_pred) diff --git a/mapie/regression/time_series_regression.py b/mapie/regression/time_series_regression.py index b96dc17d..a2c76ce9 100644 --- a/mapie/regression/time_series_regression.py +++ b/mapie/regression/time_series_regression.py @@ -407,6 +407,7 @@ def predict( alpha: Optional[Union[float, Iterable[float]]] = None, optimize_beta: bool = False, allow_infinite_bounds: bool = False, + **predict_params ) -> Union[NDArray, Tuple[NDArray, NDArray]]: """ Predict target on new samples with confidence intervals. @@ -441,6 +442,9 @@ def predict( allow_infinite_bounds: bool Allow infinite prediction intervals to be produced. + predict_params : dict + Additional predict parameters. + Returns ------- Union[NDArray, Tuple[NDArray, NDArray]] @@ -452,7 +456,8 @@ def predict( """ if alpha is None: super().predict( - X, ensemble=ensemble, alpha=alpha, optimize_beta=optimize_beta + X, ensemble=ensemble, alpha=alpha, optimize_beta=optimize_beta, + **predict_params ) if self.method == "aci": @@ -460,7 +465,7 @@ def predict( return super().predict( X, ensemble=ensemble, alpha=alpha, optimize_beta=optimize_beta, - allow_infinite_bounds=allow_infinite_bounds + allow_infinite_bounds=allow_infinite_bounds, **predict_params ) def _more_tags(self): diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py index 98c17bd2..80e57855 100644 --- a/mapie/tests/test_regression.py +++ b/mapie/tests/test_regression.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd import pytest +from scipy.stats import ttest_1samp from sklearn.compose import ColumnTransformer from sklearn.datasets import make_regression @@ -20,7 +21,6 @@ from sklearn.pipeline import Pipeline, make_pipeline from sklearn.preprocessing import OneHotEncoder from sklearn.utils.validation import check_is_fitted -from scipy.stats import ttest_1samp from typing_extensions import TypedDict from mapie._typing import NDArray @@ -44,6 +44,28 @@ random_state = 1 + +class CustomGradientBoostingRegressor(GradientBoostingRegressor): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def fit(self, X, y, **kwargs): + return super().fit(X, y, **kwargs) + + def predict(self, X, check_predict_params=False): + if check_predict_params: + return np.zeros(X.shape[0]) + return super().predict(X) + + +def early_stopping_monitor(i, est, locals): + """Returns True on the 3rd iteration.""" + if i == 2: + return True + else: + return False + + Params = TypedDict( "Params", { @@ -862,26 +884,136 @@ def test_fit_parameters_passing() -> None: only during boosting, instead of default value for n_estimators (=100). """ gb = GradientBoostingRegressor(random_state=random_state) - mapie = MapieRegressor(estimator=gb, random_state=random_state) + mapie.fit(X, y, fit_params={'monitor': early_stopping_monitor}) - def early_stopping_monitor(i, est, locals): - """Returns True on the 3rd iteration.""" - if i == 2: - return True - else: - return False + assert mapie.estimator_.single_estimator_.estimators_.shape[0] == 3 + for estimator in mapie.estimator_.estimators_: + assert estimator.estimators_.shape[0] == 3 - mapie.fit(X, y, monitor=early_stopping_monitor) - assert mapie.estimator_.single_estimator_.estimators_.shape[0] == 3 +def test_predict_parameters_passing() -> None: + """ + Test passing predict parameters. + Checks that y_pred from train are 0, y_pred from test are 0. + """ + X_train, X_test, y_train, y_test = ( + train_test_split(X, y, test_size=0.2, random_state=random_state) + ) + custom_gbr = CustomGradientBoostingRegressor(random_state=random_state) + score = AbsoluteConformityScore(sym=True) + mapie_model = MapieRegressor(estimator=custom_gbr, conformity_score=score) + predict_params = {'check_predict_params': True} + mapie_model = mapie_model.fit( + X_train, y_train, predict_params=predict_params + ) + y_pred = mapie_model.predict(X_test, **predict_params) + np.testing.assert_allclose(mapie_model.conformity_scores_, np.abs(y_train)) + np.testing.assert_allclose(y_pred, 0) - for estimator in mapie.estimator_.estimators_: + +def test_fit_params_expected_behavior_unaffected_by_predict_params() -> None: + """ + We want to verify that there are no interferences + with predict_params on the expected behavior of fit_params + Checks that underlying GradientBoosting + estimators have used 3 iterations only during boosting, + instead of default value for n_estimators (=100). + """ + X_train, X_test, y_train, y_test = ( + train_test_split(X, y, test_size=0.2, random_state=random_state) + ) + custom_gbr = CustomGradientBoostingRegressor(random_state=random_state) + mapie_model = MapieRegressor(estimator=custom_gbr) + fit_params = {'monitor': early_stopping_monitor} + predict_params = {'check_predict_params': True} + mapie_model = mapie_model.fit( + X_train, y_train, + fit_params=fit_params, predict_params=predict_params + ) + + assert mapie_model.estimator_.single_estimator_.estimators_.shape[0] == 3 + for estimator in mapie_model.estimator_.estimators_: assert estimator.estimators_.shape[0] == 3 +def test_predict_params_expected_behavior_unaffected_by_fit_params() -> None: + """ + We want to verify that there are no interferences + with fit_params on the expected behavior of predict_params + Checks that the predictions on the training and test sets + are 0 for the model with predict_params and that this is not + the case for the model without predict_params + """ + X_train, X_test, y_train, y_test = ( + train_test_split(X, y, test_size=0.2, random_state=random_state) + ) + custom_gbr = CustomGradientBoostingRegressor(random_state=random_state) + score = AbsoluteConformityScore(sym=True) + mapie_model = MapieRegressor(estimator=custom_gbr, conformity_score=score) + fit_params = {'monitor': early_stopping_monitor} + predict_params = {'check_predict_params': True} + mapie_model = mapie_model.fit( + X_train, y_train, + fit_params=fit_params, + predict_params=predict_params + ) + y_pred = mapie_model.predict(X_test, **predict_params) + + np.testing.assert_array_equal(mapie_model.conformity_scores_, + np.abs(y_train)) + np.testing.assert_allclose(y_pred, 0) + + +def test_using_one_predict_parameter_into_predict_but_not_in_fit() -> None: + """ + Test that using predict parameters in the predict method + without using predict_parameter in the fit method raises an error. + """ + custom_gbr = CustomGradientBoostingRegressor(random_state=random_state) + X_train, X_test, y_train, y_test = ( + train_test_split(X, y, test_size=0.2, random_state=random_state) + ) + mapie = MapieRegressor(estimator=custom_gbr) + predict_params = {'check_predict_params': True} + mapie_fitted = mapie.fit(X_train, y_train) + + with pytest.raises(ValueError, match=( + fr".*Using 'predict_param' '{predict_params}' " + r"without using one 'predict_param' in the fit method\..*" + r"Please ensure a similar configuration of 'predict_param' " + r"is used in the fit method before calling it in predict\..*" + )): + mapie_fitted.predict(X_test, **predict_params) + + +def test_using_one_predict_parameter_into_fit_but_not_in_predict() -> None: + """ + Test that using predict parameters in the fit method + without using predict_parameter in + the predict method raises an error. + """ + custom_gbr = CustomGradientBoostingRegressor(random_state=random_state) + X_train, X_test, y_train, y_test = ( + train_test_split(X, y, test_size=0.2, random_state=random_state) + ) + mapie = MapieRegressor(estimator=custom_gbr) + predict_params = {'check_predict_params': True} + mapie_fitted = mapie.fit(X_train, y_train, predict_params=predict_params) + + with pytest.raises(ValueError, match=( + r"Using one 'predict_param' in the fit method " + r"without using one 'predict_param' in the predict method. " + r"Please ensure a similar configuration of 'predict_param' " + r"is used in the predict method as called in the fit." + )): + mapie_fitted.predict(X_test) + + def test_predict_infinite_intervals() -> None: - """Test that MapieRegressor produces infinite bounds with alpha=0""" + """ + Test that MapieRegressor produces infinite bounds with alpha=0 + """ mapie_reg = MapieRegressor().fit(X, y) _, y_pis = mapie_reg.predict(X, alpha=0., allow_infinite_bounds=True) np.testing.assert_allclose(y_pis[:, 0, 0], -np.inf) @@ -891,7 +1023,9 @@ def test_predict_infinite_intervals() -> None: @pytest.mark.parametrize("method", ["minmax", "naive", "plus", "base"]) @pytest.mark.parametrize("cv", ["split", "prefit"]) def test_check_change_method_to_base(method: str, cv: str) -> None: - """Test of the overloading of method attribute to `base` method in fit""" + """ + Test of the overloading of method attribute to `base` method in fit + """ X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.5, random_state=random_state diff --git a/mapie/utils.py b/mapie/utils.py index 13641b15..fa781edb 100644 --- a/mapie/utils.py +++ b/mapie/utils.py @@ -1373,3 +1373,43 @@ def check_n_samples( " int in the range [1, inf)" ) return int(n_samples) + + +def check_predict_params( + predict_params_used_in_fit: bool, + predict_params: dict, + cv: Optional[Union[int, str, BaseCrossValidator]] = None +) -> None: + """ + Check that if predict_params is used in the predict method, + it is also used in the fit method. Otherwise, raise an error. + + Parameters + ---------- + predict_params_used_in_fit: bool + True if one or more predict_params are used in the fit method + + predict_param: dict + Contains all predict params used in predict method + + Raises + ------ + ValueError + If any predict_params are used in the predict method but none + are used in the fit method. + """ + if cv != "prefit": + if len(predict_params) > 0 and predict_params_used_in_fit is False: + raise ValueError( + f"Using 'predict_param' '{predict_params}' " + f"without using one 'predict_param' in the fit method. " + f"Please ensure a similar configuration of 'predict_param' " + f"is used in the fit method before calling it in predict." + ) + if len(predict_params) == 0 and predict_params_used_in_fit is True: + raise ValueError( + "Using one 'predict_param' in the fit method " + "without using one 'predict_param' in the predict method. " + "Please ensure a similar configuration of 'predict_param' " + "is used in the predict method as called in the fit." + )