Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PySMO: fix warnings #1472

Merged
merged 25 commits into from
Aug 23, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f851405
Minor change to trigger tests
OOAmusat Jul 1, 2022
38cbd23
Added abs to tolerance check
OOAmusat Jul 1, 2022
f0ed373
Run black
OOAmusat Jul 1, 2022
9a4703c
Remove string comparison
OOAmusat Jul 1, 2022
e6524fd
Tighten tolerance
OOAmusat Jul 1, 2022
050da9a
Fixed kriging test by removing variable dependence
OOAmusat Jul 1, 2022
bed7c18
Merge branch 'IDAES:main' into fix_test_errors
OOAmusat Oct 17, 2022
b3b4fb2
Merge branch 'IDAES:main' into fix_test_errors
OOAmusat Sep 7, 2023
4d2fa20
Fixing random seed for kriging test
OOAmusat Sep 7, 2023
0b0df28
Merge branch 'IDAES:main' into fix_test_errors
OOAmusat Sep 9, 2023
408e2c2
Merge branch 'IDAES:main' into fix_test_errors
OOAmusat Aug 16, 2024
2ad951a
fix regression warnings.
OOAmusat Aug 16, 2024
322dc5a
regression warnings fixed
OOAmusat Aug 16, 2024
2e4f58b
Update sampling with logging and fix warnings.
OOAmusat Aug 16, 2024
409e92a
Fix kriging numpy warnings
OOAmusat Aug 16, 2024
e9e2f02
running black...
OOAmusat Aug 17, 2024
009f50f
fix warnings in sampling test file.
OOAmusat Aug 17, 2024
33a3b0a
Fix warnings in pysmo_surrogate.py due to numpy
OOAmusat Aug 17, 2024
4c8fdf7
fixing pylint issues.
OOAmusat Aug 17, 2024
3023a0c
Added @pytest.mark.filterwarnings to catach runtime warnings.
OOAmusat Aug 19, 2024
b509086
Make pytest filters more specific
OOAmusat Aug 19, 2024
555bf4c
running black...
OOAmusat Aug 19, 2024
d34371d
Merge branch 'main' into fix_test_errors
OOAmusat Aug 22, 2024
3066db0
Merge branch 'main' into fix_test_errors
ksbeattie Aug 22, 2024
8577f4e
Merge branch 'main' into fix_test_errors
lbianchi-lbl Aug 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions idaes/core/surrogate/pysmo/kriging.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,9 +537,10 @@ def error_calculation(theta, p, mean, cov_inv, y_mu, x, y_data):
for i in range(0, x.shape[0]):
cmt = (np.matmul(((np.abs(x[i, :] - x)) ** p), theta)).transpose()
cov_matrix_tests = np.exp(-1 * cmt)
y_prediction[i, 0] = mean + np.matmul(
y_pred_val = mean + np.matmul(
np.matmul(cov_matrix_tests.transpose(), cov_inv), y_mu
)
y_prediction[i, 0] = y_pred_val.item()
ss_error = (1 / y_data.shape[0]) * (np.sum((y_data - y_prediction) ** 2))
rmse_error = np.sqrt(ss_error)
return ss_error, rmse_error, y_prediction
Expand Down Expand Up @@ -589,10 +590,11 @@ def predict_output(self, x_pred):
)
).transpose()
cov_matrix_tests = np.exp(-1 * cmt)
y_pred[i, 0] = self.optimal_mean + np.matmul(
y_pred_val = self.optimal_mean + np.matmul(
np.matmul(cov_matrix_tests.transpose(), self.covariance_matrix_inverse),
self.optimal_y_mu,
)
y_pred[i, 0] = y_pred_val.item()
return y_pred

def training(self):
Expand Down
17 changes: 7 additions & 10 deletions idaes/core/surrogate/pysmo/polynomial_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
# pylint: disable=missing-function-docstring

import os.path
import warnings
import pickle

# Imports from third parties
Expand All @@ -42,7 +41,9 @@

# Imports from IDAES namespace
from idaes.core.surrogate.pysmo.utils import NumpyEvaluator
import idaes.logger as idaeslog

_log = idaeslog.getLogger(__name__)

__author__ = "Oluwamayowa Amusat"

Expand Down Expand Up @@ -346,7 +347,7 @@ def __init__(
print("The number of cross-validation cases (3) is used.")
number_of_crossvalidations = 3
elif number_of_crossvalidations > 10:
warnings.warn(
_log.warning(
"The number of cross-validations entered is large. The simulation may take a while to run"
)
self.number_of_crossvalidations = number_of_crossvalidations
Expand All @@ -356,7 +357,7 @@ def __init__(
# pylint: disable-next=broad-exception-raised
raise Exception("Maximum polynomial order must be an integer")
elif maximum_polynomial_order > 10:
warnings.warn(
_log.warning(
"The maximum allowed polynomial order is 10. Value has been adjusted to 10."
)
maximum_polynomial_order = 10
Expand Down Expand Up @@ -1000,7 +1001,7 @@ def results_generation(self, beta, order):
print("\n------------------------------------------------------------")
print("The final coefficients of the regression terms are: \n")
print("k |", beta[0, 0])
results_df = pd.concat([results_df, pd.Series({"k": beta[0, 0]})], axis=0)
results_df = pd.Series({"k": beta[0, 0]})
if self.multinomials == 1:
for i in range(1, order + 1):
for j in range(1, self.number_of_x_vars + 1):
Expand Down Expand Up @@ -1449,9 +1450,7 @@ def polynomial_regression_fitting(self, additional_regression_features=None):
if r_square_opt > 0.95:
self.fit_status = "ok"
else:
warnings.warn(
"Polynomial regression generates poor fit for the dataset"
)
_log.warning("Polynomial regression generates poor fit for the dataset")
self.fit_status = "poor"

self.pickle_save({"model": self})
Expand Down Expand Up @@ -1551,9 +1550,7 @@ def polynomial_regression_fitting(self, additional_regression_features=None):
if r_square > 0.95:
self.fit_status = "ok"
else:
warnings.warn(
"Polynomial regression generates poor fit for the dataset"
)
_log.warning("Polynomial regression generates poor fit for the dataset")
self.fit_status = "poor"

self.pickle_save({"model": self})
Expand Down
6 changes: 4 additions & 2 deletions idaes/core/surrogate/pysmo/radial_basis_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

# Imports from the python standard library
import os.path
import warnings
import pickle

# Imports from third parties
Expand All @@ -43,6 +42,9 @@

# Imports from IDAES namespace
from idaes.core.surrogate.pysmo.sampling import FeatureScaling as fs
import idaes.logger as idaeslog

_log = idaeslog.getLogger(__name__)

__author__ = "Oluwamayowa Amusat"

Expand Down Expand Up @@ -1105,7 +1107,7 @@ def training(self):
if x_condition_number < (1 / np.finfo(float).eps):
self.solution_status = "ok"
else:
warnings.warn(
_log.warning(
"The parameter matrix A in A.x=B is ill-conditioned (condition number > 1e10). The solution returned may be inaccurate or unstable - inspect rmse error. Regularization (if not already done) may improve solution"
)
self.solution_status = "unstable solution"
Expand Down
16 changes: 9 additions & 7 deletions idaes/core/surrogate/pysmo/sampling.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
# pylint: disable=missing-class-docstring
# pylint: disable=missing-function-docstring

import warnings
import itertools

import numpy as np
import pandas as pd
import idaes.logger as idaeslog

_log = idaeslog.getLogger(__name__)

__author__ = "Oluwamayowa Amusat"

Expand Down Expand Up @@ -180,7 +182,7 @@ def sample_point_selection(self, full_data, sample_points, sampling_type):

unique_sample_points = np.unique(points_closest_unscaled, axis=0)
if unique_sample_points.shape[0] < points_closest_unscaled.shape[0]:
warnings.warn(
_log.warning(
"The returned number of samples is less than the requested number due to repetitions during nearest neighbour selection."
)
print(
Expand Down Expand Up @@ -388,7 +390,7 @@ def selection_columns_preprocessing(self, data_input, xlabels, ylabels):
warn_str = "The following columns were dropped: " + str(
dropped_cols
)
warnings.warn(warn_str)
_log.warning(warn_str)
self.x_data = data_input.filter(xlabels).values
self.data_headers = set_of_labels
self.data_headers_xvars = xlabels
Expand Down Expand Up @@ -448,7 +450,7 @@ def selection_columns_preprocessing(self, data_input, xlabels, ylabels):
warn_str = "The following columns were dropped: " + str(
dropped_cols
)
warnings.warn(warn_str)
_log.warning(warn_str)
self.x_data = data_input[:, xlabels]
self.data_headers = set_of_labels
self.data_headers_xvars = xlabels
Expand Down Expand Up @@ -1417,7 +1419,7 @@ def __init__(
elif tolerance > 0.1:
raise ValueError("Tolerance must be less than 0.1 to achieve good results")
elif tolerance < 1e-9:
warnings.warn(
_log.warning(
"Tolerance too tight. CVT algorithm may take long time to converge."
)
elif (tolerance < 0.1) and (tolerance > 1e-9):
Expand Down Expand Up @@ -1784,7 +1786,7 @@ def generate_from_dist(self, dist_name):
)
> 0
):
warnings.warn(
_log.warning(
"Points adjusted to remain within specified Gaussian bounds. This may affect the underlying distribution."
)
out_locations = [
Expand All @@ -1796,7 +1798,7 @@ def generate_from_dist(self, dist_name):
rep_value = var_values[k]
while (rep_value < 0) or (rep_value > 1):
rep_value = dist(loc=0.5, scale=1 / 6, size=1)
var_values[k] = rep_value
var_values[k] = rep_value[0]
assert (
sum([1 for i in range(0, var_values.shape[0]) if var_values[i] > 1])
+ sum(
Expand Down
15 changes: 14 additions & 1 deletion idaes/core/surrogate/pysmo/tests/test_kriging.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ def test__init__07(self, array_type):

@pytest.mark.unit
@pytest.mark.parametrize("array_type", [np.array, pd.DataFrame])
@pytest.mark.filterwarnings(
"ignore:invalid value encountered in log:RuntimeWarning"
OOAmusat marked this conversation as resolved.
Show resolved Hide resolved
)
def test__init__08(self, array_type):
input_array = array_type(self.test_data)
file_name = "test_filename.pickle"
Expand All @@ -107,6 +110,9 @@ def test__init__08(self, array_type):

@pytest.mark.unit
@pytest.mark.parametrize("array_type", [np.array, pd.DataFrame])
@pytest.mark.filterwarnings(
"ignore:invalid value encountered in log:RuntimeWarning"
)
def test__init__09(self, array_type):
input_array = array_type(self.test_data)
file_name1 = "test_filename1.pickle"
Expand Down Expand Up @@ -343,9 +349,10 @@ def test_error_calculation(self, array_type):
)
).transpose()
cov_matrix_tests = np.exp(-1 * cmt)
y_prediction_exp[i, 0] = mean + np.matmul(
y_prediction_val = mean + np.matmul(
np.matmul(cov_matrix_tests.transpose(), cov_inv), y_mu
)
y_prediction_exp[i, 0] = y_prediction_val.item()

ss_error, rmse_error, y_prediction = KrigingClass.error_calculation(
theta,
Expand Down Expand Up @@ -402,6 +409,9 @@ def test_r2_calculation(self, array_type):

@pytest.mark.unit
@pytest.mark.parametrize("array_type", [np.array, pd.DataFrame])
@pytest.mark.filterwarnings(
"ignore:invalid value encountered in log:RuntimeWarning"
)
def test_predict_output_01(self, array_type):
input_array = array_type(self.training_data)
np.random.seed(0)
Expand All @@ -412,6 +422,9 @@ def test_predict_output_01(self, array_type):

@pytest.mark.unit
@pytest.mark.parametrize("array_type", [np.array, pd.DataFrame])
@pytest.mark.filterwarnings(
"ignore:invalid value encountered in log:RuntimeWarning"
)
def test_predict_output(self, array_type):
input_array = array_type(self.training_data)
np.random.seed(0)
Expand Down
113 changes: 56 additions & 57 deletions idaes/core/surrogate/pysmo/tests/test_polynomial_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import numpy as np
import pandas as pd
import pytest
import idaes.logger as idaeslog


class TestFeatureScaling:
Expand Down Expand Up @@ -300,19 +301,23 @@ def test__init__08(self, array_type1, array_type2):
@pytest.mark.unit
@pytest.mark.parametrize("array_type1", [np.array, pd.DataFrame])
@pytest.mark.parametrize("array_type2", [np.array, pd.DataFrame])
def test__init__09(self, array_type1, array_type2):
def test__init__09(self, array_type1, array_type2, caplog):
caplog.set_level(idaeslog.WARNING)
warning_msg = "The number of cross-validations entered is large. The simulation may take a while to run"
original_data_input = array_type1(self.test_data)
regression_data_input = array_type2(self.sample_points)
with pytest.warns(Warning):
PolyClass = PolynomialRegression(
original_data_input,
regression_data_input,
maximum_polynomial_order=5,
number_of_crossvalidations=11,
)
assert (
PolyClass.number_of_crossvalidations == 11
) # Default number of cross-validations
PolyClass = PolynomialRegression(
original_data_input,
regression_data_input,
maximum_polynomial_order=5,
number_of_crossvalidations=11,
)
assert warning_msg in caplog.text
for record in caplog.records:
assert record.levelno == idaeslog.WARNING
assert (
PolyClass.number_of_crossvalidations == 11
) # Default number of cross-validations

@pytest.mark.unit
@pytest.mark.parametrize("array_type1", [np.array, pd.DataFrame])
Expand All @@ -328,14 +333,23 @@ def test__init__10(self, array_type1, array_type2):
@pytest.mark.unit
@pytest.mark.parametrize("array_type1", [np.array, pd.DataFrame])
@pytest.mark.parametrize("array_type2", [np.array, pd.DataFrame])
def test__init__11(self, array_type1, array_type2):
def test__init__11(self, array_type1, array_type2, caplog):
caplog.set_level(idaeslog.WARNING)
warning_msg = (
"The maximum allowed polynomial order is 10. Value has been adjusted to 10."
)
original_data_input = array_type1(self.test_data_large)
regression_data_input = array_type2(self.sample_points_large)
with pytest.warns(Warning):
PolyClass = PolynomialRegression(
original_data_input, regression_data_input, maximum_polynomial_order=11
)
assert PolyClass.max_polynomial_order == 10
PolyClass = PolynomialRegression(
original_data_input, regression_data_input, maximum_polynomial_order=11
)
warning_msg = (
"The maximum allowed polynomial order is 10. Value has been adjusted to 10."
)
assert warning_msg in caplog.text
for record in caplog.records:
assert record.levelno == idaeslog.WARNING
assert PolyClass.max_polynomial_order == 10

@pytest.mark.unit
@pytest.mark.parametrize("array_type1", [np.array, pd.DataFrame])
Expand Down Expand Up @@ -1645,17 +1659,12 @@ def test_results_generation_01(self, array_type1, array_type2):
beta = np.array([[0], [0], [0]])
expected_df = pd.Series()
row_list = np.array([["k"], ["(x_1)^1"], ["(x_2)^1"]])
expected_df = pd.concat(
[
expected_df,
pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
}
),
]
expected_df = pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
}
)
output_df = data_feed.results_generation(beta, order)
assert output_df.index.to_list() == expected_df.index.to_list()
Expand Down Expand Up @@ -1687,21 +1696,16 @@ def test_results_generation_02(self, array_type1, array_type2):
["(x_2)^3"],
]
)
expected_df = pd.concat(
[
expected_df,
pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
row_list[3, 0]: beta[3, 0],
row_list[4, 0]: beta[4, 0],
row_list[5, 0]: beta[5, 0],
row_list[6, 0]: beta[6, 0],
}
),
]
expected_df = pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
row_list[3, 0]: beta[3, 0],
row_list[4, 0]: beta[4, 0],
row_list[5, 0]: beta[5, 0],
row_list[6, 0]: beta[6, 0],
}
)
output_df = data_feed.results_generation(beta, order)
assert output_df.index.to_list() == expected_df.index.to_list()
Expand All @@ -1725,20 +1729,15 @@ def test_results_generation_03(self, array_type1, array_type2):
row_list = np.array(
[["k"], ["(x_1)^1"], ["(x_2)^1"], ["(x_1)^2"], ["(x_2)^2"], ["(x_1).(x_2)"]]
)
expected_df = pd.concat(
[
expected_df,
pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
row_list[3, 0]: beta[3, 0],
row_list[4, 0]: beta[4, 0],
row_list[5, 0]: beta[5, 0],
}
),
]
expected_df = pd.Series(
{
row_list[0, 0]: beta[0, 0],
row_list[1, 0]: beta[1, 0],
row_list[2, 0]: beta[2, 0],
row_list[3, 0]: beta[3, 0],
row_list[4, 0]: beta[4, 0],
row_list[5, 0]: beta[5, 0],
}
)
output_df = data_feed.results_generation(beta, order)
assert output_df.index.to_list() == expected_df.index.to_list()
Expand Down
Loading
Loading