resolving bokeh and pandas new version changes

Open-ET · Aug 28, 2023 · 91e6698 · 91e6698
1 parent b35d057
commit 91e6698
Show file tree

Hide file tree

Showing 6 changed files with 638 additions and 766 deletions.
diff --git a/examples/Basic_usage/Tutorial.ipynb b/examples/Basic_usage/Tutorial.ipynb
diff --git a/examples/Config_options/advanced_config_options.ipynb b/examples/Config_options/advanced_config_options.ipynb
diff --git a/examples/Config_options/config_for_multiple_soil_vars.ini b/examples/Config_options/config_for_multiple_soil_vars.ini
@@ -1,13 +1,3 @@
-########################################################################################################################
-# Configuration file for flux-data-qaqc module for processing eddy covariance data
-#
-# Examples for bringing in n soil heat flux/moisture measurements for weighted averaging are described in the DATA section
-#
-# The purpose of this configuration file is to specify:
-#	1.	The metadata of the weather station and path to the data file that you are processing
-#	2.	The format and names of weather data variables within the data file
-#
-########################################################################################################################
 [METADATA]
 station_latitude = 36.6058
 station_longitude = -97.4888
@@ -18,8 +8,6 @@ skiprows = 2
 date_parser = %Y%m%d%H%M
 site_id = US-ARM_multipe_soilvars
 var_name_delim = ;
-gridmet_file_path = gridMET_data/US-ARM_36.6083N_-97.4750W.csv
-# below are added metadata that are not used explicitly by flux-data-qaqc
 country = USA
 doi_contributor_name = Sebastien Biraud
 doi_contributor_role = Author
@@ -121,19 +109,6 @@ soil_depth_date = 2007.0
 soil_tex_date = 2007.0
 
 [DATA]
-####################
-# Data Organization
-# In this section, type the name of the variable column as it appears within the header of the input file
-# Then type the units in a way that matches one of the accepted unit strings
-# Put 'na' or 'NA' or do not list entries for variables not provided
-
-# Accepted units (note these may be updated at any time to include more possible units, 
-# see the QaQc.allowable_units attribute for up-to-date options for specific variables):
-# TEMPERATURE: f, c
-# RADIATION:   w/m2
-# PRESSURE:    kpa
-# SPEED:       m/s
-# LENGTH:      in, mm
 datestring_col = TIMESTAMP_START
 net_radiation_col = NETRAD_1_1_1
 net_radiation_units = w/m2
@@ -155,60 +130,6 @@ avg_temp_col = T_SONIC_1_1_1
 avg_temp_units = C
 wind_spd_col = WS_1_1_1;WS_1_2_1
 wind_spd_units = m/s
-
-######
-## HOW TO ADD ADDITIONAL SOIL HEAT FLUX OR MOISTURE MEASUREMENTS FOR COMPARISON
-######
-# This is an optional addition to the main qa/qc and plotting routines.
-# Here you can also include the same variable used for the main G energy balance
-# component, i.e. the variable you assigned to "ground_flux_col" in config.ini
-#
-# Follow the following rules for specifying multiple soil heat flux 
-# or soil moisture time series measurements in order to make daily and monthly 
-# comparison plots of the variables using flux-data-qaqc:
-#
-# For SOIL HEAT FLUX the name of the variable should begin with "G_" or "g_" 
-# followed by an integer starting with 1,2,3,... for example:
-# g_1 = name_of_my_soil_heat_flux_variable
-# 
-# another valid example:
-# g_2 = g_z10
-#
-# For SOIL MOISTURE the name of the variable should begin with "theta_[number]" for example:
-# theta_1 = name_of_my_soil_moisture_variable
-#
-# To specify the units of your soil flux/moisture variables add "_units" to the name you assigned:
-# g_1_units = w/m2
-#
-# NOTE: if multiple soil variables are given their units should be all the same
-# NOTE: the names in your file, "g_z10" in previous example will be the 
-# name used in flux-data-qaqc, i.e. you will see them stored or displayed in 
-# Data, QaQc, Plot, and saved output files
-#
-######
-## HOW TO SET WEIGHTS FOR WEIGHTED AVERAGE OF MULTIPLE SOIL HEAT FLUX/ MOISTURE VARIABLES
-######
-# flux-data-qaqc will calculate the weighted average time series for multiple soil heat
-# flux or moisture variables if they are given, the time series can be saved or displayed
-# alongside the raw variables in time series plots, these variables will be saved by
-# flux-data-qaqc as g_mean and theta_mean respectively
-#
-# To set weights simply define new entries for each variable with the "_weight" suffix,
-# for example, to set weights for multiple soil heat flux variables:
-# g_1_weight = 0.25
-# g_2_weight = 0.25
-# g_3_weight = 0.5
-#
-# or for soil moisture variables:
-# theta_1_weight = 0.4
-# ...
-#
-# NOTE: if weights are not stated then the unweighted average soil heat flux and soil
-# moisture time series will be calculated
-# NOTE: if weights do not sum to 1 they will be automatically normalized upon loading data
-#
-# Below is a working example that uses data for G in the input climate file where we weight
-# sensors 2 and 3 much higher than 1 and 2:
 g_1 = G_1_1_1
 g_1_units = w/m2
 g_1_weight = 1
@@ -221,8 +142,6 @@ g_3_weight = 10
 g_4 = G_4_1_1
 g_4_units = w/m2
 g_4_weight = 1
-
-# note if we do not specify weights the arithmetic mean will be calculated
 theta_1 = SWC_1_1_1
 theta_1_units = (%): Soil water content (volumetric), range 0-100
 theta_2 = SWC_2_1_1

diff --git a/fluxdataqaqc/data.py b/fluxdataqaqc/data.py
@@ -428,7 +428,7 @@ def _calc_vpd_or_vp(self, df):
 
 
     def plot(self, ncols=1, output_type='save', out_file=None, suptitle='', 
-            plot_width=1000, plot_height=450, sizing_mode='scale_both', 
+            plot_width=1000, plot_height=450, sizing_mode='fixed', 
             merge_tools=False, link_x=True, **kwargs):
         """
         Creates a series of interactive diagnostic line and scatter 

diff --git a/fluxdataqaqc/plot.py b/fluxdataqaqc/plot.py
@@ -265,7 +265,7 @@ def scatter_plot(fig, x, y, source, color, label='',
             for t in new_tips:
                 if not t in fig.hover[0].tooltips:
                     fig.hover[0].tooltips.append(t)
-        fig.hover[0].names.append(name)
+        fig.hover[0].name = name
         fig.legend.location = "top_left"
         fig.legend.click_policy="hide"
 
@@ -359,7 +359,7 @@ def add_lines(fig, df, plt_vars, colors, x_name, source,
 
     def _plot(self, FluxObj, ncols=1, output_type='save', out_file=None, 
             suptitle='', plot_width=1000, plot_height=450, 
-            sizing_mode='scale_both', merge_tools=False, link_x=True, **kwargs): 
+            sizing_mode='fixed', merge_tools=False, link_x=True, **kwargs): 
         """ 
         Private routine for aggregated validation plots that are used by
         the :meth:`.QaQc.plot` and :meth:`.Data.plot` methods.
@@ -1208,7 +1208,7 @@ def _get_units(plt_vars, units):
                 each.x_range = monthly_line[0].x_range
         figs = daily_line + daily_scatter + monthly_line + monthly_scatter
         grid = gridplot(
-            figs, ncols=ncols, width=None, height=None, 
+            figs, ncols=ncols, width=plot_width, height=plot_height, 
             sizing_mode=sizing_mode, merge_tools=merge_tools, **kwargs
         )
         if output_type == 'show':

diff --git a/fluxdataqaqc/qaqc.py b/fluxdataqaqc/qaqc.py
@@ -666,7 +666,7 @@ def _check_daily_freq(self, drop_gaps, daily_frac, max_interp_hours,
             print('Data is being resampled to daily temporal frequency.')
             sum_cols = [k for k,v in self.agg_dict.items() if v == 'sum']
             sum_cols = list(set(sum_cols).intersection(df.columns))
-            mean_cols = set(df.columns) - set(sum_cols)
+            mean_cols = list(set(df.columns) - set(sum_cols))
 
             means = df.loc[:,mean_cols].apply(
                 pd.to_numeric, errors='coerce').resample('D').mean().copy()
@@ -696,38 +696,42 @@ def _check_daily_freq(self, drop_gaps, daily_frac, max_interp_hours,
                     ].copy()
                     grped_night.drop_duplicates(inplace=True)
                     grped_night = grped_night.groupby(
-                        pd.Grouper(freq='24H', offset='12H')).apply(
-                            lambda x: x.interpolate(
-                                method='linear', limit=max_night_gap, 
-                                limit_direction='both', limit_area='inside'
+                        pd.Grouper(freq='24H', offset='12H'), 
+                            group_keys=True).apply(
+                                lambda x: x.interpolate(
+                                    method='linear', limit=max_night_gap, 
+                                    limit_direction='both', limit_area='inside'
                             )
                         )
                     grped_day = tmp.loc[(tmp.Rn >= 0) | (tmp.Rn.isna())].copy()
                     grped_day.drop_duplicates(inplace=True)
                     grped_day = grped_day.groupby(
-                        pd.Grouper(freq='24H')).apply(
-                            lambda x: x.interpolate(
-                                method='linear', limit=max_gap, 
-                                limit_direction='both', limit_area='inside'
+                        pd.Grouper(freq='24H'),
+                            group_keys=True).apply(
+                                lambda x: x.interpolate(
+                                    method='linear', limit=max_gap, 
+                                    limit_direction='both', limit_area='inside'
                             )
                         )
                 else:
                     grped_night = tmp.copy()
                     grped_night.drop_duplicates(inplace=True)
                     grped_night = grped_night.groupby(
-                        pd.Grouper(freq='24H', offset='12H')).apply(
-                            lambda x: x.interpolate(
-                                method='linear', limit=max_night_gap, 
-                                limit_direction='both', limit_area='inside'
+                        pd.Grouper(freq='24H', offset='12H'),
+                            group_keys=True).apply(
+                                lambda x: x.interpolate(
+                                    method='linear', limit=max_night_gap, 
+                                    limit_direction='both', limit_area='inside'
                             )
                         )
                     grped_day = tmp.copy()
                     grped_day.drop_duplicates(inplace=True)
                     grped_day = grped_day.groupby(
-                        pd.Grouper(freq='24H')).apply(
-                            lambda x: x.interpolate(
-                                method='linear', limit=max_gap, 
-                                limit_direction='both', limit_area='inside'
+                        pd.Grouper(freq='24H'),
+                            group_keys=True).apply(
+                                lambda x: x.interpolate(
+                                    method='linear', limit=max_gap, 
+                                    limit_direction='both', limit_area='inside'
                             )
                         )
 
@@ -1688,10 +1692,11 @@ def _ebr_correction(self):
         # so prepend and append first and last 5 days and loop...
         doy_ebr_mean=df['ebr_corr'].groupby(df.index.dayofyear).mean().copy()
         l5days = pd.Series(
-            index=np.arange(-4,1), data=doy_ebr_mean[-5:].values)
+            index=np.arange(-4,1), data=doy_ebr_mean.iloc[-5:].values)
         f5days = pd.Series(
-            index=np.arange(367,372), data=doy_ebr_mean[:5].values)
-        doy_ebr_mean = doy_ebr_mean.append(f5days)
+            index=np.arange(367,372), data=doy_ebr_mean.iloc[:5].values)
+        #doy_ebr_mean = doy_ebr_mean.append(f5days)
+        doy_ebr_mean = pd.concat([doy_ebr_mean, f5days])
         doy_ebr_mean = pd.concat([l5days, doy_ebr_mean])
         ebr_5day_clim = pd.DataFrame(
             index=np.arange(1,367), columns=['ebr_5day_clim'])
@@ -1850,7 +1855,7 @@ def _bowen_ratio_correction(self):
         self.corrected = True
 
     def plot(self, ncols=1, output_type='save', out_file=None, suptitle='', 
-            plot_width=1000, plot_height=450, sizing_mode='scale_both', 
+            plot_width=1000, plot_height=450, sizing_mode='fixed', 
             merge_tools=False, link_x=True, **kwargs):
         """
         Creates a series of interactive diagnostic line and scatter