add inequality option for qaqc threshold flag

* update docs build dependencies
* clean up requirements.txt

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Change Log
 ==========
 
+Version 0.2.1
+-------------
+
+Added option to specify whether the threshold value used in the ``data.apply_qaqc_flags`` is to filter values that are less than or greater than the value given. Previously the function only removed data that were less than the threshold value. 
+
+Clean up dependencies in requirements.txt to match version 0.2.0 and keeping only tested and required packages. 
+
+Add new config file for ReadTheDocs, update sphinx to version 7.2.6 and fix deprecation errors.
+
 Version 0.2.0
 -------------
 

docs/source/conf.py

@@ -25,7 +25,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'flux-data-qaqc'
-copyright = '2019-2023, John Volk'
+copyright = '2019-2024, John Volk'
 author = 'John Volk'
 
 # The short X.Y version
@@ -221,7 +221,7 @@
 
 # toggle on code blocks to remove prompt ">>> "
 def setup(app):
-    app.add_javascript('copybutton.js')
-    app.add_stylesheet('custom.css')
-    app.add_javascript("custom.js")
-    app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js")
+    app.add_js_file('copybutton.js')
+    app.add_css_file('custom.css')
+    app.add_js_file("custom.js")
+    app.add_js_file("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js")

fluxdataqaqc/__init__.py

@@ -4,7 +4,7 @@
 
 __name__ = 'fluxdataqaqc'
 __author__ = 'John Volk'
-__version__ = '0.2.0'
+__version__ = '0.2.1'
 
 
 from fluxdataqaqc.data import Data

fluxdataqaqc/data.py

@@ -807,7 +807,8 @@ def _get_qc_flags(self):
         self.variables.update(tmp)
         return qc_var_pairs
 
-    def apply_qc_flags(self, threshold=None, flag=None):
+    def apply_qc_flags(self, threshold=None, flag=None, 
+            threshold_inequality='lt'):
         """
         Apply user-provided QC values or flags for climate variables to filter
         poor-quality data by converting them to null values, updates 
@@ -831,6 +832,9 @@ def apply_qc_flags(self, threshold=None, flag=None):
             flag (str, list, or tuple): default :obj:`None`. Character flag 
                 signifying data to filter out. Can be list or tuple of multiple
                 flags.
+            threshold_inequality (str): default 'lt'. 'lt' for filtering 
+                values that are less than ``threshold`` value, 'gt' for 
+                filtering values that are greater.
 
         Returns:
             :obj:`None`
@@ -893,6 +897,13 @@ def apply_qc_flags(self, threshold=None, flag=None):
             threshold = self.qc_threshold
         if not flag:
             flag = self.qc_flag
+
+        if threshold and not threshold_inequality in ('lt','gt'):
+            err_msg = ('ERROR: threshold_inequality must be "lt" or "gt", '
+                'but {} was given.'.format(
+                    climate_file
+                ))
+            raise ValueError(err_msg)
         # load dataframe if not yet accessed
         df = self.df
         # infer each columns datatype to avoid applying thresholds to strings
@@ -906,9 +917,14 @@ def apply_qc_flags(self, threshold=None, flag=None):
         if threshold:
             for var, qc in self.qc_var_pairs.items():
                 if df_types.loc[qc,'type'] != 'string':
-                    df.loc[
-                        (df[qc] < threshold) & (df[qc].notnull()) , var
-                    ] = np.nan
+                    if threshold_inequality == 'lt':
+                        df.loc[
+                            (df[qc] < threshold) & (df[qc].notnull()) , var
+                        ] = np.nan
+                    else:
+                        df.loc[
+                            (df[qc] > threshold) & (df[qc].notnull()) , var
+                        ] = np.nan
         # set values to null where flag is a certain string
         if flag:
             if isinstance(flag, str):
@@ -1284,6 +1300,7 @@ def calc_weight_avg(d, pref, df):
         # date index
         df.index = df.date
         df = df[df.index.notnull()]
+        df.index = pd.to_datetime(df.index) # ensure datetime
         df.drop('date', axis=1, inplace=True)
         self._df = df # vpd calc uses attribute
         # calc vapor pressure or vapor pressure deficit if hourly or less

requirements.txt

@@ -1,31 +1,9 @@
-bokeh==2.2.3
-certifi==2023.7.22
-cftime==1.2.1
-et-xmlfile==1.0.1
-jdcal==1.4.1
-Jinja2==2.11.3
-joblib==1.2.0
-MarkupSafe==1.1.1
-netCDF4==1.5.4
-numpy==1.22.0
-olefile==0.46
-openpyxl==3.0.5
-packaging==20.4
-pandas==1.1.4
-Pillow==9.3.0
-pip==21.1
-pyparsing==2.4.7
-python-dateutil==2.8.1
-pytz==2020.4
-PyYAML==5.4
-refet==0.3.10
-scikit-learn==0.23.2
-scipy==1.5.3
-setuptools==65.5.1
-six==1.15.0
-threadpoolctl==2.1.0
-tornado==6.1
-typing-extensions==3.7.4.3
-wheel==0.38.1
-xarray==0.16.1
-xlrd==1.2.0
+bokeh==3.2.1
+numpy==1.24.3
+openpyxl==3.0.10
+pandas==2.0.3
+pytest==7.4.0
+refet==0.4.2
+scikit_learn==1.3.0
+setuptools==68.0.0
+xarray==2023.6.0

setup.py

@@ -8,7 +8,7 @@
     version = re.search(r"__version__ = \'(.*?)\'", f.read()).group(1)
 
 requires = [
-    'bokeh==2.2.3',
+    'bokeh>=3',
     'netCDF4',
     'numpy>=1.15',
     'pandas>=1.0',