Create utilities module which is neccessary to fix #104

cenpy/products.py

@@ -1,7 +1,10 @@
+from .utilities import _replace_missing as replace_missing_func 
+from .utilities import _fuzzy_match as fuzzy_match
+from .utilities import _coerce as coerce
+from .utilities import _can_int as can_int
 from .remote import APIConnection
 from .explorer import fips_table as _ft
 from shapely import geometry
-from fuzzywuzzy import fuzz
 from warnings import warn
 import geopandas
 import pandas
@@ -16,8 +19,6 @@
 
 __all__ = ["Decennial2010", "ACS"]
 
-_ACS_MISSING = (-999999999, -888888888, -666666666, -555555555, -333333333, -222222222)
-
 
 class _Product(object):
  """The fundamental building block to make pre-configured Census Products, like ACS or Decennial2010."""
@@ -182,10 +183,10 @@ def from_place(
  else:
  raise Exception()
 
- placematch = _fuzzy_match(name.strip(), searchtarget)
+ placematch = fuzzy_match(name.strip(), searchtarget)
  placerow = _places.loc[placematch.name]
 
- env_name = _fuzzy_match(
+ env_name = fuzzy_match(
  placerow.TYPE, [layer.__repr__() for layer in self._api.mapservice.layers]
  )
 
@@ -316,7 +317,7 @@ def chunked_query(elements_in_chunk):
 
  if replace_missing:
  for variable in variables:
- data[variable] = _replace_missing(_coerce(data[variable], float))
+ data[variable] = replace_missing_func(coerce(data[variable], float))
 
  if return_geometry:
  data = geopandas.GeoDataFrame(data)
@@ -337,7 +338,7 @@ def _environment_from_layer(
  A helper function to extract the right "container", or "environment" to
  conduct a query against. 
  """
- layername_match = _fuzzy_match(
+ layername_match = fuzzy_match(
  layername, [f.__repr__() for f in self._api.mapservice.layers]
  )
  layer = self._api.mapservice.layers[layername_match.name]
@@ -438,7 +439,7 @@ def check_match(
  and `ratio` matches. 
 
  """
- layer_result = _fuzzy_match(
+ layer_result = fuzzy_match(
  level,
  [f.__repr__() for f in self._api.mapservice.layers],
  return_table=return_table,
@@ -470,7 +471,7 @@ def check_match(
  lambda x: ", ".join(x), axis=1
  )
  self._cache.update({cache_name: cache})
- result = _fuzzy_match(name, cache.BASENAME, return_table=return_table)
+ result = fuzzy_match(name, cache.BASENAME, return_table=return_table)
  if return_level:
  return result, layer_result
  else:
@@ -639,7 +640,7 @@ def tables(self):
  )
  self._stems["columns"] = groups.apply(lambda x: x.index.tolist())
 
- is_table = numpy.asarray([_can_int(x[-1]) for x in self._stems.index])
+ is_table = numpy.asarray([can_int(x[-1]) for x in self._stems.index])
  self._tables = self._stems[is_table]
  self._crosstabs = self._stems[~is_table]
 
@@ -852,7 +853,7 @@ def tables(self):
  result = stems.drop("GEO", axis=0, errors="ignore")
  self._stems = result
  # keep around the main tables only if they're not crosstabs (ending in alphanumeric)
- self._tables = result.loc[[ix for ix in result.index if _can_int(ix[-1])]]
+ self._tables = result.loc[[ix for ix in result.index if can_int(ix[-1])]]
  return self._tables
 
  @property
@@ -882,129 +883,3 @@ def crosstab_tables(self):
  ]
  return self._crosstabs
 
-
-#############
-# UTILITIES #
-#############
-
-
-def _fuzzy_match(matchtarget, matchlist, return_table=False):
- """
- Conduct a fuzzy match with matchtarget, within the list of possible match candidates in matchlist. 
-
- Parameters
- ---------
- matchtarget : str
- a string to be matched to a set of possible candidates
- matchlist : list of str
- a list (or iterable) containing strings we are interested in matching
- return_table: bool
- whether to return the full table of scored candidates, or to return only the single
- best match. If False (the default), only the best match is returned.
-
- Notes
- -----
- consult the docstring for Product.check_match for more information on how the actual matching
- algorithm works. 
- """
- split = matchtarget.split(",")
- if len(split) == 2:
- target, state = split
- elif len(split) == 1:
- target = split[0]
- else:
- raise AssertionError(
- "Uncertain place identifier {}. The place identifier should "
- 'look something like "placename, state" or, for larger areas, '
- "like Combined Statistical Areas or Metropolitan Statistical Areas,"
- "placename1-placename2, state1-state2-state3".format(target)
- )
-
- table = pandas.DataFrame({"target": matchlist})
- table["score"] = table.target.apply(
- lambda x: fuzz.partial_ratio(target.strip().lower(), x.lower())
- )
- if len(split) == 1:
- if (table.score == table.score.max()).sum() > 1:
- ixmax, rowmax = _break_ties(matchtarget, table)
- else:
- ixmax = table.score.idxmax()
- rowmax = table.loc[ixmax]
- if return_table:
- return rowmax, table.sort_values("score")
- return rowmax
-
- in_state = table.target.str.lower().str.endswith(state.strip().lower())
-
- assert any(in_state), (
- "State {} is not found from place {}. "
- "Should be a standard Census abbreviation, like"
- " CA, AZ, NC, or PR".format(state, matchtarget)
- )
- table = table[in_state]
- if (table.score == table.score.max()).sum() > 1:
- ixmax, rowmax = _break_ties(matchtarget, table)
- else:
- ixmax = table.score.idxmax()
- rowmax = table.loc[ixmax]
- if return_table:
- return rowmax, table.sort_values("score")
- return rowmax
-
-
-def _coerce(column, kind):
- """
- Converty type of column to kind, or keep column unchanged
- if that conversion fails.
- """
- try:
- return column.astype(kind)
- except ValueError:
- return column
-
-
-def _replace_missing(column, missings=_ACS_MISSING):
- """
- replace ACS missing values using numpy.nan. 
- """
- for val in _ACS_MISSING:
- column.replace(val, numpy.nan, inplace=True)
- return column
-
-
-def _break_ties(matchtarget, table):
- """
- break ties in the fuzzy matching algorithm using a second scoring method 
- which prioritizes full string matches over substring matches. 
- """
- split = matchtarget.split(",")
- if len(split) == 2:
- target, state = split
- else:
- target = split[0]
- table["score2"] = table.target.apply(
- lambda x: fuzz.ratio(target.strip().lower(), x.lower())
- )
- among_winners = table[table.score == table.score.max()]
- double_winners = among_winners[among_winners.score2 == among_winners.score2.max()]
- if double_winners.shape[0] > 1:
- ixmax = double_winners.score2.idxmax()
- ixmax_row = double_winners.loc[ixmax]
- warn(
- "Cannot disambiguate placename {}. Picking the shortest, best "
- "matched placename, {}, from {}".format(
- matchtarget, ixmax_row.target, ", ".join(double_winners.target.tolist())
- )
- )
- return ixmax, ixmax_row
- ixmax = double_winners.score2.idxmax()
- return ixmax, double_winners.loc[ixmax]
-
-
-def _can_int(char):
- """check if a character can be turned into an integer"""
- try:
- int(char)
- return True
- except ValueError:
- return False

cenpy/remote.py

@@ -6,6 +6,7 @@
 from . import tiger as tig
 import math
 from six import iteritems, PY3
+from .utilities import _coerce as coerce
 
 if PY3:
  unicode = str
@@ -220,7 +221,7 @@ def query(self, cols=None, geo_unit="", geo_filter={}, apikey="", **kwargs):
  df = pd.DataFrame().from_records(json_content[1:], columns=json_content[0])
  assert all([col in df.columns for col in cols])
  if convert_numeric:
- df = df.infer_objects()
+ df[cols] = coerce(df[cols], int)
  if index is not "":
  df.index = df[index]
  return df

cenpy/tests/test_utilities.py

@@ -0,0 +1,55 @@
+import unittest
+import pandas
+import numpy
+from cenpy.utilities import _coerce as coerce
+from cenpy.utilities import _replace_missing as replace_missing
+
+class TestUtilities(unittest.TestCase):
+
+ def test_coerce(self):
+ # Make sure coerce works on Series and doesn't change them
+ ser_orig = pandas.Series([3,4,5])
+ ser_floats = coerce(ser_orig, cast_to = numpy.float64)
+ self.assertFalse(ser_orig.equals(ser_floats))
+
+ # Make sure coerce changes what columns it can and doesn't alter
+ # original data
+ df_orig = pandas.DataFrame({"ints": [1,2,3],
+ "floats": [0.1, 3.79, 14.9],
+ "strings": ["fst", "sec", "thd"]})
+ df_floats = coerce(df_orig, cast_to = numpy.float64)
+ # Correct types of columns after coercion:
+ float_dtypes = pandas.Series(["float64", "float64", "object"],
+ index = ["ints", "floats", "strings"])
+ # Make sure that the coerced dtypes are as expected
+ self.assertFalse(df_orig.equals(df_floats)) 
+ self.assertTrue(float_dtypes.equals(df_floats.dtypes))
+
+ # Cast castable columns into strings - 
+ # Confusingly enough, pandas calls them "objects" 
+ df_objects = coerce(df_orig, cast_to = str)
+ object_dtypes = pandas.Series(["object", "object", "object"],
+ index = ["ints", "floats", "strings"]) 
+ self.assertTrue(object_dtypes.equals(df_objects.dtypes))
+
+ # Make sure an error gets raised if a non-Series/DataFrame object is used
+ arr = numpy.zeros((2,2))
+ self.assertRaises(TypeError, coerce, arr)
+
+
+ def test_replace_missing(self):
+ df_orig = pandas.DataFrame({"ints": [-888888888,2,3],
+ "floats": [-555555555, 3.79, -333333333]})
+ df_replaced = replace_missing(df_orig)
+ # Correct output after replacing missing values
+ df_correct = pandas.DataFrame({"ints": [numpy.nan,2,3],
+ "floats": [numpy.nan, 3.79, numpy.nan]})
+ self.assertTrue(df_replaced.equals(df_correct))
+
+ # Make sure an error is raised if non-Series/DataFrame types are used
+ arr = numpy.zeros((2,2))
+ self.assertRaises(TypeError, replace_missing, arr)
+
+
+if __name__ == "__main__":
+ unittest.main()

cenpy/tiger.py

@@ -262,9 +262,9 @@ def query(self, **kwargs):
  """
  layer_result = kwargs.pop("layer", None)
  if isinstance(layer_result, str):
- from .products import _fuzzy_match
+ from .utilities import _fuzzy_match as fuzzy_match
 
- layer_result = _fuzzy_match(
+ layer_result = fuzzy_match(
  layer_result, [f.__repr__() for f in self.layers]
  ).index
  if layer_result is None: