Merge pull request #2186 from AllenInstitute/cache_fixup

adds ophys_cells_table to consistency check
AllenInstitute · Aug 11, 2021 · 570f2f6 · 570f2f6
2 parents 878a7db + 6bfca50
commit 570f2f6
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,9 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.12.1] = 2021-08-11
+- minor fix to cloud cache
+
 ## [2.12.0] = 2021-08-11
 - Added ability to specify a static cache directory (use_static_cache=True) to instantiate VisualBehaviorOphysProjectCache.from_local_cache()
 - Added 'experience_level', 'passive' and 'image_set' columns to ophys_experiments_table

diff --git a/allensdk/__init__.py b/allensdk/__init__.py
@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.12.0'
+__version__ = '2.12.1'
 
 
 try:

diff --git a/...vatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/...vatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py
@@ -100,7 +100,8 @@ def load_manifest(self, manifest_name: Optional[str] = None):
 
         expected_metadata = set(["behavior_session_table",
                                  "ophys_session_table",
-                                 "ophys_experiment_table"])
+                                 "ophys_experiment_table",
+                                 "ophys_cells_table"])
 
         if self.cache._manifest.metadata_file_names is None:
             raise RuntimeError("S3CloudCache object has no metadata "
@@ -124,6 +125,7 @@ def load_manifest(self, manifest_name: Optional[str] = None):
         self._get_ophys_session_table()
         self._get_behavior_session_table()
         self._get_ophys_experiment_table()
+        self._get_ophys_cells_table()
 
     @staticmethod
     def from_s3_cache(cache_dir: Union[str, Path],

diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py
@@ -71,6 +71,17 @@ def s3_cloud_cache_data():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     all_versions['data'][version] = data
     all_versions['metadata'][version] = metadata
 
@@ -133,6 +144,17 @@ def s3_cloud_cache_data():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 1080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     all_versions['data'][version] = data
     all_versions['metadata'][version] = metadata
 
@@ -198,4 +220,15 @@ def data_update():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     return {'data': data, 'metadata': metadata}
diff --git a/...test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py b/...test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py
@@ -14,13 +14,16 @@ def __init__(self,
                  behavior_session_table,
                  ophys_session_table,
                  ophys_experiment_table,
+                 ophys_cells_table,
                  cachedir):
         self.file_id_column = "file_id"
         self.session_table_path = cachedir / "session.csv"
         self.behavior_session_table_path = cachedir / "behavior_session.csv"
         self.ophys_experiment_table_path = cachedir / "ophys_experiment.csv"
+        self.ophys_cells_table_path = cachedir / "ophys_cells.csv"
 
         ophys_session_table.to_csv(self.session_table_path, index=False)
+        ophys_cells_table.to_csv(self.ophys_cells_table_path, index=False)
         behavior_session_table.to_csv(self.behavior_session_table_path,
                                       index=False)
         ophys_experiment_table.to_csv(self.ophys_experiment_table_path,
@@ -29,11 +32,13 @@ def __init__(self,
         self._manifest = MagicMock()
         self._manifest.metadata_file_names = ["behavior_session_table",
                                               "ophys_session_table",
-                                              "ophys_experiment_table"]
+                                              "ophys_experiment_table",
+                                              "ophys_cells_table"]
         self._metadata_name_path_map = {
                 "behavior_session_table": self.behavior_session_table_path,
                 "ophys_session_table": self.session_table_path,
-                "ophys_experiment_table": self.ophys_experiment_table_path}
+                "ophys_experiment_table": self.ophys_experiment_table_path,
+                "ophys_cells_table": self.ophys_cells_table_path}
 
     def download_metadata(self, fname):
         return self._metadata_name_path_map[fname]
@@ -63,6 +68,7 @@ def mock_cache(request, tmpdir):
     bst = request.param.get("behavior_session_table")
     ost = request.param.get("ophys_session_table")
     oet = request.param.get("ophys_experiment_table")
+    clt = request.param.get("ophys_cells_table")
 
     # round-trip the tables through csv to pick up
     # pandas mods to lists
@@ -73,7 +79,9 @@ def mock_cache(request, tmpdir):
     ost = pd.read_csv(fname)
     oet.to_csv(fname, index=False)
     oet = pd.read_csv(fname)
-    yield (MockCache(bst, ost, oet, tmpdir), request.param)
+    clt.to_csv(fname, index=False)
+    clt = pd.read_csv(fname)
+    yield (MockCache(bst, ost, oet, clt, tmpdir), request.param)
 
 
 @pytest.mark.parametrize(
@@ -89,7 +97,11 @@ def mock_cache(request, tmpdir):
                     "ophys_experiment_id": [4, 5, 6, [7, 8, 9]]}),
                 "ophys_experiment_table": pd.DataFrame({
                     "ophys_experiment_id": [4, 5, 6, 7, 8, 9],
-                    "file_id": [4, 5, 6, 7, 8, 9]})},
+                    "file_id": [4, 5, 6, 7, 8, 9]}),
+                "ophys_cells_table": pd.DataFrame({
+                    "cell_roi_id": [4, 5, 6],
+                    "cell_specimen_id": [104, 105, 106],
+                    "ophys_experiment_id": [4, 5, 6]})}
                 ],
         indirect=["mock_cache"])
 @pytest.mark.parametrize("local", [True, False])
@@ -168,7 +180,8 @@ def test_from_local_cache(monkeypatch):
     mock_manifest.metadata_file_names = {
         'ophys_experiment_table',
         'ophys_session_table',
-        'behavior_session_table'
+        'behavior_session_table',
+        'ophys_cells_table'
     }
     mock_manifest._data_pipeline = [
         {

diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py
@@ -99,7 +99,8 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data):
         local_manifest = json.load(in_file)
     fnames = set([pathlib.Path(k).name for k in local_manifest])
     assert 'ophys_file_1.nwb' not in fnames
-    assert len(local_manifest) == 3
+    print(local_manifest)
+    assert len(local_manifest) == 4
 
     cache.construct_local_manifest()
     assert cache.fetch_api.cache._downloaded_data_path.is_file()
@@ -108,7 +109,7 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data):
         local_manifest = json.load(in_file)
     fnames = set([pathlib.Path(k).name for k in local_manifest])
     assert 'ophys_file_1.nwb' in fnames
-    assert len(local_manifest) == 7  # 6 metadata files and 1 data file
+    assert len(local_manifest) == 9  # 8 metadata files and 1 data file
 
 
 @mock_s3

diff --git a/doc_template/index.rst b/doc_template/index.rst
@@ -118,6 +118,10 @@ The Allen SDK provides Python code for accessing experimental metadata along wit
 
 See the `mouse connectivity section <connectivity.html>`_ for more details.
 
+What's New - 2.12.1
+-----------------------------------------------------------------------
+- minor fix to cloud cache consistency check
+
 What's New - 2.12.0
 -----------------------------------------------------------------------
 - Added ability to specify a static cache directory (use_static_cache=True) to instantiate VisualBehaviorOphysProjectCache.from_local_cache()