fixes tests related to metadata

AllenInstitute · Aug 11, 2021 · 6bfca50 · 6bfca50
1 parent cffe84d
commit 6bfca50
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 7 deletions.
diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py
@@ -71,6 +71,17 @@ def s3_cloud_cache_data():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     all_versions['data'][version] = data
     all_versions['metadata'][version] = metadata
 
@@ -133,6 +144,17 @@ def s3_cloud_cache_data():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 1080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     all_versions['data'][version] = data
     all_versions['metadata'][version] = metadata
 
@@ -198,4 +220,15 @@ def data_update():
 
     metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8')
 
+    o_cells = {
+            'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843},
+            'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838},
+            'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}}
+    o_cells = pd.DataFrame(o_cells)
+    buff = io.StringIO()
+    o_cells.to_csv(buff, index=False)
+    buff.seek(0)
+
+    metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8')
+
     return {'data': data, 'metadata': metadata}
diff --git a/...test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py b/...test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py
@@ -14,13 +14,16 @@ def __init__(self,
                  behavior_session_table,
                  ophys_session_table,
                  ophys_experiment_table,
+                 ophys_cells_table,
                  cachedir):
         self.file_id_column = "file_id"
         self.session_table_path = cachedir / "session.csv"
         self.behavior_session_table_path = cachedir / "behavior_session.csv"
         self.ophys_experiment_table_path = cachedir / "ophys_experiment.csv"
+        self.ophys_cells_table_path = cachedir / "ophys_cells.csv"
 
         ophys_session_table.to_csv(self.session_table_path, index=False)
+        ophys_cells_table.to_csv(self.ophys_cells_table_path, index=False)
         behavior_session_table.to_csv(self.behavior_session_table_path,
                                       index=False)
         ophys_experiment_table.to_csv(self.ophys_experiment_table_path,
@@ -29,11 +32,13 @@ def __init__(self,
         self._manifest = MagicMock()
         self._manifest.metadata_file_names = ["behavior_session_table",
                                               "ophys_session_table",
-                                              "ophys_experiment_table"]
+                                              "ophys_experiment_table",
+                                              "ophys_cells_table"]
         self._metadata_name_path_map = {
                 "behavior_session_table": self.behavior_session_table_path,
                 "ophys_session_table": self.session_table_path,
-                "ophys_experiment_table": self.ophys_experiment_table_path}
+                "ophys_experiment_table": self.ophys_experiment_table_path,
+                "ophys_cells_table": self.ophys_cells_table_path}
 
     def download_metadata(self, fname):
         return self._metadata_name_path_map[fname]
@@ -63,6 +68,7 @@ def mock_cache(request, tmpdir):
     bst = request.param.get("behavior_session_table")
     ost = request.param.get("ophys_session_table")
     oet = request.param.get("ophys_experiment_table")
+    clt = request.param.get("ophys_cells_table")
 
     # round-trip the tables through csv to pick up
     # pandas mods to lists
@@ -73,7 +79,9 @@ def mock_cache(request, tmpdir):
     ost = pd.read_csv(fname)
     oet.to_csv(fname, index=False)
     oet = pd.read_csv(fname)
-    yield (MockCache(bst, ost, oet, tmpdir), request.param)
+    clt.to_csv(fname, index=False)
+    clt = pd.read_csv(fname)
+    yield (MockCache(bst, ost, oet, clt, tmpdir), request.param)
 
 
 @pytest.mark.parametrize(
@@ -89,7 +97,11 @@ def mock_cache(request, tmpdir):
                     "ophys_experiment_id": [4, 5, 6, [7, 8, 9]]}),
                 "ophys_experiment_table": pd.DataFrame({
                     "ophys_experiment_id": [4, 5, 6, 7, 8, 9],
-                    "file_id": [4, 5, 6, 7, 8, 9]})},
+                    "file_id": [4, 5, 6, 7, 8, 9]}),
+                "ophys_cells_table": pd.DataFrame({
+                    "cell_roi_id": [4, 5, 6],
+                    "cell_specimen_id": [104, 105, 106],
+                    "ophys_experiment_id": [4, 5, 6]})}
                 ],
         indirect=["mock_cache"])
 @pytest.mark.parametrize("local", [True, False])
@@ -168,7 +180,8 @@ def test_from_local_cache(monkeypatch):
     mock_manifest.metadata_file_names = {
         'ophys_experiment_table',
         'ophys_session_table',
-        'behavior_session_table'
+        'behavior_session_table',
+        'ophys_cells_table'
     }
     mock_manifest._data_pipeline = [
         {

diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py
@@ -99,7 +99,8 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data):
         local_manifest = json.load(in_file)
     fnames = set([pathlib.Path(k).name for k in local_manifest])
     assert 'ophys_file_1.nwb' not in fnames
-    assert len(local_manifest) == 3
+    print(local_manifest)
+    assert len(local_manifest) == 4
 
     cache.construct_local_manifest()
     assert cache.fetch_api.cache._downloaded_data_path.is_file()
@@ -108,7 +109,7 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data):
         local_manifest = json.load(in_file)
     fnames = set([pathlib.Path(k).name for k in local_manifest])
     assert 'ophys_file_1.nwb' in fnames
-    assert len(local_manifest) == 7  # 6 metadata files and 1 data file
+    assert len(local_manifest) == 9  # 8 metadata files and 1 data file
 
 
 @mock_s3