From 6a4fcb53ddbabbe43aea1845aabc9ecc692f7a56 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 11 Aug 2021 15:21:03 -0700 Subject: [PATCH 1/3] adds ophys_cells_table to consistency check --- CHANGELOG.md | 3 +++ allensdk/__init__.py | 2 +- .../project_apis/data_io/behavior_project_cloud_api.py | 3 ++- doc_template/index.rst | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 608b5c2f9..a31fe054a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Change Log All notable changes to this project will be documented in this file. +## [2.12.1] = 2021-08-11 +- minor fix to cloud cache + ## [2.12.0] = 2021-08-11 - Added ability to specify a static cache directory (use_static_cache=True) to instantiate VisualBehaviorOphysProjectCache.from_local_cache() - Added 'experience_level', 'passive' and 'image_set' columns to ophys_experiments_table diff --git a/allensdk/__init__.py b/allensdk/__init__.py index efdca9771..d8817a8e9 100644 --- a/allensdk/__init__.py +++ b/allensdk/__init__.py @@ -35,7 +35,7 @@ # import logging -__version__ = '2.12.0' +__version__ = '2.12.1' try: diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py index 44feb637c..4e648e27b 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py @@ -100,7 +100,8 @@ def load_manifest(self, manifest_name: Optional[str] = None): expected_metadata = set(["behavior_session_table", "ophys_session_table", - "ophys_experiment_table"]) + "ophys_experiment_table", + "ophys_cells_table"]) if self.cache._manifest.metadata_file_names is None: raise RuntimeError("S3CloudCache object has no metadata " diff --git a/doc_template/index.rst b/doc_template/index.rst index 5bc970af2..a1a653c32 100644 --- a/doc_template/index.rst +++ b/doc_template/index.rst @@ -118,6 +118,10 @@ The Allen SDK provides Python code for accessing experimental metadata along wit See the `mouse connectivity section `_ for more details. +What's New - 2.12.1 +----------------------------------------------------------------------- +- minor fix to cloud cache consistency check + What's New - 2.12.0 ----------------------------------------------------------------------- - Added ability to specify a static cache directory (use_static_cache=True) to instantiate VisualBehaviorOphysProjectCache.from_local_cache() From cffe84dac8f1cc37ceea630e2af0078a1e6bb69e Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 11 Aug 2021 15:29:20 -0700 Subject: [PATCH 2/3] adds cells table getter to load_manifest --- .../project_apis/data_io/behavior_project_cloud_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py index 4e648e27b..360e901ff 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py @@ -125,6 +125,7 @@ def load_manifest(self, manifest_name: Optional[str] = None): self._get_ophys_session_table() self._get_behavior_session_table() self._get_ophys_experiment_table() + self._get_ophys_cells_table() @staticmethod def from_s3_cache(cache_dir: Union[str, Path], From 6bfca509d1ca7f84116181766457d839425e56f5 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 11 Aug 2021 16:29:31 -0700 Subject: [PATCH 3/3] fixes tests related to metadata --- .../behavior_project_cache/conftest.py | 33 +++++++++++++++++++ .../test_behavior_project_cloud_api.py | 23 ++++++++++--- .../behavior_project_cache/test_from_s3.py | 5 +-- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py index 931250b25..dbb70291e 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache/conftest.py @@ -71,6 +71,17 @@ def s3_cloud_cache_data(): metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8') + o_cells = { + 'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843}, + 'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838}, + 'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}} + o_cells = pd.DataFrame(o_cells) + buff = io.StringIO() + o_cells.to_csv(buff, index=False) + buff.seek(0) + + metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8') + all_versions['data'][version] = data all_versions['metadata'][version] = metadata @@ -133,6 +144,17 @@ def s3_cloud_cache_data(): metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8') + o_cells = { + 'cell_roi_id': {0: 1080884343, 1: 1080884173, 2: 1080883843}, + 'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838}, + 'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}} + o_cells = pd.DataFrame(o_cells) + buff = io.StringIO() + o_cells.to_csv(buff, index=False) + buff.seek(0) + + metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8') + all_versions['data'][version] = data all_versions['metadata'][version] = metadata @@ -198,4 +220,15 @@ def data_update(): metadata['ophys_experiment_table'] = bytes(buff.read(), 'utf-8') + o_cells = { + 'cell_roi_id': {0: 9080884343, 1: 1080884173, 2: 1080883843}, + 'cell_specimen_id': {0: 1086496928, 1: 1086496914, 2: 1086496838}, + 'ophys_experiment_id': {0: 775614751, 1: 775614751, 2: 775614751}} + o_cells = pd.DataFrame(o_cells) + buff = io.StringIO() + o_cells.to_csv(buff, index=False) + buff.seek(0) + + metadata['ophys_cells_table'] = bytes(buff.read(), 'utf-8') + return {'data': data, 'metadata': metadata} diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py index 622dfb7a4..96c37be37 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py @@ -14,13 +14,16 @@ def __init__(self, behavior_session_table, ophys_session_table, ophys_experiment_table, + ophys_cells_table, cachedir): self.file_id_column = "file_id" self.session_table_path = cachedir / "session.csv" self.behavior_session_table_path = cachedir / "behavior_session.csv" self.ophys_experiment_table_path = cachedir / "ophys_experiment.csv" + self.ophys_cells_table_path = cachedir / "ophys_cells.csv" ophys_session_table.to_csv(self.session_table_path, index=False) + ophys_cells_table.to_csv(self.ophys_cells_table_path, index=False) behavior_session_table.to_csv(self.behavior_session_table_path, index=False) ophys_experiment_table.to_csv(self.ophys_experiment_table_path, @@ -29,11 +32,13 @@ def __init__(self, self._manifest = MagicMock() self._manifest.metadata_file_names = ["behavior_session_table", "ophys_session_table", - "ophys_experiment_table"] + "ophys_experiment_table", + "ophys_cells_table"] self._metadata_name_path_map = { "behavior_session_table": self.behavior_session_table_path, "ophys_session_table": self.session_table_path, - "ophys_experiment_table": self.ophys_experiment_table_path} + "ophys_experiment_table": self.ophys_experiment_table_path, + "ophys_cells_table": self.ophys_cells_table_path} def download_metadata(self, fname): return self._metadata_name_path_map[fname] @@ -63,6 +68,7 @@ def mock_cache(request, tmpdir): bst = request.param.get("behavior_session_table") ost = request.param.get("ophys_session_table") oet = request.param.get("ophys_experiment_table") + clt = request.param.get("ophys_cells_table") # round-trip the tables through csv to pick up # pandas mods to lists @@ -73,7 +79,9 @@ def mock_cache(request, tmpdir): ost = pd.read_csv(fname) oet.to_csv(fname, index=False) oet = pd.read_csv(fname) - yield (MockCache(bst, ost, oet, tmpdir), request.param) + clt.to_csv(fname, index=False) + clt = pd.read_csv(fname) + yield (MockCache(bst, ost, oet, clt, tmpdir), request.param) @pytest.mark.parametrize( @@ -89,7 +97,11 @@ def mock_cache(request, tmpdir): "ophys_experiment_id": [4, 5, 6, [7, 8, 9]]}), "ophys_experiment_table": pd.DataFrame({ "ophys_experiment_id": [4, 5, 6, 7, 8, 9], - "file_id": [4, 5, 6, 7, 8, 9]})}, + "file_id": [4, 5, 6, 7, 8, 9]}), + "ophys_cells_table": pd.DataFrame({ + "cell_roi_id": [4, 5, 6], + "cell_specimen_id": [104, 105, 106], + "ophys_experiment_id": [4, 5, 6]})} ], indirect=["mock_cache"]) @pytest.mark.parametrize("local", [True, False]) @@ -168,7 +180,8 @@ def test_from_local_cache(monkeypatch): mock_manifest.metadata_file_names = { 'ophys_experiment_table', 'ophys_session_table', - 'behavior_session_table' + 'behavior_session_table', + 'ophys_cells_table' } mock_manifest._data_pipeline = [ { diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py index 88af11aaf..f80de0820 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_from_s3.py @@ -99,7 +99,8 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data): local_manifest = json.load(in_file) fnames = set([pathlib.Path(k).name for k in local_manifest]) assert 'ophys_file_1.nwb' not in fnames - assert len(local_manifest) == 3 + print(local_manifest) + assert len(local_manifest) == 4 cache.construct_local_manifest() assert cache.fetch_api.cache._downloaded_data_path.is_file() @@ -108,7 +109,7 @@ def test_local_cache_construction(tmpdir, s3_cloud_cache_data): local_manifest = json.load(in_file) fnames = set([pathlib.Path(k).name for k in local_manifest]) assert 'ophys_file_1.nwb' in fnames - assert len(local_manifest) == 7 # 6 metadata files and 1 data file + assert len(local_manifest) == 9 # 8 metadata files and 1 data file @mock_s3