Atlas packaging template (#400)

* First version of the template script

* make example_mouse follow the new template

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix error in example mouse

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* mixed up reference and annotation

* added mandatory import to template

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove import from example

* added mandatory import to template

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* include case where variables should be passed between functions

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

Co-authored-by: Alessandro Felder <[email protected]>

* keep lines shorter than limit and only run code in __main__

* reformat example mouse

* use pooch to validate hash

* add bg_root_dir global

* fix references to annotation and reference

* fix error in the way structure id paths were shown in the example

* fix typo

* Update brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

Co-authored-by: Alessandro Felder <[email protected]>

* adjust comment suggestion to be compliant with the 79 character line limit

* add additional reference to packaging template script

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Alessandro Felder <[email protected]>

brainglobe_atlasapi/atlas_generation/atlas_scripts/example_mouse.py

@@ -1,7 +1,6 @@
-__version__ = "2"
-
 from pathlib import Path
 
+import pooch
 from allensdk.api.queries.ontologies_api import OntologiesApi
 from allensdk.api.queries.reference_space_api import ReferenceSpaceApi
 from allensdk.core.reference_space_cache import ReferenceSpaceCache
@@ -10,58 +9,136 @@
 
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 
-
-def create_atlas(working_dir, resolution):
-    # Specify information about the atlas:
-    RES_UM = resolution  # 100
-    ATLAS_NAME = "example_mouse"
-    SPECIES = "Mus musculus"
-    ATLAS_LINK = "http://www.brain-map.org"
-    CITATION = "Wang et al 2020, https://doi.org/10.1016/j.cell.2020.04.007"
-    ORIENTATION = "asr"
-
-    # Temporary folder for nrrd files download:
-    download_dir_path = working_dir / "downloading_path"
+# a working example atlas-packaging script, which makes a simplified version
+# of the Allen Mouse Brain Atlas, at 100um resolution. See `template_script.py`
+# for a starting point to package your own atlas.
+__version__ = 0  # This will make the example mouse version 1.0
+ATLAS_NAME = "example_mouse"
+CITATION = "Wang et al 2020, https://doi.org/10.1016/j.cell.2020.04.007"
+SPECIES = "Mus musculus"  # The scientific name of the species,
+ATLAS_LINK = "http://www.brain-map.org"  # The URL for the data files
+ORIENTATION = "asr"  # The orientation of the atlas
+ROOT_ID = 997  # The id of the highest level of the atlas.
+RESOLUTION = 100  # The resolution of your volume in microns.
+
+BG_ROOT_DIR = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+
+
+def download_resources():
+    """
+    Download the necessary resources for the atlas. Here we don't, because we
+    can out-source this to the Allen SDK in later functions.
+    """
+    pass
+
+
+def retrieve_reference_and_annotation():
+    """
+    Retrieve the Allen Mouse atlas reference and annotation as two numpy arrays
+    using the allen_sdk.
+
+    Returns:
+        tuple: A tuple containing two numpy arrays. The first array is the
+        reference volume, and the second array is the annotated volume.
+    """
+    # Create temporary download directory
+    download_dir_path = BG_ROOT_DIR / "downloading_path"
+    print(download_dir_path)
     download_dir_path.mkdir(exist_ok=True)
 
-    # Download annotated and template volume:
-    #########################################
+    # Setup the reference space cache
     spacecache = ReferenceSpaceCache(
         manifest=download_dir_path / "manifest.json",
-        # downloaded files are stored relative to here
-        resolution=RES_UM,
+        resolution=RESOLUTION,
         reference_space_key="annotation/ccf_2017",
-        # use the latest version of the CCF
     )
 
-    # Download
-    annotated_volume, _ = spacecache.get_annotation_volume()
-    template_volume, _ = spacecache.get_template_volume()
-    print("Download completed...")
+    reference_volume, _ = spacecache.get_template_volume()
+    annotation_volume, _ = spacecache.get_annotation_volume()
+    expected_reference_hash = (
+        "6c24cae773a5cf256586b0384af0ac93ad68564d211c9bdcff4bee9acf07786a"
+    )
+    expected_annotation_hash = (
+        "451e6a82f531d3db4b58056d024d3e2311703c2adc15cefa75e0268e7f0e69a4"
+    )
+    reference_hash = pooch.file_hash(
+        download_dir_path / "average_template_100.nrrd"
+    )
+    annotation_hash = pooch.file_hash(
+        download_dir_path / "annotation" / "ccf_2017" / "annotation_100.nrrd"
+    )
+    assert (
+        reference_hash == expected_reference_hash
+    ), "The hash of the reference volume does not match the expected hash."
+
+    assert (
+        annotation_hash == expected_annotation_hash
+    ), "The hash of the annotation volume does not match the expected hash."
+    # Download annotated and template volumes
+    return reference_volume, annotation_volume
+
+
+def retrieve_hemisphere_map():
+    """
+    The Allen atlas is symmetrical, so we can just return `None` in this
+    function.
 
-    # Download structures tree and meshes:
-    ######################################
-    oapi = OntologiesApi()  # ontologies
-    struct_tree = spacecache.get_structure_tree()  # structures tree
+        Returns:
+            numpy.array or None: A numpy array representing the hemisphere map,
+            or None if the atlas is symmetrical.
+    """
+    return None
+
+
+def retrieve_structure_information():
+    """
+    Retrieve the structures tree and meshes for the Allen mouse brain atlas.
+
+    Returns:
+        pandas.DataFrame: A DataFrame containing the atlas information.
+    """
+    download_dir_path = BG_ROOT_DIR / "downloading_path"
+    oapi = OntologiesApi()
+
+    spacecache = ReferenceSpaceCache(
+        manifest=download_dir_path / "manifest.json",
+        resolution=RESOLUTION,
+        reference_space_key="annotation/ccf_2017",
+    )
+    struct_tree = spacecache.get_structure_tree()  # Download structures tree
 
-    # Find id of set of regions with mesh:
     select_set = (
         "Structures whose surfaces are represented by a precomputed mesh"
     )
-
     mesh_set_ids = [
         s["id"]
         for s in oapi.get_structure_sets()
         if s["description"] == select_set
     ]
-
     structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids)[:3]
 
-    # Directory for mesh saving:
-    meshes_dir = working_dir / "mesh_temp_download"
+    # Loop over structures, remove entries not used
+    for struct in structs_with_mesh:
+        [
+            struct.pop(k)
+            for k in ["graph_id", "structure_set_ids", "graph_order"]
+        ]
+    return structs_with_mesh
 
+
+def retrieve_or_construct_meshes():
+    """
+    This function should return a dictionary of ids and corresponding paths to
+    mesh files. This atlas comes packaged with mesh files, so we don't need to
+    use our helper functions to create them ourselves in this case.
+    """
     space = ReferenceSpaceApi()
+    meshes_dir = BG_ROOT_DIR / "mesh_temp_download"
+    meshes_dir.mkdir(exist_ok=True)
+
     meshes_dict = dict()
+    structs_with_mesh = retrieve_structure_information()
+
     for s in tqdm(structs_with_mesh):
         name = s["id"]
         filename = meshes_dir / f"{name}.obj"
@@ -74,41 +151,34 @@ def create_atlas(working_dir, resolution):
             meshes_dict[name] = filename
         except (exceptions.HTTPError, ConnectionError):
             print(s)
+    return meshes_dict
 
-    # Loop over structures, remove entries not used:
-    for struct in structs_with_mesh:
-        [
-            struct.pop(k)
-            for k in ["graph_id", "structure_set_ids", "graph_order"]
-        ]
 
-    # Wrap up, compress, and remove file:
-    print("Finalising atlas")
+# Set up for the example mouse done: use default code to wrap up the atlas
+BG_ROOT_DIR.mkdir(exist_ok=True)
+download_resources()
+reference_volume, annotated_volume = retrieve_reference_and_annotation()
+hemispheres_stack = retrieve_hemisphere_map()
+structures = retrieve_structure_information()
+meshes_dict = retrieve_or_construct_meshes()
+if __name__ == "__main__":
+
     output_filename = wrapup_atlas_from_data(
         atlas_name=ATLAS_NAME,
         atlas_minor_version=__version__,
         citation=CITATION,
         atlas_link=ATLAS_LINK,
         species=SPECIES,
-        resolution=(RES_UM,) * 3,
+        resolution=(RESOLUTION,) * 3,
         orientation=ORIENTATION,
-        root_id=997,
-        reference_stack=template_volume,
+        root_id=ROOT_ID,
+        reference_stack=reference_volume,
         annotation_stack=annotated_volume,
-        structures_list=structs_with_mesh,
+        structures_list=structures,
         meshes_dict=meshes_dict,
-        working_dir=working_dir,
-        hemispheres_stack=None,
+        working_dir=BG_ROOT_DIR,
+        hemispheres_stack=hemispheres_stack,
         cleanup_files=False,
         compress=True,
+        scale_meshes=True,
     )
-
-    return output_filename
-
-
-if __name__ == "__main__":
-    # Generated atlas path:
-    bg_root_dir = Path.home() / "brainglobe_workingdir" / "example"
-    bg_root_dir.mkdir(exist_ok=True)
-
-    # create_atlas(working_dir, 100)

brainglobe_atlasapi/atlas_generation/atlas_scripts/template_script.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+
+from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+
+# Copy-paste this script into a new file and fill in the functions to package
+# your own atlas.
+
+### Metadata ###
+
+# The minor version of the atlas in the brainglobe_atlasapi, this is internal,
+# if this is the first time this atlas has been added the value should be 0
+# (minor version is the first number after the decimal point, ie the minor
+# version of 1.2 is 2)
+__version__ = 0
+
+# The expected format is FirstAuthor_SpeciesCommonName, e.g. kleven_rat, or
+# Institution_SpeciesCommonName, e.g. allen_mouse.
+ATLAS_NAME = "example_mouse"
+
+# DOI of the most relevant citable document
+CITATION = None
+
+# The scientific name of the species, ie; Rattus norvegicus
+SPECIES = None
+
+# The URL for the data files
+ATLAS_LINK = None
+
+# The orientation of the **original** atlas data, in BrainGlobe convention:
+# https://brainglobe.info/documentation/setting-up/image-definition.html#orientation
+ORIENTATION = "asr"
+
+# The id of the highest level of the atlas. This is commonly called root or
+# brain. Include some information on what to do if your atlas is not
+# hierarchical
+ROOT_ID = None
+
+# The resolution of your volume in microns. Details on how to format this
+# parameter for non isotropic datasets or datasets with multiple resolutions.
+RESOLUTION = None
+
+
+def download_resources():
+    """
+    Download the necessary resources for the atlas.
+
+    If possible, please use the Pooch library to retrieve any resources.
+    """
+    pass
+
+
+def retrieve_reference_and_annotation():
+    """
+    Retrieve the desired reference and annotation as two numpy arrays.
+
+    Returns:
+        tuple: A tuple containing two numpy arrays. The first array is the
+        reference volume, and the second array is the annotation volume.
+    """
+    reference = None
+    annotation = None
+    return reference, annotation
+
+
+def retrieve_hemisphere_map():
+    """
+    Retrieve a hemisphere map for the atlas.
+
+    If your atlas is asymmetrical, you may want to use a hemisphere map.
+    This is an array in the same shape as your template,
+    with 0's marking the left hemisphere, and 1's marking the right.
+
+    If your atlas is symmetrical, ignore this function.
+
+    Returns:
+        numpy.array or None: A numpy array representing the hemisphere map,
+        or None if the atlas is symmetrical.
+    """
+    return None
+
+
+def retrieve_structure_information():
+    """
+    This function should return a pandas DataFrame with information about your
+    atlas.
+
+    The DataFrame should be in the following format:
+
+    ╭────┬───────────────────┬─────────┬───────────────────┬─────────────────╮
+    | id | name              | acronym | structure_id_path | rgb_triplet     |
+    |    |                   |         |                   |                 |
+    ├────┼───────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 997| root              | root    | [997]             | [255, 255, 255] |
+    ├────┼───────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 8  | Basic cell groups | grey    | [997, 8]          | [191, 218, 227] |
+    ├────┼───────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 567| Cerebrum          | CH      | [997, 8, 567]     | [176, 240, 255] |
+    ╰────┴───────────────────┴─────────┴───────────────────┴─────────────────╯
+
+    Returns:
+        pandas.DataFrame: A DataFrame containing the atlas information.
+    """
+    return None
+
+
+def retrieve_or_construct_meshes():
+    """
+    This function should return a dictionary of ids and corresponding paths to
+    mesh files. Some atlases are packaged with mesh files, in these cases we
+    should use these files. Then this function should download those meshes.
+    In other cases we need to construct the meshes ourselves. For this we have
+    helper functions to achieve this.
+    """
+    meshes_dict = {}
+    return meshes_dict
+
+
+def retrieve_additional_references():
+    """This function only needs editing if the atlas has additional reference
+    images. It should return a dictionary that maps the name of each
+    additional reference image to an image stack containing its data.
+    """
+    additional_references = {}
+    return additional_references
+
+
+### If the code above this line has been filled correctly, nothing needs to be
+### edited below (unless variables need to be passed between the functions).
+if __name__ == "__main__":
+    bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+    bg_root_dir.mkdir(exist_ok=True)
+    download_resources()
+    reference_volume, annotated_volume = retrieve_reference_and_annotation()
+    additional_references = retrieve_additional_references()
+    hemispheres_stack = retrieve_hemisphere_map()
+    structures = retrieve_structure_information()
+    meshes_dict = retrieve_or_construct_meshes()
+
+    output_filename = wrapup_atlas_from_data(
+        atlas_name=ATLAS_NAME,
+        atlas_minor_version=__version__,
+        citation=CITATION,
+        atlas_link=ATLAS_LINK,
+        species=SPECIES,
+        resolution=(RESOLUTION,) * 3,
+        orientation=ORIENTATION,
+        root_id=ROOT_ID,
+        reference_stack=reference_volume,
+        annotation_stack=annotated_volume,
+        structures_list=structures,
+        meshes_dict=meshes_dict,
+        working_dir=bg_root_dir,
+        hemispheres_stack=None,
+        cleanup_files=False,
+        compress=True,
+        scale_meshes=True,
+        additional_references=additional_references,
+    )