Loading atlas with no Internet (#358)

* list_atlas_no_internet

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* list_atlas_no_internet

* WIP graceful failure with no internet connection

* Refactoring

* Reverted error message for invalid atlas name

* Added tests for new utils functions

* Added tests for get_atlases_last_versions

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Igor Tatarnikov <[email protected]>

brainglobe_atlasapi/bg_atlas.py

@@ -8,7 +8,11 @@
 from rich.console import Console
 
 from brainglobe_atlasapi import config, core, descriptors, utils
-from brainglobe_atlasapi.utils import _rich_atlas_metadata
+from brainglobe_atlasapi.utils import (
+    _rich_atlas_metadata,
+    check_gin_status,
+    check_internet_connection,
+)
 
 COMPRESSED_FILENAME = "atlas.tar.gz"
 
@@ -80,13 +84,13 @@ def __init__(
         # Look for this atlas in local brainglobe folder:
         if self.local_full_name is None:
             if self.remote_version is None:
-                raise ValueError(f"{atlas_name} is not a valid atlas name!")
+                check_internet_connection(raise_error=True)
+                check_gin_status(raise_error=True)
 
-            rprint(
-                f"[magenta2]brainglobe_atlasapi: {self.atlas_name} "
-                "not found locally. Downloading...[magenta2]"
-            )
-            self.download_extract_file()
+                # If internet and GIN are up, then the atlas name was invalid
+                raise ValueError(f"{atlas_name} is not a valid atlas name!")
+            else:
+                self.download_extract_file()
 
         # Instantiate after eventual download:
         super().__init__(self.brainglobe_dir / self.local_full_name)
@@ -113,11 +117,11 @@ def remote_version(self):
         """
         remote_url = self._remote_url_base.format("last_versions.conf")
 
-        # Grasp remote version if a connection is available:
         try:
+            # Grasp remote version
             versions_conf = utils.conf_from_url(remote_url)
         except requests.ConnectionError:
-            return
+            return None
 
         try:
             return _version_tuple_from_str(
@@ -161,7 +165,8 @@ def remote_url(self):
 
     def download_extract_file(self):
         """Download and extract atlas from remote url."""
-        utils.check_internet_connection()
+        check_internet_connection()
+        check_gin_status()
 
         # Get path to folder where data will be saved
         destination_path = self.interm_download_dir / COMPRESSED_FILENAME

brainglobe_atlasapi/list_atlases.py

@@ -51,12 +51,20 @@ def get_local_atlas_version(atlas_name):
 
 
 def get_all_atlases_lastversions():
-    """Read from URL all available last versions"""
-    available_atlases = utils.conf_from_url(
-        descriptors.remote_url_base.format("last_versions.conf")
-    )
-    available_atlases = dict(available_atlases["atlases"])
-    return available_atlases
+    """Read from URL or local cache all available last versions"""
+    cache_path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    if utils.check_internet_connection(
+        raise_error=False
+    ) and utils.check_gin_status(raise_error=False):
+        available_atlases = utils.conf_from_url(
+            descriptors.remote_url_base.format("last_versions.conf")
+        )
+    else:
+        print("Cannot fetch latest atlas versions from the server.")
+        available_atlases = utils.conf_from_file(cache_path)
+
+    return dict(available_atlases["atlases"])
 
 
 def get_atlases_lastversions():

brainglobe_atlasapi/utils.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import re
+from pathlib import Path
 from typing import Callable, Optional
 
 import requests
@@ -19,6 +20,8 @@
 from rich.table import Table
 from rich.text import Text
 
+from brainglobe_atlasapi import config
+
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
@@ -119,15 +122,41 @@ def check_internet_connection(
 
     try:
         _ = requests.get(url, timeout=timeout)
+
         return True
-    except requests.ConnectionError:
+    except requests.ConnectionError as e:
         if not raise_error:
             print("No internet connection available.")
         else:
             raise ConnectionError(
                 "No internet connection, try again when you are "
                 "connected to the internet."
-            )
+            ) from e
+
+    return False
+
+
+def check_gin_status(timeout=5, raise_error=True):
+    """Check that the GIN server is up.
+
+    timeout : int
+        timeout to wait for [in seconds] (Default value = 5).
+    raise_error : bool
+        if false, warning but no error.
+    """
+    url = "https://gin.g-node.org/"
+
+    try:
+        _ = requests.get(url, timeout=timeout)
+
+        return True
+    except requests.ConnectionError as e:
+        error_message = "GIN server is down."
+        if not raise_error:
+            print(error_message)
+        else:
+            raise ConnectionError(error_message) from e
+
     return False
 
 
@@ -163,9 +192,9 @@ def retrieve_over_http(
     )
 
     CHUNK_SIZE = 4096
-    response = requests.get(url, stream=True)
 
     try:
+        response = requests.get(url, stream=True)
         with progress:
             tot = int(response.headers.get("content-length", 0))
 
@@ -261,8 +290,8 @@ def get_download_size(url: str) -> int:
         raise IndexError("Improperly formatted URL")
 
 
-def conf_from_url(url):
-    """Read conf file from an URL.
+def conf_from_url(url) -> configparser.ConfigParser:
+    """Read conf file from an URL. And cache a copy in the brainglobe dir.
     Parameters
     ----------
     url : str
@@ -274,6 +303,35 @@ def conf_from_url(url):
 
     """
     text = requests.get(url).text
+    config_obj = configparser.ConfigParser()
+    config_obj.read_string(text)
+    cache_path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    # Cache the available atlases
+    with open(cache_path, "w") as f_out:
+        config_obj.write(f_out)
+
+    return config_obj
+
+
+def conf_from_file(file_path: Path) -> configparser.ConfigParser:
+    """Read conf file from a local file path.
+    Parameters
+    ----------
+    file_path : Path
+        conf file path (obtained from config.get_brainglobe_dir())
+
+    Returns
+    -------
+    conf object if file available
+
+    """
+    if not file_path.exists():
+        raise FileNotFoundError("Last versions cache file not found.")
+
+    with open(file_path, "r") as file:
+        text = file.read()
+
     config = configparser.ConfigParser()
     config.read_string(text)
 

tests/atlasapi/test_list_atlases.py

@@ -1,4 +1,8 @@
+from unittest import mock
+
+from brainglobe_atlasapi import config
 from brainglobe_atlasapi.list_atlases import (
+    get_all_atlases_lastversions,
     get_atlases_lastversions,
     get_downloaded_atlases,
     get_local_atlas_version,
@@ -39,3 +43,71 @@ def test_lastversions():
 def test_show_atlases():
     # TODO add more valid testing than just look for errors when running:
     show_atlases(show_local_path=True)
+
+
+def test_get_all_atlases_lastversions():
+    last_versions = get_all_atlases_lastversions()
+
+    assert "example_mouse_100um" in last_versions
+    assert "osten_mouse_50um" in last_versions
+    assert "allen_mouse_25um" in last_versions
+
+
+def test_get_all_atlases_lastversions_offline():
+    cleanup_cache = False
+    cache_path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    if not cache_path.exists():
+        cache_path.touch()
+        cache_path.write_text(
+            """
+            [atlases]
+            example_mouse_100um = 1.0
+            osten_mouse_50um = 1.0
+            allen_mouse_25um = 1.0
+            """
+        )
+        cleanup_cache = True
+
+    with mock.patch(
+        "brainglobe_atlasapi.utils.check_internet_connection"
+    ) as mock_check_internet_connection:
+        mock_check_internet_connection.return_value = False
+        last_versions = get_all_atlases_lastversions()
+
+        assert "example_mouse_100um" in last_versions
+        assert "osten_mouse_50um" in last_versions
+        assert "allen_mouse_25um" in last_versions
+
+    if cleanup_cache:
+        cache_path.unlink()
+
+
+def test_get_all_atlases_lastversions_gin_down():
+    cleanup_cache = False
+    cache_path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    if not cache_path.exists():
+        cache_path.touch()
+        cache_path.write_text(
+            """
+            [atlases]
+            example_mouse_100um = 1.0
+            osten_mouse_50um = 1.0
+            allen_mouse_25um = 1.0
+            """
+        )
+        cleanup_cache = True
+
+    with mock.patch(
+        "brainglobe_atlasapi.utils.check_gin_status"
+    ) as mock_check_internet_connection:
+        mock_check_internet_connection.return_value = False
+        last_versions = get_all_atlases_lastversions()
+
+        assert "example_mouse_100um" in last_versions
+        assert "osten_mouse_50um" in last_versions
+        assert "allen_mouse_25um" in last_versions
+
+    if cleanup_cache:
+        cache_path.unlink()

tests/atlasapi/test_utils.py

@@ -81,3 +81,60 @@ def test_get_download_size_HTTPError():
 
         with pytest.raises(HTTPError):
             utils.get_download_size(test_url)
+
+
+def test_check_gin_status():
+    # Test with requests.get returning a valid response
+    with mock.patch("requests.get", autospec=True) as mock_request:
+        mock_response = mock.Mock(spec=requests.Response)
+        mock_response.status_code = 200
+        mock_request.return_value = mock_response
+
+        assert utils.check_gin_status()
+
+
+def test_check_gin_status_down():
+    # Test with requests.get returning a 404 response
+    with mock.patch("requests.get", autospec=True) as mock_request:
+        mock_request.side_effect = requests.ConnectionError()
+
+        with pytest.raises(ConnectionError) as e:
+            utils.check_gin_status()
+            assert "GIN server is down" == e.value
+
+
+def test_check_gin_status_down_no_error():
+    # Test with requests.get returning a 404 response
+    with mock.patch("requests.get", autospec=True) as mock_request:
+        mock_request.side_effect = requests.ConnectionError()
+
+        assert not utils.check_gin_status(raise_error=False)
+
+
+def test_conf_from_file(temp_path):
+    conf_path = temp_path / "conf.conf"
+    content = (
+        "[atlases]\n"
+        "example_mouse_100um = 1.2\n"
+        "allen_mouse_10um = 1.2\n"
+        "allen_mouse_25um = 1.2"
+    )
+    conf_path.write_text(content)
+    # Test with a valid file
+    conf = utils.conf_from_file(conf_path)
+
+    assert dict(conf["atlases"]) == {
+        "example_mouse_100um": "1.2",
+        "allen_mouse_10um": "1.2",
+        "allen_mouse_25um": "1.2",
+    }
+
+
+def test_conf_from_file_no_file(temp_path):
+    conf_path = temp_path / "conf.conf"
+
+    # Test with a non-existing file
+    with pytest.raises(FileNotFoundError) as e:
+        utils.conf_from_file(conf_path)
+
+        assert "Last versions cache file not found." == str(e)