[WIP] hash.tsv new format plus linting and auto-adding hashes

.gitignore

@@ -7,6 +7,7 @@ scripts/scramble/lib
 scripts/scramble/archives
 
 # Python virtualenv
+venv
 .venv
 .venv3
 

lib/galaxy/tool_util/deps/mulled/mulled_build_files.py

@@ -58,6 +58,19 @@ def main(argv=None):
             sys.exit(ret)
 
 
+def generate_targets_from_file(target_source_file):
+    line_tuple = FALLBACK_LINE_TUPLE
+    with open(target_source_file) as f:
+        for line in f.readlines():
+            if line:
+                line = line.strip()
+                if line.startswith("#"):
+                    # headers can define a different column order
+                    line_tuple = tuple_from_header(line)
+                else:
+                    yield line_to_targets(line, line_tuple)
+
+
 def generate_targets(target_source):
     """Generate all targets from TSV files in specified file or directory."""
     target_source = os.path.abspath(target_source)
@@ -69,17 +82,7 @@ def generate_targets(target_source):
     for target_source_file in target_source_files:
         # If no headers are defined we use the 4 default fields in the order
         # that has been used in galaxy-tool-util / galaxy-lib < 20.01
-        line_tuple = FALLBACK_LINE_TUPLE
-        with open(target_source_file) as f:
-            for line in f.readlines():
-                if line:
-                    line = line.strip()
-                    if line.startswith("#"):
-                        # headers can define a different column order
-                        line_tuple = tuple_from_header(line)
-                    else:
-                        yield line_to_targets(line, line_tuple)
-
+        yield from generate_targets_from_file(target_source_file)
 
 def tuple_from_header(header):
     fields = header[1:].split("\t")
@@ -105,6 +108,15 @@ def line_to_targets(line_str, line_tuple):
     return line_tuple(*line_parts)
 
 
+def str_from_target(target):
+    target_str = target.package
+    if target.version:
+        target_str += "=" + target.version
+    if target.build:
+        target_str += "--" + target.build
+    return target_str
+
+
 __all__ = ("main",)
 
 

lib/galaxy/tool_util/deps/mulled/util.py

@@ -239,6 +239,17 @@ def _simple_image_name(targets, image_build=None):
     return f"{target.package_name}{suffix}"
 
 
+def sort_build_targets(build_targets):
+    """Sort build targets by package name.
+
+    >>> ordered_targets = [Target(package_name='bioblend=1.0.0', version=None, build=None, package='bioblend=1.0.0'), Target(package_name='galaxyxml=0.4.14', version=None, build=None, package='galaxyxml=0.4.14')]
+    >>> unordered_targets = reversed(ordered_targets)
+    >>> assert ordered_targets == sort_build_targets(ordered_targets)
+    >>> assert ordered_targets == sort_build_targets(unordered_targets)
+    """
+    return sorted(build_targets, key=lambda t: t.package_name)
+
+
 def v1_image_name(targets, image_build=None, name_override=None):
     """Generate mulled hash version 1 container identifier for supplied arguments.
 
@@ -270,7 +281,7 @@ def v1_image_name(targets, image_build=None, name_override=None):
     if len(targets) == 1:
         return _simple_image_name(targets, image_build=image_build)
     else:
-        targets_order = sorted(targets, key=lambda t: t.package_name)
+        targets_order = sort_build_targets(targets)
         requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
         m = hashlib.sha1()
         m.update(requirements_buffer.encode())
@@ -318,7 +329,7 @@ def v2_image_name(targets, image_build=None, name_override=None):
     if len(targets) == 1:
         return _simple_image_name(targets, image_build=image_build)
     else:
-        targets_order = sorted(targets, key=lambda t: t.package_name)
+        targets_order = sort_build_targets(targets)
         package_name_buffer = "\n".join(map(lambda t: t.package_name, targets_order))
         package_hash = hashlib.sha1()
         package_hash.update(package_name_buffer.encode())

test/unit/tool_util/mulled/test_mulled_build_files.py

@@ -7,7 +7,10 @@
 from galaxy.tool_util.deps.mulled.mulled_build_files import (
     FALLBACK_LINE_TUPLE,
     generate_targets,
+    generate_targets_from_file,
+    str_from_target,
 )
+from galaxy.tool_util.deps.mulled.util import build_target
 
 TESTCASES = yaml.safe_load(
     r"""
@@ -55,14 +58,17 @@
       targets: samtools
 """
 )
-TEST_IDS = [next(iter(k.keys())) for k in TESTCASES]
+TESTCASES = {k: v for t in TESTCASES for k, v in t.items()}
+for key in TESTCASES:
+    TESTCASES[key]["equals"]["targets"] = target_str_to_targets(TESTCASES[key]["equals"]["targets"])
 
 
 @pytest.mark.parametrize(
-    "content, equals", [(d[k]["content"], d[k]["equals"]) for k, d in zip(TEST_IDS, TESTCASES)], ids=TEST_IDS
+    "content, equals", 
+    [(test_case["content"], test_case["equals"]) for _, test_case in TESTCASES.items()], 
+    ids=TESTCASES.keys()
 )
 def test_generate_targets(content, equals):
-    equals["targets"] = target_str_to_targets(equals["targets"])
     equals = FALLBACK_LINE_TUPLE(**equals)
     with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
         tmpfile.write(content)
@@ -72,3 +78,33 @@ def test_generate_targets(content, equals):
     assert generated_target.image_build == equals.image_build
     assert generated_target.name_override == equals.name_override
     assert generated_target.base_image == equals.base_image
+
+
+@pytest.mark.parametrize(
+    "content, equals",
+    [(test_case["content"], test_case["equals"]) for _, test_case in TESTCASES.items()], 
+    ids=TESTCASES.keys()
+)
+def test_generate_targets_from_file(content, equals):
+    equals = FALLBACK_LINE_TUPLE(**equals)
+    with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
+        tmpfile.write(content)
+        tmpfile.flush()
+        generated_target = next(generate_targets_from_file(tmpfile.name))
+    assert generated_target.targets == equals.targets
+    assert generated_target.image_build == equals.image_build
+    assert generated_target.name_override == equals.name_override
+    assert generated_target.base_image == equals.base_image
+
+
+def test_str_from_target():
+    target_str = "atarget"
+    assert target_str == str_from_target(build_target(target_str))
+
+    version = "0.1"
+    target_str_versioned = f"{target_str}={version}"
+    assert target_str_versioned == str_from_target(build_target(target_str, version))
+
+    build = "2"
+    target_str_version_build = f"{target_str}={version}--{build}"
+    assert target_str_version_build == str_from_target(build_target(target_str, version, build))