Merge branch 'release/v0.1.5'

ebi-gene-expression-group · May 4, 2022 · 53f427e · 53f427e
2 parents 943cde5 + fa364e7
commit 53f427e
Show file tree

Hide file tree

Showing 9 changed files with 154 additions and 90 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -0,0 +1,46 @@
+name: Test experiment_metadata tasks
+
+on: [pull_request]
+
+jobs:
+  setup:
+    name: ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Add workspace to path
+        run: |
+          echo "${GITHUB_WORKSPACE}" >> $GITHUB_PATH
+
+      - name: Cache conda
+        uses: actions/cache@v1
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ matrix.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles('test-environment.yml') }}
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: test
+          environment-file: test-environment.yml
+          python-version: 3.6
+          channels: ebi-gene-expression-group,conda-forge,bioconda,defaults
+          allow-softlinks: true
+          channel-priority: flexible
+          show-channel-urls: true
+          use-only-tar-bz2: true
+
+      - name: Run tests
+        run: |
+          atlas-experiment-metadata-test.bats
diff --git a/.travis.yml b/.travis.yml
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.1.4
+0.1.5
diff --git a/atlas-experiment-metadata-test.bats b/atlas-experiment-metadata-test.bats
@@ -12,11 +12,14 @@ setup() {
     implicit_pl_out_dir='implicit_pl'
     explicit_pl_out="${explicit_pl_out_dir}/E-MTAB-6077.condensed-sdrf.tsv"
     implicit_pl_out="${implicit_pl_out_dir}/E-MTAB-6077.condensed-sdrf.tsv"
+    implicit_pl_output_sdrf="${implicit_pl_out_dir}/${test_exp_acc}.sdrf.txt"
     explicit_sc_sh_out_dir='explicit_sc_sh'
     implicit_sc_sh_out_dir='implicit_sc_sh'
-    zooma_exclusions="test_data/zooma_exclusions.yml"
+    implicit_sc_sh_exc_out_dir='implicit_sc_sh_env_exc'
+    zooma_exclusions="$test_data_dir/zooma_exclusions.yml"
     explicit_sc_sh_out="${explicit_sc_sh_out_dir}/E-MTAB-6077.condensed-sdrf.tsv"
     implicit_sc_sh_out="${implicit_sc_sh_out_dir}/E-MTAB-6077.condensed-sdrf.tsv"
+    implicit_sc_sh_out_env_exc="${implicit_sc_sh_exc_out_dir}/E-MTAB-6077.condensed-sdrf.tsv"
     celltype_fields="inferred cell type"
 }
 
@@ -32,18 +35,19 @@ setup() {
     [ -f "$explicit_pl_out" ]
 }
 
-@test "Test single-cell condense perl script with implicit IDF" {
+@test "Test single-cell condense perl script with implicit IDF and SDRF copy" {
     if [ -f "$implicit_pl_out" ]; then
         skip "Output from implicit condense pl exists"
     fi
 
-    run mkdir -p $implicit_pl_out_dir && env ATLAS_PROD=$test_data_dir $condense_pl -sc -e $test_exp_acc -o $implicit_pl_out_dir
+    run mkdir -p $implicit_pl_out_dir && env ATLAS_PROD=$test_data_dir $condense_pl -sc -s -e $test_exp_acc -o $implicit_pl_out_dir
 
     [ "$status" -eq 0 ]
     [ -f "$implicit_pl_out" ]
+    [ -f "$implicit_pl_output_sdrf" ]
 }
 
-@test "Test single-cell condense wrapper with explicit IDF" {
+@test "Test single-cell condense wrapper with explicit IDF and Zooma" {
     if [ -f "$explicit_sc_sh_out" ]; then
         skip "Output from SC sh condense wrapper exists"
     fi
@@ -54,7 +58,7 @@ setup() {
     [ -f "$explicit_sc_sh_out" ]
 }
 
-@test "Test single-cell condense wrapper with implicit IDF" {
+@test "Test single-cell condense wrapper with implicit IDF and Zooma" {
     if [ -f "$implicit_sc_sh_out" ]; then
         skip "Output from SC sh condense wrapper exists"
     fi
@@ -65,6 +69,17 @@ setup() {
     [ -f "$implicit_sc_sh_out" ]
 }
 
+@test "Test single-cell condense wrapper with implicit IDF and Zooma from env" {
+    if [ -f "$implicit_sc_sh_out_env_exc" ]; then
+        skip "Output from SC sh condense wrapper exists"
+    fi
+
+    run mkdir -p $implicit_sc_sh_exc_out_dir && env ATLAS_SC_EXPERIMENTS=$test_data_dir ATLAS_META_CONFIG=$test_data_dir bash single_cell_condensed_sdrf.sh -t "$celltype_fields" -e E-MTAB-6077 -o $implicit_sc_sh_exc_out_dir
+
+    [ "$status" -eq 0 ]
+    [ -f "$implicit_sc_sh_out_env_exc" ]
+}
+
 @test "Test unmelt for condensed SDRFs" {
     if [ -f "$test_unmelted_sdrf" ]; then
         skip "Output from unmelt exists"

diff --git a/condense_sdrf.pl b/condense_sdrf.pl
@@ -45,6 +45,10 @@ =head1 OPTIONS
 
 Optional. Copy IDF file from ArrayExpress load directory to output directory.
 
+=item -s --copySDRF
+
+Optional. Copy SDRF file from the IDF determined location to the output directory.
+
 =item -o --outdir
 
 Optional. Destination directory for output file(s). Will use current working
@@ -101,10 +105,12 @@ =head1 AUTHOR
     get_idfFile_path
     get_singlecell_idfFile_path
 );
+use Atlas::Util qw( get_supporting_file);
 use Atlas::ZoomaClient;
 use Atlas::ZoomaClient::MappingResult;
 use Atlas::AtlasConfig::Reader qw( parseAtlasFactors );
 use File::Basename;
+use File::Spec;
 use Data::Dumper;
 
 $| = 1;
@@ -164,8 +170,13 @@ =head1 AUTHOR
         ignore_datafiles    => 1
     });
 
-my $magetab = $reader->parse;
+my ($investigation, $magetab) = $reader->parse;
 $logger->info( "Successfully read MAGETAB." );
+
+if( $args->{ "copySDRF" } ) {
+    copy_sdrf_to_output_dir($investigation, $args->{ "output_directory" }, $idfFile);
+}
+
 $logger->info( "Merging technical replicates if available.") if( $args->{ "mergeTechReplicates" } );
 my $atlasAssays = create_all_atlas_assays( $magetab, $args->{ "mergeTechReplicates" } );
 $logger->debug( Dumper( $atlasAssays ) );
@@ -247,6 +258,7 @@ sub parse_args {
         "z|zooma"           => \$args{ "zooma" },
         "x|zoomaExclusions=s" => \$args{ "zooma_exclusions_path" },
         "i|idf"             => \$args{ "idf" },
+        "s|copySDRF"        => \$args{"copySDRF"},
         "f|factors=s"       => \$args{ "factors_file" },
         "b|bioreps"         => \$args{ "bioreps" },
         "d|debug"           => \$args{ "debug" },
@@ -274,10 +286,12 @@ sub parse_args {
         print "WARN  - No output directory specified, will write output files in ", Cwd::cwd(), "\n";
         $args{ "output_directory" } = Cwd::cwd();
     }
-    unless($args{ "zooma_exclusions_path" }) {
-        my $defaultExclusionsFile="$abs_path/../supporting_files/zooma_exclusions.yml";
-        print "Using default exclusions file path of $defaultExclusionsFile\n";
-        $args{ "zooma_exclusions_path" } = $defaultExclusionsFile ;
+    if($args{ "zooma" } ){
+        unless($args{ "zooma_exclusions_path" }) {
+            my $defaultExclusionsFile=get_supporting_file('zooma_exclusions.yml');
+            print "Using default exclusions file path of $defaultExclusionsFile\n";
+            $args{ "zooma_exclusions_path" } = $defaultExclusionsFile ;
+        }
     }
 
     # If one was specified, check that it's writable and die if not.
@@ -308,6 +322,25 @@ sub copy_idf_from_ae {
     }
 }
 
+sub copy_sdrf_to_output_dir {
+    my ( $investigation, $output_dir, $idf_abs_path ) = @_;
+    foreach my $sdrf ( @{ $investigation->get_sdrfs() } ) {
+        my $filename = $sdrf->get_uri()->file();
+        if( !File::Spec->file_name_is_absolute( $filename ) ) {
+            # append IDF path
+            my $dir = dirname($idf_abs_path);
+            $filename = File::Spec->catfile( $dir, $filename );
+        } 
+        `cp $filename $output_dir`;
+        unless( $? ) {
+            $logger->info( "Successfully copied SDRF." );
+        }
+        else {
+            $logger->logdie( "Could not copy SDRF: $!" );
+        }
+    }
+}
+
 sub create_all_atlas_assays {
     my ( $magetab, $mergeTechReplicates ) = @_;
     # Create a relaxed AtlasAssayFactory.

diff --git a/run_zooma_condensed.pl b/run_zooma_condensed.pl
@@ -131,7 +131,6 @@ sub parse_args {
 
 sub run_zooma_mapping {
     my ( $allPropertiesAssays, $cell2organism, $zoomaExclusionsFilename, $zoomificationsFilename, $expAcc ) = @_;
-    #my $zoomaExclusionsFile = "$abs_path/../supporting_files/zooma_exclusions.yml";
     my $zoomaExclusions = Config::YAML->new(
         config => $zoomaExclusionsFilename
     ) if $zoomaExclusionsFilename;

diff --git a/single_cell_condensed_sdrf.sh b/single_cell_condensed_sdrf.sh
@@ -18,7 +18,7 @@ expId="$EXP_ID"
 idfFile=
 experimentDir="$ATLAS_SC_EXPERIMENTS"
 skipZooma="$SKIP_ZOOMA"
-zoomaExclusions="$scriptDir/../supporting_files/zooma_exclusions.yml"
+zoomaExclusions="$ATLAS_META_CONFIG/zooma_exclusions.yml"
 outputDir=
 
 while getopts ":e:f:s:o:z:t:" o; do
@@ -59,7 +59,7 @@ if [ -z "$outputDir" ]; then
 fi
 
 if [ -z "$cellGroupTypes" ]; then
-    cellGroupTypes="inferred cell type - ontology labels,inferred cell type - authors labels"
+    cellGroupTypes="authors cell type - ontology labels,authors cell type,inferred cell type - ontology labels,inferred cell type - authors labels"
 fi
 
 # If an actual file is specified, we can pass that directly
@@ -202,7 +202,7 @@ use_cell_types_In_condensed() {
 
       if [ -n "$col_num_ct" ]; then 
         awk -v fieldName="$field_to_extract" -F'\t' 'BEGIN { OFS = "\t" } NR == FNR { cell[$1]; type[$1]=$2; next } $3 in cell { print $1, $2, $3, "factor", fieldName, type[$3] }' \
-            <( awk -F'\t' -v cellCol=$col_num_cell_id -v ctCol=$col_num_ct 'BEGIN { OFS = "\t" } { print $cellCol, $ctCol }' $CT ) \
+            <( awk -F'\t' -v cellCol=$col_num_cell_id -v ctCol=$col_num_ct 'BEGIN { OFS = "\t" } {if ($ctCol ~ /\S/) print $cellCol, $ctCol }' $CT ) \
             <( awk -F'\t' 'BEGIN { OFS = "\t" } {print $1, $2, $3}' $COND | sort | uniq) > $COND\.with_ct
 
         actual_cell_type_field=$(head -1 $CT | awk -F"\t" -v col=$col_num_ct '{print $col}')

diff --git a/test-environment.yml b/test-environment.yml
@@ -0,0 +1,10 @@
+name: test
+dependencies:
+  - bats
+  - coreutils
+  - perl
+  - perl-atlas-modules>=0.3.1
+  - libgfortran
+  - r-optparse=1.6.0
+  - r-reshape2
+  - r-data.table
diff --git a/updateExperimentMetadata.sh b/updateExperimentMetadata.sh
@@ -27,6 +27,7 @@ fi
 
 notifEmail=$1
 mode=$2
+retryWithoutZooma=${3:-yes}
 
 checkZooma() {
   log=$1
@@ -165,43 +166,49 @@ while [ $jobCnt -lt $numLsfJobsSubmitted ]; do
       # Log error condition, but don't fail - condense SDRF without Zooma mapping.
       errors=$(cat $lsfErr)
       errors="$errors"`grep "Failed to query ZOOMA" $lsfOut | grep ":ERROR"`
+      isZoomaError=$?
       errors="$errors"`grep "Exception:" $lsfOut`
       errors="$errors"`grep "^ERROR" $lsfOut`
 
-      if [ ! -z "$errors" ]; then
+      if [ -n "$errors" ]; then
         echo -e "\n\nErrors for ${expAcc} (condense_sdrf.pl call FAILED): " >> $log
         echo -e $errors >> $log
       fi
 
-      echo "Condensing SDRF $expAcc without Zooma mapping..." >> $log
-
-      if [ "$mode" == "atlas" ]; then
-        # Get the experiment type from the experiment config.
-        expType=$(get_experiment_type_from_xml.pl $expAcc/$expAcc-configuration.xml)
-        if [ $? -ne 0 ]; then
-          echo "ERROR: failed to get $expAcc experiment type from XML config. Cannot generate condensed SDRF."
-          exit 1
-        fi
-
-        if [[ $expType == *baseline ]]; then
-          condense_sdrf.pl -e $expAcc -f $expAcc/$expAcc-factors.xml -i -o $expAcc
-        else
-          condense_sdrf.pl -e $expAcc -i -o $expAcc
-        fi
-
-      elif [ "$mode" == "single_cell" ]; then
-        export EXP_ID=$expAcc
-        export SKIP_ZOOMA=yes
-        single_cell_condensed_sdrf.sh
-
-      elif [ "$mode" == "irap_single_lib" ]; then
-        condense_sdrf.pl -e $expAcc -i -o $expAcc -b
-      fi
-
-      if [ $? -ne 0 ]; then
-        echo -e "\nFailed to condense SDRF for $expAcc without Zooma mappings, following error from trying with Zooma mappings" >> $log
+      if [ "$isZoomaError" -eq '0' ] && [ "$retryWithoutZooma" = 'yes' ]; then
+
+          echo "Condensing SDRF $expAcc without Zooma mapping..." >> $log
+
+          if [ "$mode" == "atlas" ]; then
+            # Get the experiment type from the experiment config.
+            expType=$(get_experiment_type_from_xml.pl $expAcc/$expAcc-configuration.xml)
+            if [ $? -ne 0 ]; then
+              echo "ERROR: failed to get $expAcc experiment type from XML config. Cannot generate condensed SDRF."
+              exit 1
+            fi
+
+            if [[ $expType == *baseline ]]; then
+              condense_sdrf.pl -e $expAcc -f $expAcc/$expAcc-factors.xml -i -o $expAcc
+            else
+              condense_sdrf.pl -e $expAcc -i -o $expAcc
+            fi
+
+          elif [ "$mode" == "single_cell" ]; then
+            export EXP_ID=$expAcc
+            export SKIP_ZOOMA=yes
+            single_cell_condensed_sdrf.sh
+
+          elif [ "$mode" == "irap_single_lib" ]; then
+            condense_sdrf.pl -e $expAcc -i -o $expAcc -b
+          fi
+
+          if [ $? -ne 0 ]; then
+            echo -e "\nFailed to condense SDRF for $expAcc without Zooma mappings, following error from trying with Zooma mappings" >> $log
+          else
+            echo "Done" >> $log
+          fi
       else
-        echo "Done" >> $log
+        echo -e "\nFailed to condense SDRF for $expAcc, This is "$(if [ "$isZoomaError" -ne '0' ]; then echo "not a zooma error"; else echo "an issue with Zooma mapping"; fi)  >> $log
       fi
 
       rm -rf $lsfOut