Add export tsv

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
       hooks:
           - id: black
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v0.990
+      rev: v1.1.1
       hooks:
           - id: mypy
             additional_dependencies:

pyproject.toml

@@ -22,7 +22,8 @@ dependencies = [
   "dash",
   "dash-bootstrap-components",
   "opencv-python",
-  "PyYAML"
+  "PyYAML",
+  "shapely",
 ]
 
 classifiers = [

sample_project/input_config.yaml

@@ -2,9 +2,9 @@ videos_dir_path: ./sample_project/videos
 metadata_fields_file_path: ./sample_project/metadata_fields.yaml
 metadata_key_field_str: File
 pose_estimation_results_path: ./sample_project/pose_estimation_results
-pose_estimation_model_str: DLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000
+model_str: DLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000
 dashboard_export_data_path: ./sample_project/output
-roi_names:
+ROI_tags:
   - enclosure
   - left_box
   - top_box
@@ -16,3 +16,5 @@ event_tags:
   - box_1_open
   - box_1_closed
   - experiment_end
+use_ROIs_order_as_hierarchy: False
+buffer_around_ROIs_boundaries: 1e-09

sample_project/videos/jwaspE_nectar-open-close_control.metadata.yaml

@@ -1,10 +1,21 @@
+File: jwaspE_nectar-open-close_control.avi
+Species_name: Ampulex_compressa
 Common_name: jewel_wasp
-Date_end: 5/8/22
+Subject: E_male
+Treatment: nectar-open-close_control
+Treatment_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
+  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
 Date_start: 5/8/22
-File: jwaspE_nectar-open-close_control.avi
-Further_description: '-'
+Time_start: '11:47:53'
+Date_end: 5/8/22
+Time_end: '12:19:29'
+Time_recorded: 00:31:36
+Video_length: 00:31:36
 Hardware_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
   or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
+Software_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
+  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
+Further_description: '-'
 ROIs:
 - drawn_on_frame: 57000
   line_color: '#2E91E5'
@@ -30,17 +41,6 @@ ROIs:
   line_color: '#222A2A'
   name: tube
   path: M722.1672180890754,559.5676484911008L718.4197091286674,572.4757349102841L718.4197091286674,584.5510415604879L720.0852686666265,586.216601098447L731.7441854323404,588.2985505208959L741.3211527756054,592.0460594813039L755.0620189637682,594.1280089037529L768.3864952674413,597.4591279796712L778.7962423796859,600.3738571710996L785.4584805315224,602.8721964780383L799.1993467196852,605.7869256694668L814.1893825613174,610.7836042833442L816.6877218682562,609.1180447453851L820.0188409441744,602.0394167090587L823.7663499045824,596.2099583262018L825.0155195580518,589.1313302898755L825.4319094425416,586.216601098447Z
-Software_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
-  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
-Species_name: Ampulex_compressa
-Subject: E_male
-Time_end: '12:19:29'
-Time_recorded: 00:31:36
-Time_start: '11:47:53'
-Treatment: nectar-open-close_control
-Treatment_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
-  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
-Video_length: 00:31:36
 Events:
   experiment_start: 0
   box_1_open: 25123

wazp/callbacks/dashboard.py

@@ -237,6 +237,14 @@ def create_buttons_and_message() -> html.Div:
         style={"margin-right": "10px", "margin-left": "10px"},
     )
 
+    export_options = dcc.Checklist(
+        id="export-options",
+        options=["as .h5 file", "as .tsv file"],
+        value=["as .h5 file"],
+        style={"margin-left": "10px"},
+        labelStyle={"display": "flex", "align-items": "center"},
+    )
+
     export_message = dbc.Alert(
         children="Data export message",
         id="export-message",
@@ -256,6 +264,7 @@ def create_buttons_and_message() -> html.Div:
             select_all_videos_button,
             unselect_all_videos_button,
             export_button,
+            export_options,
             export_message,
         ]
     )
@@ -341,6 +350,7 @@ def create_video_data_table_slider_and_buttons(
         Output("pose-data-unavailable-message", "message"),
         Output("pose-data-unavailable-message", "displayed"),
         Output("export-message", "children"),
+        Output("export-options", "value"),
         Output("export-message", "is_open"),
         Output("export-message", "color"),
         Input("video-data-table", "selected_rows"),
@@ -353,6 +363,7 @@ def create_video_data_table_slider_and_buttons(
         State("pose-data-unavailable-message", "message"),
         State("pose-data-unavailable-message", "displayed"),
         State("export-message", "children"),
+        State("export-options", "value"),
         State("export-message", "is_open"),
         State("export-message", "color"),
         State("session-storage", "data"),
@@ -367,11 +378,12 @@ def modify_rows_selection(
         slider_marks: dict,
         pose_unavail_message_str: str,
         pose_unavail_message_state: bool,
-        export_message_str,
-        export_message_state,
-        export_message_color,
+        export_message_children: list,
+        export_option_selected: list,
+        export_message_state: bool,
+        export_message_color: str,
         app_storage: dict,
-    ) -> tuple[list, int, int, int, str, bool, str, bool, str]:
+    ) -> tuple[list, int, int, int, str, bool, list, list, bool, str]:
         """Modify the selection status of the rows in the videos table.
 
         A row's selection status (i.e., its checkbox) is modified if (1) the
@@ -401,8 +413,10 @@ def modify_rows_selection(
             text content of the 'pose data unavailable' message
         pose_unavail_message_state : bool
             visibility state of the 'pose data unavailable' message
-        export_message_str : _type_
+        export_message_children : _type_
             text content of the export message
+        export_message_opt : str
+            export option selected
         export_message_state : _type_
             visibility state of the export message
         export_message_color : _type_
@@ -425,15 +439,17 @@ def modify_rows_selection(
             text content of the 'pose data unavailable' message
         pose_unavail_message_state : bool
             visibility state of the 'pose data unavailable' message
-        export_message_str : str
-            text content of the export message
+        export_message_children : str
+            list of children for export message component
+        export_message_opt : str
+            export option selected
         export_message_state : bool
             visibility state of the export message
         export_message_color : str
             color of the export message
         """
 
-        # TODO: select all rows *per page*
+        # TODO: select all rows per page?
         list_missing_pose_data_bool = [
             videos_table_data[r][POSE_DATA_STR] == FALSE_EMOJI
             for r in range(len(videos_table_data))
@@ -472,16 +488,16 @@ def modify_rows_selection(
         # ---------------------------
         # If export button is clicked
         if n_clicks_export > 0:
-            # if no rows selected show warning
+            # if no rows are selected show warning
             if not list_selected_rows:
                 n_clicks_export = 0
 
-                export_message_str = "No data to export"
-                # TODO: add timestamp of message?
+                export_message_children = ["No data to export"]
                 export_message_color = "warning"
                 export_message_state = True
 
-            # if rows are selected: export combined df
+            # if rows are selected: export combined
+            # dataframe
             else:
 
                 # get list of selected videos
@@ -500,7 +516,7 @@ def modify_rows_selection(
 
                 # get list of dataframes to combine
                 # - one dataframe per selected video
-                # - we add 'File', 'event_tag' and 'ROI' data to each dataframe
+                # - we add the fields 'video_file', 'event_tag' and 'ROI'
                 # - we select only the frames within the interval
                 #   set by the slider
                 list_df_to_export = utils.get_dataframes_to_combine(
@@ -513,43 +529,92 @@ def modify_rows_selection(
                 df = pd.concat(list_df_to_export)
 
                 # ---------
-                # Save df as h5
-                # get output path
+                # Export dataframe as h5
+                # TODO: provide an option to export as h5 or csv?
+
+                # Get path to output directory
                 # if not specified in config, use dir where
-                # 'start_wazp_server.sh' is at
+                # server was launched from
                 output_path = pl.Path(
                     app_storage["config"].get(
                         "dashboard_export_data_path", "."
                     )
                 )
 
-                # if output dir does not exist, create it
+                # If output directory does not exist,
+                # create it
                 if not output_path.is_dir():
                     os.mkdir(output_path)
 
-                # save combined df as h5 file
-                h5_file_path = output_path / pl.Path(
+                # Filepath without extension
+                file_path = output_path / pl.Path(
                     "df_export_"
                     + datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-                    + ".h5"
-                )
-                df.to_hdf(
-                    h5_file_path,
-                    key="df",
-                    mode="w",
                 )
 
-                # ---------
-                # reset triggers and states
-                list_selected_rows = []
-                n_clicks_export = 0
-                export_message_str = (
-                    "Combined dataframe ",
-                    f"exported successfully at: '{h5_file_path}'",
-                )
-                # TODO: add timestamp to message?
-                export_message_color = "success"
-                export_message_state = True
+                # Export combined dataframe as h5 and/or tsv file
+                if not export_option_selected:
+                    n_clicks_export = 0
+                    export_message_children = ["No export format selected"]
+                    export_message_color = "warning"
+                    export_message_state = True
+
+                else:
+                    list_suffixes = []
+                    if "as .h5 file" in export_option_selected:
+                        suffix = ".h5"
+                        df.to_hdf(  # ---- changes fn
+                            file_path.with_suffix(
+                                suffix
+                            ),  # ---- changes suffix
+                            key="df",  # ---changes args
+                            mode="w",
+                        )
+                        list_suffixes.append(suffix)
+
+                    if "as .tsv file" in export_option_selected:
+                        suffix = ".tsv"
+                        df.to_csv(
+                            file_path.with_suffix(suffix),
+                            sep="\t",
+                            index=False,
+                        )
+                        list_suffixes.append(suffix)
+                        # NOTE: to read, use pandas.read_csv with sep='\t',
+                        # and keep_default_na=False
+                        # TODO: should I print this out too in the message?
+
+                    # ---------
+                    # Reset triggers and states
+                    list_selected_rows = []
+                    n_clicks_export = 0
+
+                    # Successful export message
+                    export_message_base = [
+                        "Combined dataframe successfully exported at: "
+                    ]
+                    # interleave paths to exported data with line break
+                    # components (html.Br())
+                    # TODO: to combine with clipboard functionality, I'll
+                    # need to interleave here two clipboard components,
+                    # one after each path
+                    export_message_w_line_breaks = [
+                        val
+                        for pair in zip(
+                            [html.Br()] * len(list_suffixes),
+                            [
+                                f"'{file_path.with_suffix(x)}'"
+                                for x in list_suffixes
+                            ],
+                        )
+                        for val in pair
+                    ]
+                    export_message_children = (
+                        export_message_base + export_message_w_line_breaks
+                    )
+                    export_message_color = "success"
+                    export_message_state = True
+                    export_option_selected = []
 
         return (
             list_selected_rows,
@@ -558,7 +623,8 @@ def modify_rows_selection(
             n_clicks_export,
             pose_unavail_message_str,
             pose_unavail_message_state,
-            export_message_str,
+            export_message_children,
+            export_option_selected,
             export_message_state,
             export_message_color,
         )

wazp/callbacks/roi.py

@@ -109,7 +109,7 @@ def update_roi_select_options(
         if "config" in app_storage.keys():
             # Get ROI names from stored config
             config = app_storage["config"]
-            roi_names = config["roi_names"]
+            roi_names = config["ROI_tags"]
             options = [{"label": r, "value": r} for r in roi_names]
             value = roi_names[0]