Merge pull request #446 from ttngu207/datajoint_pipeline

Add `ExperimentTimeline`, fix SLEAP `anchor_part` ingestion, fix pyrat API

aeon/dj_pipeline/acquisition.py

@@ -166,6 +166,24 @@ def get_data_directories(cls, experiment_key, directory_types=None, as_posix=Fal
         ]
 
 
+@schema
+class ExperimentTimeline(dj.Manual):
+    definition = """  # different parts of an experiment timeline
+    -> Experiment
+    name: varchar(32)  # e.g. presocial, social, postsocial
+    ---
+    start: datetime
+    end: datetime
+    note='': varchar(1000)
+    """
+
+    class Subject(dj.Part):
+        definition = """  # the subjects participating in this part of the experiment timeline
+        -> master
+        -> Experiment.Subject
+        """
+
+
 # ------------------- ACQUISITION EPOCH --------------------
 
 

aeon/dj_pipeline/subject.py

@@ -64,6 +64,7 @@ def make(self, key):
             "o": 0,
             "l": 10,
             "eartag": eartag_or_id,
+            "state": ["live", "sacrificed", "exported"],
         }
         animal_resp = get_pyrat_data(endpoint="animals", params=params)
         if len(animal_resp) == 0:
@@ -105,6 +106,7 @@ def make(self, key):
             "strain_id": animal_resp["strain_id"],
             "cage_number": animal_resp["cagenumber"],
             "lab_id": animal_resp["labid"],
+            "available": animal_resp.get("state", "") == "live",
         }
         if animal_resp["gen_bg_id"] is not None:
             GeneticBackground.insert1(
@@ -250,7 +252,10 @@ def make(self, key):
         new_eartags = []
         for responsible_id in lab.User.fetch("responsible_id"):
             # 1 - retrieve all animals from this user
-            animal_resp = get_pyrat_data(endpoint="animals", params={"responsible_id": responsible_id})
+            animal_resp = get_pyrat_data(
+                endpoint="animals",
+                params={"responsible_id": responsible_id, "state": ["live", "sacrificed", "exported"]}
+            )
             for animal_entry in animal_resp:
                 # 2 - find animal with comment - Project Aeon
                 eartag_or_id = animal_entry["eartag_or_id"]
@@ -318,7 +323,7 @@ def make(self, key):
             if e.args[0].endswith("response code: 404"):
                 SubjectDetail.update1(
                     {
-                        **key,
+                        "subject": key["subject"],
                         "available": False,
                     }
                 )
@@ -347,6 +352,21 @@ def make(self, key):
             # compute/update reference weight
             SubjectReferenceWeight.get_reference_weight(eartag_or_id)
         finally:
+            # recheck for "state" to see if the animal is still available
+            animal_resp = get_pyrat_data(
+                endpoint="animals",
+                params={"k": ["labid", "state"],
+                        "eartag": eartag_or_id,
+                        "state": ["live", "sacrificed", "exported"]})
+            animal_resp = animal_resp[0]
+            SubjectDetail.update1(
+                {
+                    "subject": key["subject"],
+                    "available": animal_resp.get("state", "") == "live",
+                    "lab_id": animal_resp["labid"],
+                }
+            )
+
             completion_time = datetime.utcnow()
             self.insert1(
                 {

aeon/dj_pipeline/tracking.py

@@ -188,22 +188,18 @@ def make(self, key):
         class_names = np.unique(pose_data.identity)
         identity_mapping = {n: i for i, n in enumerate(class_names)}
 
+        # get anchor part
+        # this logic is valid only if the different animals have the same skeleton and anchor part
+        #   which should be the case within one chunk
+        anchor_part = next(v.replace("_x", "") for v in stream_reader.columns if v.endswith("_x"))
+
         # ingest parts and classes
         pose_identity_entries, part_entries = [], []
         for identity in identity_mapping:
             identity_position = pose_data[pose_data["identity"] == identity]
             if identity_position.empty:
                 continue
 
-            # get anchor part - always the first one of all the body parts
-            # FIXME: the logic below to get "anchor_part" is not robust, it relies on the ordering of the unique parts
-            #  but if there are missing frames for the actual anchor part, it will be missed
-            #  and another part will be incorrectly chosen as "anchor_part"
-            #  (2024-10-31) - we recently discovered that the parts are not sorted in the same order across frames
-            #               - further exacerbating the flaw in the logic below
-            #  best is to find a robust way to get the anchor part info from the config file for this chunk
-            anchor_part = np.unique(identity_position.part)[0]
-
             for part in set(identity_position.part.values):
                 part_position = identity_position[identity_position.part == part]
                 part_entries.append(

aeon/dj_pipeline/webapps/sciviz/specsheet.yaml

@@ -214,6 +214,50 @@ SciViz:
                 - type: attribute
                   input: Experiment Type
                   destination: experiment_type
+
+            New Experiment Timeline:
+              route: /exp_timeline_form
+              x: 0
+              y: 1.6
+              height: 0.4
+              width: 1
+              type: form
+              tables:
+                - aeon_acquisition.ExperimentTimeline
+              map:
+                - type: table
+                  input: Experiment Name
+                  destination: aeon_acquisition.Experiment
+                - type: attribute
+                  input: Timeline Name
+                  destination: name
+                - type: attribute
+                  input: Start Time
+                  destination: start
+                - type: attribute
+                  input: End Time
+                  destination: end
+                - type: attribute
+                  input: Note
+                  destination: note
+
+            New Timeline Subject:
+              route: /exp_timeline_subject_form
+              x: 0
+              y: 2.0
+              height: 0.3
+              width: 1
+              type: form
+              tables:
+                - aeon_acquisition.ExperimentTimeline.Subject
+              map:
+                - type: table
+                  input: Experiment Name
+                  destination: aeon_acquisition.ExperimentTimeline
+                - type: table
+                  input: Subject participating in this timeline
+                  destination: aeon_acquisition.Experiment.Subject
+
     Subjects:
       route: /subjects
       grids: