From 8358ce4b6923918920efb77d09adc769721dbb9b Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 6 Nov 2024 14:00:36 -0600 Subject: [PATCH 1/4] fix(tracking): improve logic for retrieving `anchor_part` --- aeon/dj_pipeline/tracking.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/aeon/dj_pipeline/tracking.py b/aeon/dj_pipeline/tracking.py index 0c2ff9c6..584e0384 100644 --- a/aeon/dj_pipeline/tracking.py +++ b/aeon/dj_pipeline/tracking.py @@ -188,6 +188,11 @@ def make(self, key): class_names = np.unique(pose_data.identity) identity_mapping = {n: i for i, n in enumerate(class_names)} + # get anchor part + # this logic is valid only if the different animals have the same skeleton and anchor part + # which should be the case within one chunk + anchor_part = next(v.replace("_x", "") for v in stream_reader.columns if v.endswith("_x")) + # ingest parts and classes pose_identity_entries, part_entries = [], [] for identity in identity_mapping: @@ -195,15 +200,6 @@ def make(self, key): if identity_position.empty: continue - # get anchor part - always the first one of all the body parts - # FIXME: the logic below to get "anchor_part" is not robust, it relies on the ordering of the unique parts - # but if there are missing frames for the actual anchor part, it will be missed - # and another part will be incorrectly chosen as "anchor_part" - # (2024-10-31) - we recently discovered that the parts are not sorted in the same order across frames - # - further exacerbating the flaw in the logic below - # best is to find a robust way to get the anchor part info from the config file for this chunk - anchor_part = np.unique(identity_position.part)[0] - for part in set(identity_position.part.values): part_position = identity_position[identity_position.part == part] part_entries.append( From 61ec7aef765bcaa8485436d2529054bf598037c5 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 6 Nov 2024 14:00:54 -0600 Subject: [PATCH 2/4] feat(acquisition): add `ExperimentTimeline` table --- aeon/dj_pipeline/acquisition.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index e943adc9..b5eb9929 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -166,6 +166,24 @@ def get_data_directories(cls, experiment_key, directory_types=None, as_posix=Fal ] +@schema +class ExperimentTimeline(dj.Manual): + definition = """ # different parts of an experiment timeline + -> Experiment + name: varchar(32) # e.g. presocial, social, postsocial + --- + start: datetime + end: datetime + note='': varchar(1000) + """ + + class Subject(dj.Part): + definition = """ # the subjects participating in this part of the experiment timeline + -> master + -> Experiment.Subject + """ + + # ------------------- ACQUISITION EPOCH -------------------- From f564c3b4bbca0fa6234378de3beead4a5da6b8d7 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 6 Nov 2024 14:53:18 -0600 Subject: [PATCH 3/4] update(subject): update logic to keep up with the latest changes in Pyrat API In the latest pyrat API - by default, only subjects with `state='live'` is returned --- aeon/dj_pipeline/subject.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/subject.py b/aeon/dj_pipeline/subject.py index 09027961..18aee718 100644 --- a/aeon/dj_pipeline/subject.py +++ b/aeon/dj_pipeline/subject.py @@ -64,6 +64,7 @@ def make(self, key): "o": 0, "l": 10, "eartag": eartag_or_id, + "state": ["live", "sacrificed", "exported"], } animal_resp = get_pyrat_data(endpoint="animals", params=params) if len(animal_resp) == 0: @@ -105,6 +106,7 @@ def make(self, key): "strain_id": animal_resp["strain_id"], "cage_number": animal_resp["cagenumber"], "lab_id": animal_resp["labid"], + "available": animal_resp.get("state", "") == "live", } if animal_resp["gen_bg_id"] is not None: GeneticBackground.insert1( @@ -250,7 +252,10 @@ def make(self, key): new_eartags = [] for responsible_id in lab.User.fetch("responsible_id"): # 1 - retrieve all animals from this user - animal_resp = get_pyrat_data(endpoint="animals", params={"responsible_id": responsible_id}) + animal_resp = get_pyrat_data( + endpoint="animals", + params={"responsible_id": responsible_id, "state": ["live", "sacrificed", "exported"]} + ) for animal_entry in animal_resp: # 2 - find animal with comment - Project Aeon eartag_or_id = animal_entry["eartag_or_id"] @@ -318,7 +323,7 @@ def make(self, key): if e.args[0].endswith("response code: 404"): SubjectDetail.update1( { - **key, + "subject": key["subject"], "available": False, } ) @@ -347,6 +352,21 @@ def make(self, key): # compute/update reference weight SubjectReferenceWeight.get_reference_weight(eartag_or_id) finally: + # recheck for "state" to see if the animal is still available + animal_resp = get_pyrat_data( + endpoint="animals", + params={"k": ["labid", "state"], + "eartag": eartag_or_id, + "state": ["live", "sacrificed", "exported"]}) + animal_resp = animal_resp[0] + SubjectDetail.update1( + { + "subject": key["subject"], + "available": animal_resp.get("state", "") == "live", + "lab_id": animal_resp["labid"], + } + ) + completion_time = datetime.utcnow() self.insert1( { From 007512a9b96d9b75adf67bdf4acc04b441611793 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 6 Nov 2024 15:04:18 -0600 Subject: [PATCH 4/4] feat(sciviz): add entry form for `ExperimentTimeline` --- .../dj_pipeline/webapps/sciviz/specsheet.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml index 3515bde8..629ee571 100644 --- a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml +++ b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml @@ -214,6 +214,50 @@ SciViz: - type: attribute input: Experiment Type destination: experiment_type + + New Experiment Timeline: + route: /exp_timeline_form + x: 0 + y: 1.6 + height: 0.4 + width: 1 + type: form + tables: + - aeon_acquisition.ExperimentTimeline + map: + - type: table + input: Experiment Name + destination: aeon_acquisition.Experiment + - type: attribute + input: Timeline Name + destination: name + - type: attribute + input: Start Time + destination: start + - type: attribute + input: End Time + destination: end + - type: attribute + input: Note + destination: note + + New Timeline Subject: + route: /exp_timeline_subject_form + x: 0 + y: 2.0 + height: 0.3 + width: 1 + type: form + tables: + - aeon_acquisition.ExperimentTimeline.Subject + map: + - type: table + input: Experiment Name + destination: aeon_acquisition.ExperimentTimeline + - type: table + input: Subject participating in this timeline + destination: aeon_acquisition.Experiment.Subject + Subjects: route: /subjects grids: