From c38f49e459d0061e14794b6844b21e192acc95e2 Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Mon, 2 Dec 2024 03:56:52 +0000 Subject: [PATCH 1/8] docs(add new images): :bug: update diagram images (svg) --- .../docs/datajoint_analysis_diagram.svg | 344 ++++++++-------- ...t_overview_acquisition_related_diagram.svg | 178 ++++++++ ...datajoint_overview_data_stream_diagram.svg | 301 ++++++++++++++ .../docs/datajoint_overview_diagram.svg | 381 ------------------ .../docs/notebooks/analysis_diagram.svg | 222 ---------- aeon/dj_pipeline/docs/notebooks/diagram.svg | 381 ------------------ 6 files changed, 655 insertions(+), 1152 deletions(-) create mode 100644 aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg create mode 100644 aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg delete mode 100644 aeon/dj_pipeline/docs/datajoint_overview_diagram.svg delete mode 100644 aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg delete mode 100644 aeon/dj_pipeline/docs/notebooks/diagram.svg diff --git a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg index 94087629..ee69a28c 100644 --- a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg @@ -1,222 +1,230 @@ - - - - + + + + -VisitSummary - - -VisitSummary +BlockSubjectAnalysis.Preference + + +BlockSubjectAnalysis.Preference - + -VisitSubjectPosition - - -VisitSubjectPosition +Experiment.Subject + + +Experiment.Subject - - -VisitSubjectPosition.TimeSlice - - -VisitSubjectPosition.TimeSlice + + +Visit + + +Visit - + -VisitSubjectPosition->VisitSubjectPosition.TimeSlice - +Experiment.Subject->Visit + - + -VisitTimeDistribution - - -VisitTimeDistribution - - - - - -ExperimentCamera - - -ExperimentCamera +BlockSubjectAnalysis + + +BlockSubjectAnalysis - - -CameraTracking - - -CameraTracking - - - - + -ExperimentCamera->CameraTracking - +BlockSubjectAnalysis->BlockSubjectAnalysis.Preference + - - -Visit - - -Visit + + +BlockPatchPlots + + +BlockPatchPlots - + -Visit->VisitSummary - +BlockSubjectAnalysis->BlockPatchPlots + + + + +BlockSubjectPositionPlots + + +BlockSubjectPositionPlots + - + + -Visit->VisitSubjectPosition - +BlockSubjectAnalysis->BlockSubjectPositionPlots + + + + +BlockForaging + + +BlockForaging + - - -Visit->VisitTimeDistribution - - - -Place - - -Place + + +BlockSubjectAnalysis->BlockForaging + + + + +OverlapVisit.Visit + + +OverlapVisit.Visit - + -Place->Visit - +Visit->OverlapVisit.Visit + - - -Chunk - - -Chunk + + +Experiment + + +Experiment - + -Chunk->VisitSubjectPosition - +Experiment->Experiment.Subject + + + + +OverlapVisit + + +OverlapVisit + + - + -Chunk->CameraTracking - +Experiment->OverlapVisit + - - -Experiment - - -Experiment + + +Block + + +Block - + -Experiment->ExperimentCamera - +Experiment->Block + - + -Experiment->Chunk - +OverlapVisit->OverlapVisit.Visit + - - -Experiment.Subject - - -Experiment.Subject + + +BlockAnalysis + + +BlockAnalysis - + -Experiment->Experiment.Subject - +Block->BlockAnalysis + - + + +BlockAnalysis.Subject + + +BlockAnalysis.Subject + + + + -Experiment.Subject->Visit - +BlockAnalysis.Subject->BlockSubjectAnalysis.Preference + + + + +BlockAnalysis->BlockSubjectAnalysis + + + + +BlockAnalysis->BlockAnalysis.Subject + + + + +BlockAnalysis.Patch + + +BlockAnalysis.Patch + + + + + +BlockAnalysis->BlockAnalysis.Patch + + + + +Place + + +Place + + + + + +Place->Visit + + + + +Place->OverlapVisit + + + + +BlockAnalysis.Patch->BlockSubjectAnalysis.Preference + - + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg new file mode 100644 index 00000000..1004f461 --- /dev/null +++ b/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg @@ -0,0 +1,178 @@ + + + + + +Arena + + +Arena + + + + + +Experiment + + +Experiment + + + + + +Arena->Experiment + + + + +Chunk + + +Chunk + + + + + +Experiment->Chunk + + + + +ExperimentCamera + + +ExperimentCamera + + + + + +Experiment->ExperimentCamera + + + + +Experiment.Directory + + +Experiment.Directory + + + + + +Experiment->Experiment.Directory + + + + +Epoch + + +Epoch + + + + + +Experiment->Epoch + + + + +Environment + + +Environment + + + + + +Chunk->Environment + + + + +BlockDetection + + +BlockDetection + + + + + +Environment->BlockDetection + + + + +ExperimentType + + +ExperimentType + + + + + +ExperimentType->Experiment + + + + +Experiment.Directory->Chunk + + + + +Experiment.Directory->Epoch + + + + +Location + + +Location + + + + + +Location->Experiment + + + + +EventType + + +EventType + + + + + +Epoch->Chunk + + + + +Camera + + +Camera + + + + + +Camera->ExperimentCamera + + + + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg new file mode 100644 index 00000000..a2b70791 --- /dev/null +++ b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg @@ -0,0 +1,301 @@ + + + + + +Experiment.Directory + + +Experiment.Directory + + + + + +Epoch + + +Epoch + + + + + +Experiment.Directory->Epoch + + + + +Chunk + + +Chunk + + + + + +Experiment.Directory->Chunk + + + + +SubjectDetail + + +SubjectDetail + + + + + +PyratCommentWeightProcedure + + +PyratCommentWeightProcedure + + + + + +SubjectDetail->PyratCommentWeightProcedure + + + + +Subject + + +Subject + + + + + +Subject->SubjectDetail + + + + +PyratIngestion + + +PyratIngestion + + + + + +PyratIngestion->PyratCommentWeightProcedure + + + + +Experiment + + +Experiment + + + + + +Experiment->Experiment.Directory + + + + +Experiment->Epoch + + + + +SpinnakerVideoSource + + +SpinnakerVideoSource + + + + + +Experiment->SpinnakerVideoSource + + + + +UndergroundFeeder + + +UndergroundFeeder + + + + + +Experiment->UndergroundFeeder + + + + +WeightScale + + +WeightScale + + + + + +Experiment->WeightScale + + + + +Experiment->Chunk + + + + +SLEAPTracking + + +SLEAPTracking + + + + + +GeneticBackground + + +GeneticBackground + + + + + +GeneticBackground->SubjectDetail + + + + +Strain + + +Strain + + + + + +Strain->SubjectDetail + + + + +Epoch->Chunk + + + + +SpinnakerVideoSource->SLEAPTracking + + + + +CameraQC + + +CameraQC + + + + + +SpinnakerVideoSource->CameraQC + + + + +WeightScaleWeightFiltered + + +WeightScaleWeightFiltered + + + + + +Device + + +Device + + + + + +Device->SpinnakerVideoSource + + + + +Device->UndergroundFeeder + + + + +Device->WeightScale + + + + +DeviceType + + +DeviceType + + + + + +DeviceType->Device + + + + +WeightScale->WeightScaleWeightFiltered + + + + +Chunk->SLEAPTracking + + + + +Chunk->WeightScaleWeightFiltered + + + + +Chunk->CameraQC + + + + +TrackingParamSet + + +TrackingParamSet + + + + + +TrackingParamSet->SLEAPTracking + + + + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg deleted file mode 100644 index 02590b4a..00000000 --- a/aeon/dj_pipeline/docs/datajoint_overview_diagram.svg +++ /dev/null @@ -1,381 +0,0 @@ - - - - - -Arena - - -Arena - - - - - -Experiment - - -Experiment - - - - - -Arena->Experiment - - - - -ExperimentFoodPatch - - -ExperimentFoodPatch - - - - - -FoodPatchWheel - - -FoodPatchWheel - - - - - -ExperimentFoodPatch->FoodPatchWheel - - - - -WheelState - - -WheelState - - - - - -ExperimentFoodPatch->WheelState - - - - -FoodPatchEvent - - -FoodPatchEvent - - - - - -ExperimentFoodPatch->FoodPatchEvent - - - - -WeightScale - - -WeightScale - - - - - -ExperimentWeightScale - - -ExperimentWeightScale - - - - - -WeightScale->ExperimentWeightScale - - - - -Camera - - -Camera - - - - - -ExperimentCamera - - -ExperimentCamera - - - - - -Camera->ExperimentCamera - - - - -Epoch - - -Epoch - - - - - -Chunk - - -Chunk - - - - - -Epoch->Chunk - - - - -FoodPatch - - -FoodPatch - - - - - -FoodPatch->ExperimentFoodPatch - - - - -CameraTracking - - -CameraTracking - - - - - -ExperimentCamera->CameraTracking - - - - -qc.CameraQC - - -qc.CameraQC - - - - - -ExperimentCamera->qc.CameraQC - - - - -CameraTracking.Object - - -CameraTracking.Object - - - - - -CameraTracking->CameraTracking.Object - - - - -EventType - - -EventType - - - - - -EventType->FoodPatchEvent - - - - -Chunk->FoodPatchWheel - - - - -Chunk->CameraTracking - - - - -Chunk->WheelState - - - - -Chunk->qc.CameraQC - - - - -WeightMeasurement - - -WeightMeasurement - - - - - -Chunk->WeightMeasurement - - - - -Chunk->FoodPatchEvent - - - - -Experiment->ExperimentFoodPatch - - - - -Experiment->Epoch - - - - -Experiment->ExperimentCamera - - - - -Experiment->Chunk - - - - -Experiment->ExperimentWeightScale - - - - -ExperimentWeightScale->WeightMeasurement - - - - diff --git a/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg b/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg deleted file mode 100644 index 94087629..00000000 --- a/aeon/dj_pipeline/docs/notebooks/analysis_diagram.svg +++ /dev/null @@ -1,222 +0,0 @@ - - - - - -VisitSummary - - -VisitSummary - - - - - -VisitSubjectPosition - - -VisitSubjectPosition - - - - - -VisitSubjectPosition.TimeSlice - - -VisitSubjectPosition.TimeSlice - - - - - -VisitSubjectPosition->VisitSubjectPosition.TimeSlice - - - - -VisitTimeDistribution - - -VisitTimeDistribution - - - - - -ExperimentCamera - - -ExperimentCamera - - - - - -CameraTracking - - -CameraTracking - - - - - -ExperimentCamera->CameraTracking - - - - -Visit - - -Visit - - - - - -Visit->VisitSummary - - - - -Visit->VisitSubjectPosition - - - - -Visit->VisitTimeDistribution - - - - -Place - - -Place - - - - - -Place->Visit - - - - -Chunk - - -Chunk - - - - - -Chunk->VisitSubjectPosition - - - - -Chunk->CameraTracking - - - - -Experiment - - -Experiment - - - - - -Experiment->ExperimentCamera - - - - -Experiment->Chunk - - - - -Experiment.Subject - - -Experiment.Subject - - - - - -Experiment->Experiment.Subject - - - - -Experiment.Subject->Visit - - - - diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.svg b/aeon/dj_pipeline/docs/notebooks/diagram.svg deleted file mode 100644 index d7872725..00000000 --- a/aeon/dj_pipeline/docs/notebooks/diagram.svg +++ /dev/null @@ -1,381 +0,0 @@ - - - - - -CameraTracking - - -CameraTracking - - - - - -CameraTracking.Object - - -CameraTracking.Object - - - - - -CameraTracking->CameraTracking.Object - - - - -WeightScale - - -WeightScale - - - - - -ExperimentWeightScale - - -ExperimentWeightScale - - - - - -WeightScale->ExperimentWeightScale - - - - -WheelState - - -WheelState - - - - - -ExperimentFoodPatch - - -ExperimentFoodPatch - - - - - -ExperimentFoodPatch->WheelState - - - - -FoodPatchWheel - - -FoodPatchWheel - - - - - -ExperimentFoodPatch->FoodPatchWheel - - - - -FoodPatchEvent - - -FoodPatchEvent - - - - - -ExperimentFoodPatch->FoodPatchEvent - - - - -WeightMeasurement - - -WeightMeasurement - - - - - -Arena - - -Arena - - - - - -Experiment - - -Experiment - - - - - -Arena->Experiment - - - - -ExperimentWeightScale->WeightMeasurement - - - - -EventType - - -EventType - - - - - -EventType->FoodPatchEvent - - - - -ExperimentCamera - - -ExperimentCamera - - - - - -ExperimentCamera->CameraTracking - - - - -qc.CameraQC - - -qc.CameraQC - - - - - -ExperimentCamera->qc.CameraQC - - - - -Camera - - -Camera - - - - - -Camera->ExperimentCamera - - - - -Epoch - - -Epoch - - - - - -Chunk - - -Chunk - - - - - -Epoch->Chunk - - - - -Chunk->CameraTracking - - - - -Chunk->WheelState - - - - -Chunk->WeightMeasurement - - - - -Chunk->FoodPatchWheel - - - - -Chunk->qc.CameraQC - - - - -Chunk->FoodPatchEvent - - - - -Experiment->ExperimentFoodPatch - - - - -Experiment->ExperimentWeightScale - - - - -Experiment->ExperimentCamera - - - - -Experiment->Epoch - - - - -Experiment->Chunk - - - - -FoodPatch - - -FoodPatch - - - - - -FoodPatch->ExperimentFoodPatch - - - - From 426324a2f30ae2760969b429de15a9905006db6d Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Mon, 2 Dec 2024 03:57:19 +0000 Subject: [PATCH 2/8] docs(add new images): :bug: update notebook to create diagrams --- aeon/dj_pipeline/docs/notebooks/diagram.ipynb | 862 ++---------------- 1 file changed, 83 insertions(+), 779 deletions(-) diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb index fc6734d2..26e0e569 100644 --- a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb +++ b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb @@ -2,25 +2,17 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "b18d8c4a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/nfs/nhome/live/thinh/code/ProjectAeon/aeon\n" - ] - } - ], + "outputs": [], "source": [ "cd ../../../.." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "b9db4795", "metadata": {}, "outputs": [], @@ -30,42 +22,38 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "571c2760", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2022-07-28 19:53:12,486][INFO]: Connecting thinh@aeon-db2:3306\n", - "[2022-07-28 19:53:12,497][INFO]: Connected thinh@aeon-db2:3306\n" - ] - } - ], + "outputs": [], "source": [ - "_db_prefix = \"aeon_\"\n", + "_db_prefix = 'aeon_'\n", "\n", - "lab = dj.create_virtual_module(\"lab\", _db_prefix + \"lab\")\n", + "streams = dj.create_virtual_module(\"streams\", _db_prefix + \"streams\")\n", + "tracking = dj.create_virtual_module(\"tracking\", _db_prefix + \"tracking\")\n", + "analysis = dj.create_virtual_module(\"analysis\", _db_prefix + \"analysis\")\n", + "block_analysis = dj.create_virtual_module(\"block_analysis\", _db_prefix + \"block_analysis\")\n", "subject = dj.create_virtual_module(\"subject\", _db_prefix + \"subject\")\n", + "lab = dj.create_virtual_module(\"lab\", _db_prefix + \"lab\")\n", "acquisition = dj.create_virtual_module(\"acquisition\", _db_prefix + \"acquisition\")\n", - "qc = dj.create_virtual_module(\"qc\", _db_prefix + \"qc\")\n", - "tracking = dj.create_virtual_module(\"tracking\", _db_prefix + \"tracking\")\n", - "analysis = dj.create_virtual_module(\"analysis\", _db_prefix + \"analysis\")" + "qc = dj.create_virtual_module(\"qc\", _db_prefix + \"qc\")\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "bcf4bd2e", "metadata": {}, "outputs": [], "source": [ - "lab.schema.spawn_missing_classes()\n", + "streams.schema.spawn_missing_classes()\n", + "tracking.schema.spawn_missing_classes()\n", + "analysis.schema.spawn_missing_classes()\n", + "block_analysis.schema.spawn_missing_classes()\n", "subject.schema.spawn_missing_classes()\n", + "lab.schema.spawn_missing_classes()\n", "acquisition.schema.spawn_missing_classes()\n", - "tracking.schema.spawn_missing_classes()\n", - "analysis.schema.spawn_missing_classes()" + "qc.schema.spawn_missing_classes()" ] }, { @@ -78,433 +66,65 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "5c447e42", + "execution_count": null, + "id": "6e9442ac", + "metadata": {}, + "outputs": [], + "source": [ + "# `acquisition`-related tasks\n", + "acquisition_diagram = (\n", + " dj.Diagram(acquisition.Experiment) +\n", + " dj.Diagram(acquisition.ExperimentCamera) +\n", + " dj.Diagram(acquisition.Epoch) +\n", + " dj.Diagram(acquisition.Chunk) +\n", + " dj.Diagram(acquisition.EventType) +\n", + " dj.Diagram(acquisition.Environment) +\n", + " dj.Diagram(block_analysis.BlockDetection)\n", + ") - 1\n", + "\n", + "acquisition_diagram" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0600b4b7", + "metadata": {}, + "outputs": [], + "source": [ + "acquisition_diagram.save(\"aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aea37324", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "Arena\n", - "\n", - "\n", - "Arena\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Experiment\n", - "\n", - "\n", - "Experiment\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Arena->Experiment\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentFoodPatch\n", - "\n", - "\n", - "ExperimentFoodPatch\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "FoodPatchWheel\n", - "\n", - "\n", - "FoodPatchWheel\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentFoodPatch->FoodPatchWheel\n", - "\n", - "\n", - "\n", - "\n", - "WheelState\n", - "\n", - "\n", - "WheelState\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentFoodPatch->WheelState\n", - "\n", - "\n", - "\n", - "\n", - "FoodPatchEvent\n", - "\n", - "\n", - "FoodPatchEvent\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentFoodPatch->FoodPatchEvent\n", - "\n", - "\n", - "\n", - "\n", - "WeightScale\n", - "\n", - "\n", - "WeightScale\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentWeightScale\n", - "\n", - "\n", - "ExperimentWeightScale\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "WeightScale->ExperimentWeightScale\n", - "\n", - "\n", - "\n", - "\n", - "Camera\n", - "\n", - "\n", - "Camera\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentCamera\n", - "\n", - "\n", - "ExperimentCamera\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Camera->ExperimentCamera\n", - "\n", - "\n", - "\n", - "\n", - "Epoch\n", - "\n", - "\n", - "Epoch\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Chunk\n", - "\n", - "\n", - "Chunk\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Epoch->Chunk\n", - "\n", - "\n", - "\n", - "\n", - "FoodPatch\n", - "\n", - "\n", - "FoodPatch\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "FoodPatch->ExperimentFoodPatch\n", - "\n", - "\n", - "\n", - "\n", - "CameraTracking\n", - "\n", - "\n", - "CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentCamera->CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "qc.CameraQC\n", - "\n", - "\n", - "qc.CameraQC\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentCamera->qc.CameraQC\n", - "\n", - "\n", - "\n", - "\n", - "CameraTracking.Object\n", - "\n", - "\n", - "CameraTracking.Object\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "CameraTracking->CameraTracking.Object\n", - "\n", - "\n", - "\n", - "\n", - "EventType\n", - "\n", - "\n", - "EventType\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "EventType->FoodPatchEvent\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->FoodPatchWheel\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->WheelState\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->qc.CameraQC\n", - "\n", - "\n", - "\n", - "\n", - "WeightMeasurement\n", - "\n", - "\n", - "WeightMeasurement\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->WeightMeasurement\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->FoodPatchEvent\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->ExperimentFoodPatch\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->Epoch\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->ExperimentCamera\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->Chunk\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->ExperimentWeightScale\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentWeightScale->WeightMeasurement\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "diagram = (\n", - " dj.Diagram(acquisition.Epoch)\n", - " + lab.Arena\n", - " + lab.FoodPatch\n", - " + lab.Camera\n", - " + lab.WeightScale\n", - " + acquisition.Experiment\n", - " + acquisition.ExperimentCamera\n", - " + acquisition.FoodPatchWheel\n", - " + acquisition.FoodPatchEvent\n", - " + acquisition.WheelState\n", - " + acquisition.EventType\n", - " + acquisition.WeightMeasurement\n", - " + qc.CameraQC\n", - " + tracking.CameraTracking.Object\n", - ")\n", + "# data streams\n", + "\n", + "data_stream_diagram = (\n", + " dj.Diagram(subject.SubjectDetail) + \n", + " dj.Diagram(subject.PyratCommentWeightProcedure) + \n", + " dj.Diagram(streams.WeightScaleWeightFiltered) + \n", + " dj.Diagram(streams.Device) + \n", + " dj.Diagram(streams.UndergroundFeeder) + \n", + " dj.Diagram(qc.CameraQC) +\n", + " dj.Diagram(tracking.SLEAPTracking)\n", + ") - 1\n", "\n", - "diagram" + "data_stream_diagram" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "fc5e04ff", "metadata": {}, "outputs": [], "source": [ - "diagram.save(\"aeon/dj_pipeline/docs/datajoint_overview_diagram.svg\")" + "data_stream_diagram.save(\"aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg\")" ] }, { @@ -517,361 +137,45 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "ead30859", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "OverlapVisit.Visit\n", - "\n", - "\n", - "OverlapVisit.Visit\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "VisitSummary\n", - "\n", - "\n", - "VisitSummary\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "VisitSubjectPosition\n", - "\n", - "\n", - "VisitSubjectPosition\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "VisitSubjectPosition.TimeSlice\n", - "\n", - "\n", - "VisitSubjectPosition.TimeSlice\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "VisitSubjectPosition->VisitSubjectPosition.TimeSlice\n", - "\n", - "\n", - "\n", - "\n", - "VisitTimeDistribution\n", - "\n", - "\n", - "VisitTimeDistribution\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "VisitEnd\n", - "\n", - "\n", - "VisitEnd\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentCamera\n", - "\n", - "\n", - "ExperimentCamera\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "CameraTracking\n", - "\n", - "\n", - "CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ExperimentCamera->CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "Visit\n", - "\n", - "\n", - "Visit\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Visit->OverlapVisit.Visit\n", - "\n", - "\n", - "\n", - "\n", - "Visit->VisitSummary\n", - "\n", - "\n", - "\n", - "\n", - "Visit->VisitSubjectPosition\n", - "\n", - "\n", - "\n", - "\n", - "Visit->VisitTimeDistribution\n", - "\n", - "\n", - "\n", - "\n", - "Visit->VisitEnd\n", - "\n", - "\n", - "\n", - "\n", - "Place\n", - "\n", - "\n", - "Place\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Place->Visit\n", - "\n", - "\n", - "\n", - "\n", - "OverlapVisit\n", - "\n", - "\n", - "OverlapVisit\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Place->OverlapVisit\n", - "\n", - "\n", - "\n", - "\n", - "Chunk\n", - "\n", - "\n", - "Chunk\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->VisitSubjectPosition\n", - "\n", - "\n", - "\n", - "\n", - "Chunk->CameraTracking\n", - "\n", - "\n", - "\n", - "\n", - "Experiment\n", - "\n", - "\n", - "Experiment\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->ExperimentCamera\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->Chunk\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->OverlapVisit\n", - "\n", - "\n", - "\n", - "\n", - "Experiment.Subject\n", - "\n", - "\n", - "Experiment.Subject\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Experiment->Experiment.Subject\n", - "\n", - "\n", - "\n", - "\n", - "OverlapVisit->OverlapVisit.Visit\n", - "\n", - "\n", - "\n", - "\n", - "Experiment.Subject->Visit\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ + "# analysis tables\n", "analysis_diagram = (\n", - " dj.Diagram(analysis.Visit)\n", - " + analysis.Place\n", - " + analysis.VisitEnd\n", - " + analysis.OverlapVisit.Visit\n", - " + acquisition.Experiment\n", - " + tracking.CameraTracking\n", - " + analysis.VisitSubjectPosition.TimeSlice\n", - " + analysis.VisitTimeDistribution\n", - " + analysis.VisitSummary\n", - ")\n", + " dj.Diagram(analysis.Visit) +\n", + " dj.Diagram(analysis.OverlapVisit.Visit) +\n", + " dj.Diagram(block_analysis.Block) +\n", + " dj.Diagram(block_analysis.BlockAnalysis) +\n", + " dj.Diagram(block_analysis.BlockSubjectAnalysis.Preference) +\n", + " dj.Diagram(block_analysis.BlockForaging) +\n", + " dj.Diagram(block_analysis.BlockPatchPlots) +\n", + " dj.Diagram(block_analysis.BlockSubjectPositionPlots) \n", + ") - 1\n", "\n", "analysis_diagram" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "56a3b461", "metadata": {}, "outputs": [], "source": [ - "diagram.save(\"aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg\")" + "analysis_diagram.save(\"aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg\")" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "fd2745c0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "datajoint_analysis_diagram.svg datajoint_overview_diagram.svg \u001b[0m\u001b[01;34mnotebooks\u001b[0m/\r\n" - ] - } - ], + "outputs": [], "source": [ "ls aeon/dj_pipeline/docs/" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fce5849b", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From da74c4b5a645ea7100e67488a75b81dc7c0d189f Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Mon, 2 Dec 2024 03:58:38 +0000 Subject: [PATCH 3/8] docs(update README): :memo: update README in `dj_pipeline` --- aeon/dj_pipeline/README.md | 101 ++++++++++--------------------------- 1 file changed, 28 insertions(+), 73 deletions(-) diff --git a/aeon/dj_pipeline/README.md b/aeon/dj_pipeline/README.md index 545f0f35..27f21310 100644 --- a/aeon/dj_pipeline/README.md +++ b/aeon/dj_pipeline/README.md @@ -1,111 +1,66 @@ # DataJoint Pipeline for Project Aeon -This pipeline models the data organization and data flow custom-built for Project Aeon. You can find Aeon acquisition system here: [aeon_aquisition](https://github.com/SainsburyWellcomeCentre/aeon_acquisition) +This DataJoint pipeline models the data organization and data flow tailored to the project's Aeon requirements. You can access the Aeon acquisition system here: [aeon_aquisition](https://github.com/SainsburyWellcomeCentre/aeon_acquisition) -## Pipeline architecture +## Pipeline Architecture -The diagram below shows the high level overview of the diagram (only the subset of the tables that are most relevant). +The following diagrams provide a high-level overview of the pipeline's components and processes: -![datajoint_pipeline](./docs/datajoint_overview_diagram.svg) +The diagram below illustrates the structure of the **acquisition-related tasks within the pipeline**, focusing on the most relevant subset of tables. +![datajoint_overview_acquisition_diagram](./docs/datajoint_overview_acquisition_related_diagram.svg) -The diagram below shows the analysis portion of the pipeline (work in progress). +The diagram below represents the **data stream flow within the pipeline**, highlighting the subset of tables critical to understanding the process. + +![datajoint_overview_data_stream_diagram](./docs/datajoint_overview_data_stream_diagram.svg) + +The diagram below shows the **analysis portion of the pipeline**. ![datajoint_analysis_pipeline](./docs/datajoint_analysis_diagram.svg) -From the diagram above, we can see that the pipeline is organized in layers of -tables, going top down, from `lookup`-tier (in gray) and `manual`-tier (in green) tables -to `imported`-tier (in purple) and `computed`-tier (in red) tables. +The pipeline is structured into hierarchical layers of tables, which are depicted in the diagrams above. These layers include: + ++ `lookup`-tier tables (gray): Define static reference information ++ `manual`-tier tables (green): Contain user-inputted data ++ `imported`-tier tables (purple): Store data ingested from external sources ++ `computed`-tier tables (red): Represent results of automated computations -Such is also the way the data flows through the pipeline, by a combination of ingestion and -computation routines. +Data flows through the pipeline in a top-down manner, driven by a combination of ingestion and computation routines. This layered organization facilitates efficient data processing and modular analysis. ## Core tables #### Experiment and data acquisition -1. `Experiment` - the `aquisition.Experiment` table stores meta information about the experiments ++ `Experiment` - the `aquisition.Experiment` table stores meta information about the experiments done in Project Aeon, with secondary information such as the lab/room the experiment is carried out, which animals participating, the directory storing the raw data, etc. -2. `Epoch` - A recording period reflecting on/off of the hardware acquisition system. ++ `Epoch` - A recording period reflecting on/off of the hardware acquisition system. The `aquisition.Epoch` table records all acquisition epochs and their associated configuration for any particular experiment (in the above `aquisition.Experiment` table). -3.`Chunk` - the raw data are acquired by Bonsai and stored as ++ `Chunk` - the raw data are acquired by Bonsai and stored as a collection of files every one hour - we call this one-hour a time chunk. The `aquisition.Chunk` table records all time chunks and their associated raw data files for any particular experiment (in the above `aquisition.Experiment` table). A chunk must belong to one epoch. -#### Devices - -5. `ExperimentCamera` - the cameras and associated specifications used for this experiment - -e.g. camera serial number, frame rate, location, time of installation and removal, etc. - -6. `ExperimentFoodPatch` - the food-patches and associated specifications used for this experiment - -e.g. patch serial number, sampling rate of the wheel, location, time of installation and removal, etc. - -7. `ExperimentWeightScale` - the scales for measuring animal weights, usually placed at the nest, one per nest - -#### Data streams - -8. `FoodPatchEvent` - all events (e.g. pellet triggered, pellet delivered, etc.) -from a particular `ExperimentFoodPatch` - -9. `FoodPatchWheel` - wheel data (angle, intensity) from a particular `ExperimentFoodPatch` - -10. `WheelState` - wheel states (threshold, d1, delta) associated with a given `ExperimentFoodPatch` - -11. `WeightMeasurement` - scale measurements associated with a given `ExperimentScale` - - #### Position data -12. `qc.CameraQC` - quality control procedure applied to each `ExperimentCamera` (e.g. missing frame, etc.) ++ `qc.CameraQC` - quality control procedure applied to each `ExperimentCamera` (e.g. missing frame, etc.) -13. `tracking.CameraTracking` - position tracking for object(s), from each `ExperimentCamera` ++ `tracking.SLEAPTracking` - position tracking for object(s), from a particular `VideoSource` per chunk #### Standard analyses -14. `Visit` - a `Visit` is defined as a ***period of time*** -that a particular ***animal*** spends time at a particular ***place*** - -15. `VisitSubjectPosition` - position data (x, y, z, area) of the subject for any particular visit. -Position data per visit are stored in smaller time slices (10-minute long) allowing for -more efficient searches, queries and fetches from the database. - -16. `VisitSummary` - a table for computation and storing some summary statistics on a -per-session level - i.e. total pellet delivered, total distance the animal travelled, total -distance the wheel travelled (or per food-patch), etc. - -17. `VisitTimeDistribution` - a table for computation and storing where the animal is at, -for each timepoint, e.g. in the nest, in corridor, in arena, in each of the food patches. -This can be used to produce the ethogram plot. - -## Operating the pipeline - how the auto ingestion/processing work? - -Some meta information about the experiment is entered - e.g. experiment name, participating -animals, cameras, food patches setup, etc. -+ These information are either entered by hand, or parsed and inserted from configuration - yaml files. -+ For experiments these info can be inserted by running - + [create_experiment_01](create_experiments/create_experiment_01.py) - + [create_socialexperiment_0](create_experiments/create_socialexperiment_0.py) - + [create_experiment_02](create_experiments/create_experiment_02.py) - (just need to do this once) - -Tables in DataJoint are written with a `make()` function - -instruction to generate and insert new records to itself, based on data from upstream tables. -Triggering the auto ingestion and processing/computation routine is essentially -calling the `.populate()` method for all relevant tables. ++ `Visit` - a `Visit` is defined as a ***period of time*** during which a particular ***animal*** remains at a specific ***place***. -These routines are prepared in this [auto-processing script](populate/process.py). -Essentially, turning on the auto-processing routine amounts to running the -following 2 commands (in different processing threads) ++ `BlockAnalysis` - higher-level aggregation of events and metrics that occur within a defined block of time during an experiment. It integrates data from multiple subjects, positions, and interactions to enable further analysis of behavior and environmental interactions. ++ `BlockSubjectAnalysis` - A detailed analysis for each subject in a given block, partitioned into the following components: + - `Patch`: tracks the interactions of each subject with specific patches (areas of interests) + - `Preference`: measures a subject's preference for specific patches using various analyses, including cumulative preferences based on time spent and distance traveled in relation to each patch. - aeon_ingest high +#### Data stream - aeon_ingest mid From bd77e2598e384ee78019ed756f2cd5680c7c08b0 Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Tue, 3 Dec 2024 16:05:41 +0000 Subject: [PATCH 4/8] docs(add new images): update images after internal review --- .../docs/datajoint_analysis_diagram.svg | 308 +++++----- ...t_overview_acquisition_related_diagram.svg | 297 +++++---- ...datajoint_overview_data_stream_diagram.svg | 581 ++++++++++++------ ...tajoint_overview_pyrat_related_diagram.svg | 84 +++ 4 files changed, 809 insertions(+), 461 deletions(-) create mode 100644 aeon/dj_pipeline/docs/datajoint_overview_pyrat_related_diagram.svg diff --git a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg index ee69a28c..2dc0f3b7 100644 --- a/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_analysis_diagram.svg @@ -1,107 +1,130 @@ - + - - + + -BlockSubjectAnalysis.Preference - - -BlockSubjectAnalysis.Preference - - - - - -Experiment.Subject - - -Experiment.Subject +BlockAnalysis.Subject + + +BlockAnalysis.Subject - - -Visit - - -Visit + + +BlockSubjectAnalysis.Preference + + +BlockSubjectAnalysis.Preference - + -Experiment.Subject->Visit - +BlockAnalysis.Subject->BlockSubjectAnalysis.Preference + - - -BlockSubjectAnalysis - - -BlockSubjectAnalysis + + +BlockSubjectAnalysis.Patch + + +BlockSubjectAnalysis.Patch - + -BlockSubjectAnalysis->BlockSubjectAnalysis.Preference - +BlockAnalysis.Subject->BlockSubjectAnalysis.Patch + - - -BlockPatchPlots - - -BlockPatchPlots + + +Place + + +Place - + + +OverlapVisit + + +OverlapVisit + + + + -BlockSubjectAnalysis->BlockPatchPlots - +Place->OverlapVisit + - - -BlockSubjectPositionPlots - - -BlockSubjectPositionPlots + + +Visit + + +Visit - + -BlockSubjectAnalysis->BlockSubjectPositionPlots - +Place->Visit + - - -BlockForaging - - -BlockForaging + + +BlockAnalysis + + +BlockAnalysis - + -BlockSubjectAnalysis->BlockForaging - +BlockAnalysis->BlockAnalysis.Subject + - - -OverlapVisit.Visit - - -OverlapVisit.Visit + + +BlockAnalysis.Patch + + +BlockAnalysis.Patch - + -Visit->OverlapVisit.Visit - +BlockAnalysis->BlockAnalysis.Patch + + + + +BlockSubjectAnalysis + + +BlockSubjectAnalysis + + + + + +BlockAnalysis->BlockSubjectAnalysis + + + + +BlockPatchPlots + + +BlockPatchPlots + + @@ -112,119 +135,120 @@ - - -Experiment->Experiment.Subject - - - - -OverlapVisit - - -OverlapVisit - - - Experiment->OverlapVisit - + Block - - -Block + + +Block Experiment->Block - + - + + +Experiment.Subject + + +Experiment.Subject + + + + -OverlapVisit->OverlapVisit.Visit - +Experiment->Experiment.Subject + - - -BlockAnalysis - - -BlockAnalysis + + +OverlapVisit.Visit + + +OverlapVisit.Visit - + -Block->BlockAnalysis - +OverlapVisit->OverlapVisit.Visit + - + -BlockAnalysis.Subject - - -BlockAnalysis.Subject +BlockForaging + + +BlockForaging - + -BlockAnalysis.Subject->BlockSubjectAnalysis.Preference - +BlockAnalysis.Patch->BlockSubjectAnalysis.Preference + - + -BlockAnalysis->BlockSubjectAnalysis - +BlockAnalysis.Patch->BlockSubjectAnalysis.Patch + - + -BlockAnalysis->BlockAnalysis.Subject - +Visit->OverlapVisit.Visit + - - -BlockAnalysis.Patch - - -BlockAnalysis.Patch + + +BlockSubjectPositionPlots + + +BlockSubjectPositionPlots - + -BlockAnalysis->BlockAnalysis.Patch - - - - -Place - - -Place - - +BlockSubjectAnalysis->BlockPatchPlots + - + -Place->Visit - +BlockSubjectAnalysis->BlockSubjectAnalysis.Preference + - + -Place->OverlapVisit - +BlockSubjectAnalysis->BlockForaging + - + -BlockAnalysis.Patch->BlockSubjectAnalysis.Preference - +BlockSubjectAnalysis->BlockSubjectPositionPlots + + + + +BlockSubjectAnalysis->BlockSubjectAnalysis.Patch + + + + +Block->BlockAnalysis + + + + +Experiment.Subject->Visit + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg index 1004f461..f0420b7c 100644 --- a/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_overview_acquisition_related_diagram.svg @@ -1,178 +1,243 @@ - - - - + + + + -Arena - - -Arena +Environment.SubjectVisits + + +Environment.SubjectVisits - + +EpochConfig + + +EpochConfig + + + + + +Environment.EnvironmentState + + +Environment.EnvironmentState + + + + + +Location + + +Location + + + + + Experiment - - -Experiment + + +Experiment - + +Location->Experiment + + + + +Arena + + +Arena + + + + + Arena->Experiment - + + + + +Epoch + + +Epoch + + + + + +Experiment->Epoch + - + Chunk - - -Chunk + + +Chunk - + Experiment->Chunk - - - - -ExperimentCamera - - -ExperimentCamera + + + + +Experiment.Subject + + +Experiment.Subject - - -Experiment->ExperimentCamera - + + +Experiment->Experiment.Subject + - + -Experiment.Directory - - -Experiment.Directory +Environment.SubjectWeight + + +Environment.SubjectWeight - - -Experiment->Experiment.Directory - - - - -Epoch - - -Epoch + + +Subject + + +Subject - - -Experiment->Epoch - + + +Subject->Experiment.Subject + + + + +Epoch->EpochConfig + + + + +Epoch->Chunk + + + + +Environment.BlockState + + +Environment.BlockState + + - + Environment - - -Environment + + +Environment - + Chunk->Environment - + - - -BlockDetection - - -BlockDetection + + +Environment.SubjectState + + +Environment.SubjectState - - -Environment->BlockDetection - - - + ExperimentType - - -ExperimentType + + +ExperimentType - -ExperimentType->Experiment - - - - -Experiment.Directory->Chunk - - - -Experiment.Directory->Epoch - - - - -Location - - -Location +ExperimentType->Experiment + + + + +Environment.MessageLog + + +Environment.MessageLog - + -Location->Experiment - - - - -EventType - - -EventType - +Environment->Environment.SubjectVisits + - - + -Epoch->Chunk - +Environment->Environment.EnvironmentState + - - -Camera - - -Camera + + +Environment->Environment.SubjectWeight + + + + +Environment->Environment.BlockState + + + + +Environment->Environment.SubjectState + + + + +Environment->Environment.MessageLog + + + + +Environment.LightEvents + + +Environment.LightEvents - - -Camera->ExperimentCamera - + + +Environment->Environment.LightEvents + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg index a2b70791..df4eee45 100644 --- a/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg @@ -1,301 +1,476 @@ - - - - + + + + -Experiment.Directory - - -Experiment.Directory +RfidReader.Attribute + + +RfidReader.Attribute - - -Epoch - - -Epoch + + +UndergroundFeederRetriedDelivery + + +UndergroundFeederRetriedDelivery - - -Experiment.Directory->Epoch - - - - -Chunk - - -Chunk + + +UndergroundFeeder.RemovalTime + + +UndergroundFeeder.RemovalTime - - -Experiment.Directory->Chunk - + + +WeightScaleWeightFiltered + + +WeightScaleWeightFiltered + - - -SubjectDetail - - -SubjectDetail + + + +Device + + +Device - - -PyratCommentWeightProcedure - - -PyratCommentWeightProcedure + + +UndergroundFeeder + + +UndergroundFeeder - - -SubjectDetail->PyratCommentWeightProcedure - + + +Device->UndergroundFeeder + - - -Subject - - -Subject + + +SpinnakerVideoSource + + +SpinnakerVideoSource - - -Subject->SubjectDetail - - - - -PyratIngestion - - -PyratIngestion + + +Device->SpinnakerVideoSource + + + + +RfidReader + + +RfidReader - - -PyratIngestion->PyratCommentWeightProcedure - + + +Device->RfidReader + - - -Experiment - - -Experiment + + +WeightScale + + +WeightScale - - -Experiment->Experiment.Directory - + + +Device->WeightScale + - - -Experiment->Epoch - + + +SpinnakerVideoSource.Attribute + + +SpinnakerVideoSource.Attribute + - - -SpinnakerVideoSource - - -SpinnakerVideoSource + + + +SpinnakerVideoSourceVideo + + +SpinnakerVideoSourceVideo - - -Experiment->SpinnakerVideoSource - + + +RfidReaderRfidEvents + + +RfidReaderRfidEvents + - - -UndergroundFeeder - - -UndergroundFeeder + + + +Experiment + + +Experiment - + Experiment->UndergroundFeeder - + - - -WeightScale - - -WeightScale - + + +Experiment->SpinnakerVideoSource + + + +Experiment->RfidReader + - + Experiment->WeightScale - + + + + +Chunk + + +Chunk + + - + Experiment->Chunk - + - - -SLEAPTracking - - -SLEAPTracking + + +UndergroundFeeder.Attribute + + +UndergroundFeeder.Attribute - - -GeneticBackground - - -GeneticBackground + + +WeightScale.Attribute + + +WeightScale.Attribute - + + +UndergroundFeeder->UndergroundFeederRetriedDelivery + + + + +UndergroundFeeder->UndergroundFeeder.RemovalTime + + + -GeneticBackground->SubjectDetail - +UndergroundFeeder->UndergroundFeeder.Attribute + - - -Strain - - -Strain + + +UndergroundFeederManualDelivery + + +UndergroundFeederManualDelivery - + -Strain->SubjectDetail - +UndergroundFeeder->UndergroundFeederManualDelivery + + + + +UndergroundFeederMissedPellet + + +UndergroundFeederMissedPellet + + - + -Epoch->Chunk - +UndergroundFeeder->UndergroundFeederMissedPellet + + + + +UndergroundFeederDeliverPellet + + +UndergroundFeederDeliverPellet + - - -SpinnakerVideoSource->SLEAPTracking - - - -CameraQC - - -CameraQC + + +UndergroundFeeder->UndergroundFeederDeliverPellet + + + + +UndergroundFeederBeamBreak + + +UndergroundFeederBeamBreak - + -SpinnakerVideoSource->CameraQC - +UndergroundFeeder->UndergroundFeederBeamBreak + + + + +UndergroundFeederDepletionState + + +UndergroundFeederDepletionState + - - -WeightScaleWeightFiltered - - -WeightScaleWeightFiltered + + + +UndergroundFeeder->UndergroundFeederDepletionState + + + + +UndergroundFeederEncoder + + +UndergroundFeederEncoder - + + +UndergroundFeeder->UndergroundFeederEncoder + + + -Device - - -Device +WeightScaleWeightRaw + + +WeightScaleWeightRaw - - -Device->SpinnakerVideoSource - + + +StreamType + + +StreamType + - - -Device->UndergroundFeeder - - + + +DeviceType.Stream + + +DeviceType.Stream + + + + -Device->WeightScale - +StreamType->DeviceType.Stream + + + + +SpinnakerVideoSource->SpinnakerVideoSource.Attribute + + + + +SpinnakerVideoSource->SpinnakerVideoSourceVideo + + + + +SpinnakerVideoSource.RemovalTime + + +SpinnakerVideoSource.RemovalTime + + + + + +SpinnakerVideoSource->SpinnakerVideoSource.RemovalTime + - + DeviceType - - -DeviceType + + +DeviceType - + DeviceType->Device - + - - -WeightScale->WeightScaleWeightFiltered - + + +DeviceType->DeviceType.Stream + - - -Chunk->SLEAPTracking - + + +RfidReader->RfidReader.Attribute + + + + +RfidReader->RfidReaderRfidEvents + + + + +RfidReader.RemovalTime + + +RfidReader.RemovalTime + - - -Chunk->WeightScaleWeightFiltered - - - -Chunk->CameraQC - + + +RfidReader->RfidReader.RemovalTime + - - -TrackingParamSet - - -TrackingParamSet + + +WeightScale->WeightScaleWeightFiltered + + + + +WeightScale->WeightScale.Attribute + + + + +WeightScale->WeightScaleWeightRaw + + + + +WeightScale.RemovalTime + + +WeightScale.RemovalTime - - -TrackingParamSet->SLEAPTracking - + + +WeightScale->WeightScale.RemovalTime + + + + +Chunk->UndergroundFeederRetriedDelivery + + + + +Chunk->WeightScaleWeightFiltered + + + + +Chunk->SpinnakerVideoSourceVideo + + + + +Chunk->RfidReaderRfidEvents + + + + +Chunk->UndergroundFeederManualDelivery + + + + +Chunk->WeightScaleWeightRaw + + + + +Chunk->UndergroundFeederMissedPellet + + + + +Chunk->UndergroundFeederDeliverPellet + + + + +Chunk->UndergroundFeederBeamBreak + + + + +Chunk->UndergroundFeederDepletionState + + + + +Chunk->UndergroundFeederEncoder + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/datajoint_overview_pyrat_related_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_pyrat_related_diagram.svg new file mode 100644 index 00000000..59e4fe43 --- /dev/null +++ b/aeon/dj_pipeline/docs/datajoint_overview_pyrat_related_diagram.svg @@ -0,0 +1,84 @@ + + + + + +Strain + + +Strain + + + + + +SubjectDetail + + +SubjectDetail + + + + + +Strain->SubjectDetail + + + + +PyratIngestion + + +PyratIngestion + + + + + +PyratCommentWeightProcedure + + +PyratCommentWeightProcedure + + + + + +PyratIngestion->PyratCommentWeightProcedure + + + + +Subject + + +Subject + + + + + +Subject->SubjectDetail + + + + +GeneticBackground + + +GeneticBackground + + + + + +GeneticBackground->SubjectDetail + + + + +SubjectDetail->PyratCommentWeightProcedure + + + + \ No newline at end of file From 7761864902a9a4a66465d0bac6147b9647d422b5 Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Tue, 3 Dec 2024 16:06:46 +0000 Subject: [PATCH 5/8] docs(update notebook): :memo: update diagram.ipynb after internal review --- aeon/dj_pipeline/docs/notebooks/diagram.ipynb | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb index 26e0e569..5de05e3c 100644 --- a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb +++ b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb @@ -71,16 +71,26 @@ "metadata": {}, "outputs": [], "source": [ - "# `acquisition`-related tasks\n", + "# acquisition-related tasks\n", "acquisition_diagram = (\n", - " dj.Diagram(acquisition.Experiment) +\n", - " dj.Diagram(acquisition.ExperimentCamera) +\n", - " dj.Diagram(acquisition.Epoch) +\n", - " dj.Diagram(acquisition.Chunk) +\n", - " dj.Diagram(acquisition.EventType) +\n", - " dj.Diagram(acquisition.Environment) +\n", - " dj.Diagram(block_analysis.BlockDetection)\n", - ") - 1\n", + " dj.Diagram(lab.Location) +\n", + " lab.Arena +\n", + " acquisition.ExperimentType +\n", + " subject.Subject +\n", + " acquisition.Experiment.Subject +\n", + " acquisition.Experiment +\n", + " acquisition.Epoch +\n", + " acquisition.EpochConfig +\n", + " acquisition.Chunk +\n", + " acquisition.Environment +\n", + " acquisition.Environment.SubjectWeight +\n", + " acquisition.Environment.EnvironmentState +\n", + " acquisition.Environment.SubjectState +\n", + " acquisition.Environment.MessageLog +\n", + " acquisition.Environment.SubjectVisits +\n", + " acquisition.Environment.BlockState +\n", + " acquisition.Environment.LightEvents\n", + " ) - dj.Diagram(acquisition.Experiment)\n", "\n", "acquisition_diagram" ] @@ -105,14 +115,9 @@ "# data streams\n", "\n", "data_stream_diagram = (\n", - " dj.Diagram(subject.SubjectDetail) + \n", - " dj.Diagram(subject.PyratCommentWeightProcedure) + \n", - " dj.Diagram(streams.WeightScaleWeightFiltered) + \n", - " dj.Diagram(streams.Device) + \n", - " dj.Diagram(streams.UndergroundFeeder) + \n", - " dj.Diagram(qc.CameraQC) +\n", - " dj.Diagram(tracking.SLEAPTracking)\n", - ") - 1\n", + " dj.Diagram(streams) + \n", + " acquisition.Experiment\n", + ")\n", "\n", "data_stream_diagram" ] @@ -127,6 +132,33 @@ "data_stream_diagram.save(\"aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d3f7a66", + "metadata": {}, + "outputs": [], + "source": [ + "# Pyrat-related tables\n", + "\n", + "pyrat_diagram = (\n", + " dj.Diagram(subject.SubjectDetail) + \n", + " subject.PyratCommentWeightProcedure\n", + ") - 1\n", + "\n", + "pyrat_diagram" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ff44b6f", + "metadata": {}, + "outputs": [], + "source": [ + "pyrat_diagram.save(\"aeon/dj_pipeline/docs/datajoint_overview_pyrat_related_diagram.svg\")" + ] + }, { "cell_type": "markdown", "id": "ff290db9", @@ -145,13 +177,14 @@ "# analysis tables\n", "analysis_diagram = (\n", " dj.Diagram(analysis.Visit) +\n", - " dj.Diagram(analysis.OverlapVisit.Visit) +\n", - " dj.Diagram(block_analysis.Block) +\n", - " dj.Diagram(block_analysis.BlockAnalysis) +\n", - " dj.Diagram(block_analysis.BlockSubjectAnalysis.Preference) +\n", - " dj.Diagram(block_analysis.BlockForaging) +\n", - " dj.Diagram(block_analysis.BlockPatchPlots) +\n", - " dj.Diagram(block_analysis.BlockSubjectPositionPlots) \n", + " analysis.OverlapVisit.Visit +\n", + " block_analysis.Block +\n", + " block_analysis.BlockAnalysis +\n", + " block_analysis.BlockSubjectAnalysis.Preference +\n", + " block_analysis.BlockSubjectAnalysis.Patch +\n", + " block_analysis.BlockForaging +\n", + " block_analysis.BlockPatchPlots +\n", + " block_analysis.BlockSubjectPositionPlots \n", ") - 1\n", "\n", "analysis_diagram" From 60f33dd71ff0bca2a59d3d64ead36afd8ba16b4f Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Tue, 3 Dec 2024 16:31:46 +0000 Subject: [PATCH 6/8] docs(update README): update README after internal review --- aeon/dj_pipeline/README.md | 56 ++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/aeon/dj_pipeline/README.md b/aeon/dj_pipeline/README.md index 27f21310..698a74ac 100644 --- a/aeon/dj_pipeline/README.md +++ b/aeon/dj_pipeline/README.md @@ -9,15 +9,19 @@ The following diagrams provide a high-level overview of the pipeline's component The diagram below illustrates the structure of the **acquisition-related tasks within the pipeline**, focusing on the most relevant subset of tables. -![datajoint_overview_acquisition_diagram](./docs/datajoint_overview_acquisition_related_diagram.svg) + The diagram below represents the **data stream flow within the pipeline**, highlighting the subset of tables critical to understanding the process. -![datajoint_overview_data_stream_diagram](./docs/datajoint_overview_data_stream_diagram.svg) + + +The diagram below illustrates the **Pyrat synchronization process within the pipeline**, highlighting the key tables involved in syncing data across different components. + + The diagram below shows the **analysis portion of the pipeline**. -![datajoint_analysis_pipeline](./docs/datajoint_analysis_diagram.svg) + The pipeline is structured into hierarchical layers of tables, which are depicted in the diagrams above. These layers include: @@ -29,6 +33,7 @@ The pipeline is structured into hierarchical layers of tables, which are depicte Data flows through the pipeline in a top-down manner, driven by a combination of ingestion and computation routines. This layered organization facilitates efficient data processing and modular analysis. + ## Core tables #### Experiment and data acquisition @@ -54,13 +59,48 @@ any particular experiment (in the above `aquisition.Experiment` table). A chunk #### Standard analyses -+ `Visit` - a `Visit` is defined as a ***period of time*** during which a particular ***animal*** remains at a specific ***place***. ++ `Visit` - a `Visit` is defined as a period of time during which a particular animal remains at a specific place. + ++ `Block` - a `Block` refers to a specific period of time, typically lasting around 3 hours, during which the reward rate for each patch is predefined to facilitate certain animal behaviors. -+ `BlockAnalysis` - higher-level aggregation of events and metrics that occur within a defined block of time during an experiment. It integrates data from multiple subjects, positions, and interactions to enable further analysis of behavior and environmental interactions. ++ `BlockAnalysis` - A higher-level aggregation of events and metrics occurring within a defined block of time during an experiment.This analysis computes patch-related and subject-related metrics separately, without combining or cross-correlating data between them. It provides an overview of behavior and environmental interactions at a broader level, integrating data from multiple subjects and patches. -+ `BlockSubjectAnalysis` - A detailed analysis for each subject in a given block, partitioned into the following components: - - `Patch`: tracks the interactions of each subject with specific patches (areas of interests) - - `Preference`: measures a subject's preference for specific patches using various analyses, including cumulative preferences based on time spent and distance traveled in relation to each patch. ++ `BlockSubjectAnalysis` - A detailed analysis focusing on individual subjects within a block, explicitly combining a subject's interactions with specific patches. This involves examining how a subject interacts with a particular patch, including metrics such as total interaction time and overall time spent at the patch. Key components include: + - `Patch`: Tracks the interactions of each subject with specific patches (areas of interests) + - `Preference`: Measures a subject's preference for specific patches using various analyses, including cumulative preferences based on time spent and distance traveled in relation to each patch. #### Data stream ++ `SpinnakerVideoSource` - Tracks the placement and operation of a Spinnaker video source at a specific location during a defined period in an experiment. This class includes metadata such as the installation time of the device, enabling the tracking of video data associated with the device. + ++ `RfidReader` - Represents the placement and operation of an RFID reader at a specific location for a defined period in an experiment. It records the installation time of the device and facilitates the collection of RFID event data, such as RFID tag detections. The data stream includes sample counts, timestamps, and RFID tag events. + ++ `WeightScale` - Monitors the placement and operation of a weight scale within an experiment. It records the installation time of the weight scale and other related metadata, enabling the collection of weight measurements. The data streams include sample counts, timestamps, and weight readings. + ++ `UndergroundFeeder` - Tracks the operation of an underground feeder device in a specific experiment. It stores installation time and other metadata, such as configuration and calibration settings. The data stream includes events like beam breaks, pellet deliveries, and depletion states. + +## Operating the pipeline - how the auto ingestion/processing work? + +Some meta information about the experiment is entered - e.g. experiment name, participating +animals, cameras, food patches setup, etc. + ++ These information are either entered by hand, or parsed and inserted from configuration + yaml files. ++ For experiments these info can be inserted by running + + [create_experiment_01](create_experiments/create_experiment_01.py) + + [create_socialexperiment_0](create_experiments/create_socialexperiment_0.py) + + [create_experiment_02](create_experiments/create_experiment_02.py) + (just need to do this once) + +Tables in DataJoint are written with a `make()` function - +instruction to generate and insert new records to itself, based on data from upstream tables. +Triggering the auto ingestion and processing/computation routine is essentially +calling the `.populate()` method for all relevant tables. + +These routines are prepared in this [auto-processing script](populate/process.py). +Essentially, turning on the auto-processing routine amounts to running the +following 2 commands (in different processing threads) + + aeon_ingest high + + aeon_ingest mid From babcec9696d08ed403c2ae21d8ce905cd1ee1bdf Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Tue, 3 Dec 2024 17:30:47 +0000 Subject: [PATCH 7/8] docs(update README): updates from second review --- aeon/dj_pipeline/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/aeon/dj_pipeline/README.md b/aeon/dj_pipeline/README.md index 698a74ac..7d898168 100644 --- a/aeon/dj_pipeline/README.md +++ b/aeon/dj_pipeline/README.md @@ -38,7 +38,7 @@ Data flows through the pipeline in a top-down manner, driven by a combination of #### Experiment and data acquisition -+ `Experiment` - the `aquisition.Experiment` table stores meta information about the experiments ++ `Experiment` - The `aquisition.Experiment` table stores meta information about the experiments done in Project Aeon, with secondary information such as the lab/room the experiment is carried out, which animals participating, the directory storing the raw data, etc. @@ -46,22 +46,24 @@ which animals participating, the directory storing the raw data, etc. The `aquisition.Epoch` table records all acquisition epochs and their associated configuration for any particular experiment (in the above `aquisition.Experiment` table). -+ `Chunk` - the raw data are acquired by Bonsai and stored as ++ `Chunk` - The raw data are acquired by Bonsai and stored as a collection of files every one hour - we call this one-hour a time chunk. The `aquisition.Chunk` table records all time chunks and their associated raw data files for any particular experiment (in the above `aquisition.Experiment` table). A chunk must belong to one epoch. #### Position data -+ `qc.CameraQC` - quality control procedure applied to each `ExperimentCamera` (e.g. missing frame, etc.) ++ `qc.CameraQC` - A quality control procedure applied to each `ExperimentCamera` (e.g. missing frame, etc.) -+ `tracking.SLEAPTracking` - position tracking for object(s), from a particular `VideoSource` per chunk ++ `tracking.SLEAPTracking` - Position tracking of object(s) from a particular `VideoSource` per chunk. Key tables include: + - `PoseIdentity` - Identifies the Subject (i.e. Identity) and stores the name of the body part used as "anchor_part". + - `Part` - Contains the inferred x,y positions over time for all body parts, as derived from the SLEAP model. #### Standard analyses -+ `Visit` - a `Visit` is defined as a period of time during which a particular animal remains at a specific place. ++ `Visit` - A `Visit` is defined as a period of time during which a particular animal remains at a specific place. -+ `Block` - a `Block` refers to a specific period of time, typically lasting around 3 hours, during which the reward rate for each patch is predefined to facilitate certain animal behaviors. ++ `Block` - A `Block` refers to a specific period of time, typically lasting around 3 hours, during which the reward rate for each patch is predefined to facilitate certain animal behaviors. + `BlockAnalysis` - A higher-level aggregation of events and metrics occurring within a defined block of time during an experiment.This analysis computes patch-related and subject-related metrics separately, without combining or cross-correlating data between them. It provides an overview of behavior and environmental interactions at a broader level, integrating data from multiple subjects and patches. From b2cd47953177a701b4f531528ad09c9a3f2aee40 Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Tue, 3 Dec 2024 18:04:28 +0000 Subject: [PATCH 8/8] docs(add new images): updates for the second round of review --- ...datajoint_overview_data_stream_diagram.svg | 601 ++++++++---------- aeon/dj_pipeline/docs/notebooks/diagram.ipynb | 21 +- 2 files changed, 280 insertions(+), 342 deletions(-) diff --git a/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg index df4eee45..c41f2603 100644 --- a/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg +++ b/aeon/dj_pipeline/docs/datajoint_overview_data_stream_diagram.svg @@ -1,476 +1,407 @@ - - - - + + + + -RfidReader.Attribute - - -RfidReader.Attribute +UndergroundFeederDepletionState + + +UndergroundFeederDepletionState - + -UndergroundFeederRetriedDelivery - - -UndergroundFeederRetriedDelivery +UndergroundFeederEncoder + + +UndergroundFeederEncoder - + -UndergroundFeeder.RemovalTime - - -UndergroundFeeder.RemovalTime +StreamType + + +StreamType - - -WeightScaleWeightFiltered - - -WeightScaleWeightFiltered + + +DeviceType.Stream + + +DeviceType.Stream - - -Device - - -Device - - + + +StreamType->DeviceType.Stream + - - -UndergroundFeeder - - -UndergroundFeeder + + +Experiment + + +Experiment - - -Device->UndergroundFeeder - - - - -SpinnakerVideoSource - - -SpinnakerVideoSource + + +Chunk + + +Chunk - + -Device->SpinnakerVideoSource - +Experiment->Chunk + - - -RfidReader - - -RfidReader + + +Epoch + + +Epoch - + -Device->RfidReader - +Experiment->Epoch + - + WeightScale - - -WeightScale + + +WeightScale - + -Device->WeightScale - +Experiment->WeightScale + - - -SpinnakerVideoSource.Attribute - - -SpinnakerVideoSource.Attribute + + +UndergroundFeeder + + +UndergroundFeeder - - -SpinnakerVideoSourceVideo - - -SpinnakerVideoSourceVideo - - + + +Experiment->UndergroundFeeder + - - -RfidReaderRfidEvents - - -RfidReaderRfidEvents + + +Experiment.Directory + + +Experiment.Directory - - -Experiment - - -Experiment - - + + +Experiment->Experiment.Directory + - - -Experiment->UndergroundFeeder - + + +RfidReader + + +RfidReader + - - -Experiment->SpinnakerVideoSource - Experiment->RfidReader - + - + + +SpinnakerVideoSource + + +SpinnakerVideoSource + + + + -Experiment->WeightScale - +Experiment->SpinnakerVideoSource + - - -Chunk - - -Chunk + + +WeightScaleWeightRaw + + +WeightScaleWeightRaw - - -Experiment->Chunk - - - - -UndergroundFeeder.Attribute - - -UndergroundFeeder.Attribute + + +UndergroundFeederBeamBreak + + +UndergroundFeederBeamBreak - - -WeightScale.Attribute - - -WeightScale.Attribute + + +UndergroundFeederRetriedDelivery + + +UndergroundFeederRetriedDelivery - + + +Chunk->UndergroundFeederDepletionState + + + -UndergroundFeeder->UndergroundFeederRetriedDelivery - +Chunk->UndergroundFeederEncoder + - + -UndergroundFeeder->UndergroundFeeder.RemovalTime - +Chunk->WeightScaleWeightRaw + - + -UndergroundFeeder->UndergroundFeeder.Attribute - - - - -UndergroundFeederManualDelivery - - -UndergroundFeederManualDelivery - - +Chunk->UndergroundFeederBeamBreak + - + -UndergroundFeeder->UndergroundFeederManualDelivery - +Chunk->UndergroundFeederRetriedDelivery + - - -UndergroundFeederMissedPellet - - -UndergroundFeederMissedPellet + + +RfidReaderRfidEvents + + +RfidReaderRfidEvents - + -UndergroundFeeder->UndergroundFeederMissedPellet - +Chunk->RfidReaderRfidEvents + - - -UndergroundFeederDeliverPellet - - -UndergroundFeederDeliverPellet + + +SpinnakerVideoSourceVideo + + +SpinnakerVideoSourceVideo - + -UndergroundFeeder->UndergroundFeederDeliverPellet - +Chunk->SpinnakerVideoSourceVideo + - - -UndergroundFeederBeamBreak - - -UndergroundFeederBeamBreak + + +WeightScaleWeightFiltered + + +WeightScaleWeightFiltered - + -UndergroundFeeder->UndergroundFeederBeamBreak - +Chunk->WeightScaleWeightFiltered + - - -UndergroundFeederDepletionState - - -UndergroundFeederDepletionState + + +UndergroundFeederMissedPellet + + +UndergroundFeederMissedPellet - + -UndergroundFeeder->UndergroundFeederDepletionState - +Chunk->UndergroundFeederMissedPellet + - - -UndergroundFeederEncoder - - -UndergroundFeederEncoder + + +UndergroundFeederManualDelivery + + +UndergroundFeederManualDelivery - + -UndergroundFeeder->UndergroundFeederEncoder - - - - -WeightScaleWeightRaw - - -WeightScaleWeightRaw - - - - - -StreamType - - -StreamType - - +Chunk->UndergroundFeederManualDelivery + - - -DeviceType.Stream - - -DeviceType.Stream + + +UndergroundFeederDeliverPellet + + +UndergroundFeederDeliverPellet - + -StreamType->DeviceType.Stream - +Chunk->UndergroundFeederDeliverPellet + - + -SpinnakerVideoSource->SpinnakerVideoSource.Attribute - +Epoch->Chunk + - + -SpinnakerVideoSource->SpinnakerVideoSourceVideo - - - - -SpinnakerVideoSource.RemovalTime - - -SpinnakerVideoSource.RemovalTime - - +WeightScale->WeightScaleWeightRaw + - + -SpinnakerVideoSource->SpinnakerVideoSource.RemovalTime - - - - -DeviceType - - -DeviceType - - +WeightScale->WeightScaleWeightFiltered + - + -DeviceType->Device - +UndergroundFeeder->UndergroundFeederDepletionState + - + -DeviceType->DeviceType.Stream - +UndergroundFeeder->UndergroundFeederEncoder + - + -RfidReader->RfidReader.Attribute - +UndergroundFeeder->UndergroundFeederBeamBreak + - + -RfidReader->RfidReaderRfidEvents - - - - -RfidReader.RemovalTime - - -RfidReader.RemovalTime - - +UndergroundFeeder->UndergroundFeederRetriedDelivery + - + -RfidReader->RfidReader.RemovalTime - +UndergroundFeeder->UndergroundFeederMissedPellet + - + -WeightScale->WeightScaleWeightFiltered - +UndergroundFeeder->UndergroundFeederManualDelivery + - + -WeightScale->WeightScale.Attribute - +UndergroundFeeder->UndergroundFeederDeliverPellet + - + -WeightScale->WeightScaleWeightRaw - - - - -WeightScale.RemovalTime - - -WeightScale.RemovalTime - +Experiment.Directory->Chunk + - - + -WeightScale->WeightScale.RemovalTime - +Experiment.Directory->Epoch + - + + +Device + + +Device + + + + -Chunk->UndergroundFeederRetriedDelivery - +Device->WeightScale + - + -Chunk->WeightScaleWeightFiltered - +Device->UndergroundFeeder + - + -Chunk->SpinnakerVideoSourceVideo - +Device->RfidReader + - + -Chunk->RfidReaderRfidEvents - +Device->SpinnakerVideoSource + - + + +DeviceType + + +DeviceType + + + + -Chunk->UndergroundFeederManualDelivery - +DeviceType->DeviceType.Stream + - + -Chunk->WeightScaleWeightRaw - +DeviceType->Device + - + -Chunk->UndergroundFeederMissedPellet - +RfidReader->RfidReaderRfidEvents + - + -Chunk->UndergroundFeederDeliverPellet - - - - -Chunk->UndergroundFeederBeamBreak - - - - -Chunk->UndergroundFeederDepletionState - - - - -Chunk->UndergroundFeederEncoder - +SpinnakerVideoSource->SpinnakerVideoSourceVideo + \ No newline at end of file diff --git a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb index 5de05e3c..c16763fb 100644 --- a/aeon/dj_pipeline/docs/notebooks/diagram.ipynb +++ b/aeon/dj_pipeline/docs/notebooks/diagram.ipynb @@ -113,13 +113,20 @@ "outputs": [], "source": [ "# data streams\n", - "\n", - "data_stream_diagram = (\n", - " dj.Diagram(streams) + \n", - " acquisition.Experiment\n", - ")\n", - "\n", - "data_stream_diagram" + "data_stream_diagram =(\n", + " dj.Diagram(streams) \n", + " - streams.SpinnakerVideoSource.RemovalTime\n", + " - streams.SpinnakerVideoSource.Attribute\n", + " - streams.UndergroundFeeder.Attribute\n", + " - streams.UndergroundFeeder.RemovalTime\n", + " - streams.RfidReader.Attribute\n", + " - streams.RfidReader.RemovalTime\n", + " - streams.WeightScale.Attribute\n", + " - streams.WeightScale.RemovalTime\n", + " + acquisition.Experiment\n", + " + acquisition.Epoch\n", + " ) \n", + "data_stream_diagram " ] }, {