Merge pull request #10 from AllenNeuralDynamics:bc-refactor-schemas

Refactor calibration models

README.md

@@ -2,11 +2,12 @@
 
 A repository containing code for data acquisition and processing for AIND behavior rigs.
 
-## Deployment
+---
 
-To deploy the Bonsai code, run `./bonsai/setup.cmd`. This small script will download and regenerate the current bonsai environment ([see tutorial for further details.](https://bonsai-rx.org/docs/articles/environments.html))
+## Deployment
 
-To deploy the Python code, regenerate an environment with `requirements.txt`
+Install the [prerequisites](#prerequisites) mentioned below.
+From the root of the repository, run `./scripts/deploy.ps1` to bootstrap both python and bonsai environments.
 
 ---
 
@@ -15,6 +16,7 @@ To deploy the Python code, regenerate an environment with `requirements.txt`
 These should only need to be installed once on a fresh new system, and are not required if simply refreshing the install or deploying to a new folder.
 
 - Windows 10 or 11
+- [Python >3.11](https://www.python.org/downloads/) (Required for the launcher and highly recommended for generating valid data schemas)
 - [Visual Studio Code](https://code.visualstudio.com/) (highly recommended for editing code scripts and git commits)
 - [Git for Windows](https://gitforwindows.org/) (highly recommended for cloning and manipulating this repository)
 - [.NET Framework 4.7.2 Developer Pack](https://dotnet.microsoft.com/download/dotnet-framework/thank-you/net472-developer-pack-offline-installer) (required for intellisense when editing code scripts)
@@ -25,14 +27,35 @@ These should only need to be installed once on a fresh new system, and are not r
 
 ---
 
-## Notes on data-schema regeneration
+## Generating valid JSON input files
+
+One of the core principles of this repository is the strict adherence to [json-schemas](https://json-schema.org/). We use [Pydantic](https://pydantic.dev/) as a way to write and compile our schemas, but also to generate valid JSON input files. These files can be used by Bonsai (powered by [Bonsai.SGen](https://github.com/bonsai-rx/sgen) code generation tool) or to simply record metadata. Examples of how to interact with the library can be found in the `./examples` folder.
+
+---
+
+## Regenerating schemas
+
+Once a Pydantic model is updated, updates to all downstream dependencies must be made to ensure that the ground-truth data schemas (and all dependent interoperability tools) are also updated. This can be achieved by running the `./scripts/renegerate.ps1` script from the root of the repository.
+This script will regenerate all `json-schemas` along with `C#` code (`./scr/Extensions`) used by the Bonsai environment.
+
+---
+
+## Contributors
+
+Contributions to this repository are welcome! However, please ensure that your code adheres to the recommended DevOps practices below:
+
+### Linting
+
+We use [flake8](https://flake8.pycqa.org/), [black](https://black.readthedocs.io/), and [isort](https://pycqa.github.io/isort/) as our linting tools.
 
- 1 - Install [bonsai.sgen dotnet tool](https://github.com/bonsai-rx/sgen) by regenerating from `.config/dotnet-tools.json` by running `dotnet tool restore` in the root of the repository.
+### Testing
 
- 2 - Run `bonsai.sgen` targeting the root schema in `src\DataSchemas`. E.g.:
+Attempt to add tests when new features are added.
+To run the currently available tests, run `python -m unittest` from the root of the repository.
 
- 3 - Regenerate by running `src\DataSchemas\regenerate.cmd`
+### Versioning
 
+Where possible, adhere to [Semantic Versioning](https://semver.org/).
 
 ## Project dependency tree
 

examples/aind_manipulator.py

@@ -1,9 +1,11 @@
 from aind_behavior_services.calibration import aind_manipulator as m
 from aind_behavior_services.base import get_commit_hash
+from aind_behavior_services.session import AindBehaviorSessionModel
 import datetime
 
 
-calibration = m.AindManipulatorCalibration(
+calibration_data = m.AindManipulatorCalibration(
+    name="AindManipulatorCalibration",
     input=m.AindManipulatorCalibrationInput(
         full_step_to_mm=m.ManipulatorPosition(x=0.01, y1=0.01, y2=0.01, z=0.01),
         axis_configuration=[
@@ -19,17 +21,22 @@
     calibration_date=datetime.datetime.now(),
 )
 
+calibration_logic = m.AindManipulatorCalibrationLogic()
 
-out_model = m.AindManipulatorCalibrationModel(
-    calibration=calibration,
+
+calibration_session = AindBehaviorSessionModel(
     root_path="C:\\Data",
+    remote_path=None,
     allow_dirty_repo=False,
-    experiment="AindManipulatorSettings",
+    experiment="AindManipulatorCalibration",
+    date=datetime.datetime.now(),
     subject="AindManipulator",
     experiment_version="manipulator_control",
-    commit_hash=get_commit_hash()
+    commit_hash=get_commit_hash(),
 )
 
-
-with open("local/aind_manipulator.json", "w") as f:
-    f.write(out_model.model_dump_json(indent=3))
+seed_path = "local/aind_manipulator_{suffix}.json"
+with open(seed_path.format(suffix="calibration_logic"), "w") as f:
+    f.write(calibration_logic.model_dump_json(indent=3))
+with open(seed_path.format(suffix="session"), "w") as f:
+    f.write(calibration_session.model_dump_json(indent=3))

examples/load_cells.py

@@ -1,12 +1,12 @@
-from datetime import datetime
+import datetime
+from aind_behavior_services.session import AindBehaviorSessionModel
 from aind_behavior_services.base import get_commit_hash
 from aind_behavior_services.calibration.load_cells import (
     LoadCellCalibration,
     LoadCellsCalibration,
     LoadCellsCalibrationInput,
     LoadCellsCalibrationOutput,
-    LoadCellsCalibrationModel,
-    LoadCellsOperationControl,
+    LoadCellsCalibrationLogic,
 )
 
 
@@ -20,25 +20,28 @@
     weight_lookup={0: (0, 0), 1: (0, 0)},
 )
 
-lc_calibration = LoadCellsCalibration(
+calibration = LoadCellsCalibration(
     input=lc_calibration_input,
     output=lc_calibration_output,
     device_name="LoadCells",
-    calibration_date=datetime.now(),
+    calibration_date=datetime.datetime.now(),
 )
 
-lc_op = LoadCellsOperationControl(channels=[0, 1], offset_buffer_size=10)
+calibration_logic = LoadCellsCalibrationLogic(channels=[0, 1], offset_buffer_size=10)
 
-out_model = LoadCellsCalibrationModel(
-    calibration=lc_calibration,
-    operation_control=lc_op,
+calibration_session = AindBehaviorSessionModel(
     root_path="C:\\Data",
+    remote_path=None,
     allow_dirty_repo=False,
     experiment="LoadCellsCalibration",
+    date=datetime.datetime.now(),
     subject="LoadCells",
-    experiment_version="LoadCellsCalibration",
-    commit_hash=get_commit_hash()
+    experiment_version="load_cells",
+    commit_hash=get_commit_hash(),
 )
 
-with open("local/load_cells.json", "w") as f:
-    f.write(out_model.model_dump_json(indent=3))
+seed_path = "local/load_cells_{suffix}.json"
+with open(seed_path.format(suffix="calibration_logic"), "w") as f:
+    f.write(calibration_logic.model_dump_json(indent=3))
+with open(seed_path.format(suffix="session"), "w") as f:
+    f.write(calibration_session.model_dump_json(indent=3))

examples/olfactometer.py

@@ -1,61 +1,69 @@
-from aind_behavior_services.base import get_commit_hash
 import datetime
 
+from aind_behavior_services.session import AindBehaviorSessionModel
+from aind_behavior_services.base import get_commit_hash
+
 from aind_behavior_services.calibration.olfactometer import (
-    HarpOlfactometerChannel,
+    OlfactometerChannelType,
     OlfactometerChannel,
     OlfactometerChannelConfig,
-    OlfactometerOperationControl,
-    OlfactometerCalibrationModel,
-    ChannelType
+    OlfactometerCalibrationLogic,
+    OlfactometerCalibration, OlfactometerCalibrationInput, OlfactometerCalibrationOutput
 )
 
 
 channels_config = [
-    OlfactometerChannel(
-        channel_index=HarpOlfactometerChannel.Channel0, channel_type=ChannelType.ODOR, flow_capacity=100
+    OlfactometerChannelConfig(
+        channel_index=OlfactometerChannel.Channel0,
+        channel_type=OlfactometerChannelType.ODOR,
+        flow_rate=100,
+        odorant="Banana",
+        odorant_dilution=0.1,
     ),
-    OlfactometerChannel(
-        channel_index=HarpOlfactometerChannel.Channel1, channel_type=ChannelType.ODOR, flow_capacity=100
+    OlfactometerChannelConfig(
+        channel_index=OlfactometerChannel.Channel1,
+        channel_type=OlfactometerChannelType.ODOR,
+        flow_rate=100,
+        odorant="Strawberry",
+        odorant_dilution=0.1,
     ),
-    OlfactometerChannel(
-        channel_index=HarpOlfactometerChannel.Channel2, channel_type=ChannelType.ODOR, flow_capacity=100
+    OlfactometerChannelConfig(
+        channel_index=OlfactometerChannel.Channel2,
+        channel_type=OlfactometerChannelType.ODOR,
+        flow_rate=100,
+        odorant="Apple",
+        odorant_dilution=0.1,
     ),
-    OlfactometerChannel(
-        channel_index=HarpOlfactometerChannel.Channel3, channel_type=ChannelType.CARRIER, flow_capacity=1000
+    OlfactometerChannelConfig(
+        channel_index=OlfactometerChannel.Channel3, channel_type=OlfactometerChannelType.CARRIER, odorant="Air"
     ),
 ]
-channels_config = {channel.channel_index: channel for channel in channels_config}
 
-stimuli_config = [
-    OlfactometerChannelConfig(channel_index=HarpOlfactometerChannel.Channel0, odorant="Banana", odorant_dilution=0.1),
-    OlfactometerChannelConfig(channel_index=HarpOlfactometerChannel.Channel1, odorant="Banana", odorant_dilution=0.1),
-    OlfactometerChannelConfig(channel_index=HarpOlfactometerChannel.Channel2, odorant="Banana", odorant_dilution=0.1),
-]
-
-stimuli_config = {channel.channel_index: channel for channel in stimuli_config}
+channels_config = {channel.channel_index: channel for channel in channels_config}
 
+calibration = OlfactometerCalibration(input=OlfactometerCalibrationInput(), output=OlfactometerCalibrationOutput())
 
-OpControl = OlfactometerOperationControl(
+calibration_logic = OlfactometerCalibrationLogic(
     channel_config=channels_config,
-    stimulus_config=stimuli_config,
     full_flow_rate=1000,
     n_repeats_per_stimulus=10,
     time_on=1,
     time_off=1,
 )
 
-out_model = OlfactometerCalibrationModel(
-    operation_control=OpControl,
-    calibration=None,
+calibration_session = AindBehaviorSessionModel(
     root_path="C:\\Data",
+    remote_path=None,
     date=datetime.datetime.now(),
     allow_dirty_repo=False,
     experiment="OlfactometerCalibration",
     experiment_version="0.0.0",
     subject="Olfactometer",
-    commit_hash=get_commit_hash()
+    commit_hash=get_commit_hash(),
 )
 
-with open("local/olfactometer.json", "w") as f:
-    f.write(out_model.model_dump_json(indent=3))
+seed_path = "local/olfactometer_{suffix}.json"
+with open(seed_path.format(suffix="calibration_logic"), "w") as f:
+    f.write(calibration_logic.model_dump_json(indent=3))
+with open(seed_path.format(suffix="session"), "w") as f:
+    f.write(calibration_session.model_dump_json(indent=3))

examples/water_valve.py

@@ -1,19 +1,22 @@
-from datetime import datetime
+import datetime
+from aind_behavior_services.session import AindBehaviorSessionModel
 from aind_behavior_services.base import get_commit_hash
 from aind_behavior_services.calibration.water_valve import (
     Measurement,
     WaterValveCalibration,
     WaterValveCalibrationInput,
     WaterValveCalibrationOutput,
-    WaterValveCalibrationModel,
-    WaterValveOperationControl,
-)
+    WaterValveCalibrationLogic)
+
+def linear_model(time, slope, offset):
+    return slope * time + offset
+
 
 _delta_times = [0.1, 0.2, 0.3, 0.4, 0.5]
 _slope = 10.1
 _offset = -0.3
-_linear_model = lambda time: _slope * time + _offset
-_water_weights = [_linear_model(x) for x in _delta_times]
+
+_water_weights = [linear_model(x, _slope, _offset) for x in _delta_times]
 _inputs = [
     Measurement(valve_open_interval=0.5, valve_open_time=t[0], water_weight=[t[1]], repeat_count=1)
     for t in zip(_delta_times, _water_weights)
@@ -29,25 +32,32 @@
 )
 
 input = WaterValveCalibrationInput(measurements=_inputs)
+
 calibration = WaterValveCalibration(
     input=input,
     output=input.calibrate_output(),
     device_name="WaterValve",
-    calibration_date=datetime.now(),
+    calibration_date=datetime.datetime.now(),
 )
 
+calibration_logic = WaterValveCalibrationLogic(
+    valve_open_time=_delta_times,
+    valve_open_interval=0.5,
+    repeat_count=200)
 
-out_model = WaterValveCalibrationModel(
-    calibration=calibration,
-    operation_control=WaterValveOperationControl(valve_open_time=[0.1, 0.2, 0.3]),
+calibration_session = AindBehaviorSessionModel(
     root_path="C:\\Data",
+    remote_path=None,
     allow_dirty_repo=False,
     experiment="WaterValveCalibration",
     subject="WaterValve",
     experiment_version="WaterValveCalibration",
-    commit_hash=get_commit_hash()
+    commit_hash=get_commit_hash(),
+    date=datetime.datetime.now(),
 )
 
-
-with open("local/water_valve.json", "w") as f:
-    f.write(out_model.model_dump_json(indent=3))
+seed_path = "local/water_valve{suffix}.json"
+with open(seed_path.format(suffix="calibration_logic"), "w") as f:
+    f.write(calibration_logic.model_dump_json(indent=3))
+with open(seed_path.format(suffix="session"), "w") as f:
+    f.write(calibration_session.model_dump_json(indent=3))

pyproject.toml

@@ -13,10 +13,11 @@ classifiers = [
 dynamic = ["version", "readme"]
 
 dependencies = [
-    'aind_data_schema == 0.31.8',
-    'harp-python==0.2.0',
-    'gitpython==3.1.43',
-    'scikit-learn==1.4.1.post1'
+    'pydantic>=2.6.4, <3.0',
+    'harp-python>=0.2',
+    'gitpython>=3.1, <4.0',
+    'scikit-learn',
+    'semver'
 ]
 
 [project.optional-dependencies]

scripts/regenerate.py

@@ -1,27 +1,34 @@
 from pathlib import Path
 
-from aind_behavior_services.calibration.load_cells import LoadCellsCalibrationModel
-from aind_behavior_services.calibration.olfactometer import OlfactometerCalibrationModel
-from aind_behavior_services.calibration.water_valve import WaterValveCalibrationModel
-from aind_behavior_services.calibration.aind_manipulator import AindManipulatorCalibrationModel
+from aind_behavior_services.session import AindBehaviorSessionModel
+from aind_behavior_services.calibration.load_cells import LoadCellsCalibrationLogic
+from aind_behavior_services.calibration.olfactometer import OlfactometerCalibrationLogic
+from aind_behavior_services.calibration.water_valve import WaterValveCalibrationLogic
+from aind_behavior_services.calibration.aind_manipulator import AindManipulatorCalibrationLogic
 from aind_behavior_services.utils import convert_pydantic_to_bonsai
 
+
 SCHEMA_ROOT = Path("./src/DataSchemas/schemas")
 EXTENSIONS_ROOT = Path("./src/Extensions/")
 NAMESPACE_PREFIX = "AindBehaviorRigCalibration"
 
 
 def main():
     models = {
-        "olfactometer_calibration": OlfactometerCalibrationModel,
-        "water_valve_calibration": WaterValveCalibrationModel,
-        "load_cells_calibration": LoadCellsCalibrationModel,
-        "aind_manipulator_calibration": AindManipulatorCalibrationModel,
+        "olfactometer_calibration": OlfactometerCalibrationLogic,
+        "water_valve_calibration": WaterValveCalibrationLogic,
+        "load_cells_calibration": LoadCellsCalibrationLogic,
+        "aind_manipulator_calibration": AindManipulatorCalibrationLogic,
     }
     convert_pydantic_to_bonsai(
         models, schema_path=SCHEMA_ROOT, output_path=EXTENSIONS_ROOT, namespace_prefix=NAMESPACE_PREFIX
     )
 
+    core_models = {"aind_behavior_session": AindBehaviorSessionModel}
+    convert_pydantic_to_bonsai(
+        core_models, schema_path=SCHEMA_ROOT, output_path=EXTENSIONS_ROOT, namespace_prefix="AindBehaviorServices"
+    )
+
 
 if __name__ == "__main__":
     main()