Skip to content

Commit

Permalink
Merge branch 'release/2.39.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@oliche
oliche committed Oct 9, 2024
2 parents 3e80794 + e89f12d commit d15b12c
Show file tree
Hide file tree
Showing 65 changed files with 2,367 additions and 6,058 deletions.
117 changes: 0 additions & 117 deletions brainbox/behavior/wheel.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
"""
Set of functions to handle wheel data.
"""
import logging
import warnings
import traceback

import numpy as np
from numpy import pi
from iblutil.numerical import between_sorted
Expand Down Expand Up @@ -68,42 +64,6 @@ def interpolate_position(re_ts, re_pos, freq=1000, kind='linear', fill_gaps=None
return yinterp, t


def velocity(re_ts, re_pos):
"""
(DEPRECATED) Compute wheel velocity from non-uniformly sampled wheel data. Returns the velocity
at the same samples locations as the position through interpolation.
Parameters
----------
re_ts : array_like
Array of timestamps
re_pos: array_like
Array of unwrapped wheel positions
Returns
-------
np.ndarray
numpy array of velocities
"""
for line in traceback.format_stack():
print(line.strip())

msg = 'brainbox.behavior.wheel.velocity will soon be removed. Use velocity_filtered instead.'
warnings.warn(msg, FutureWarning)
logging.getLogger(__name__).warning(msg)

dp = np.diff(re_pos)
dt = np.diff(re_ts)
# Compute raw velocity
vel = dp / dt
# Compute velocity time scale
tv = re_ts[:-1] + dt / 2
# interpolate over original time scale
if tv.size > 1:
ifcn = interpolate.interp1d(tv, vel, fill_value="extrapolate")
return ifcn(re_ts)


def velocity_filtered(pos, fs, corner_frequency=20, order=8):
"""
Compute wheel velocity from uniformly sampled wheel data.
Expand All @@ -130,83 +90,6 @@ def velocity_filtered(pos, fs, corner_frequency=20, order=8):
return vel, acc


def velocity_smoothed(pos, freq, smooth_size=0.03):
"""
(DEPRECATED) Compute wheel velocity from uniformly sampled wheel data.
Parameters
----------
pos : array_like
Array of wheel positions
smooth_size : float
Size of Gaussian smoothing window in seconds
freq : float
Sampling frequency of the data
Returns
-------
vel : np.ndarray
Array of velocity values
acc : np.ndarray
Array of acceleration values
"""
for line in traceback.format_stack():
print(line.strip())

msg = 'brainbox.behavior.wheel.velocity_smoothed will be removed. Use velocity_filtered instead.'
warnings.warn(msg, FutureWarning)
logging.getLogger(__name__).warning(msg)

# Define our smoothing window with an area of 1 so the units won't be changed
std_samps = np.round(smooth_size * freq) # Standard deviation relative to sampling frequency
N = std_samps * 6 # Number of points in the Gaussian covering +/-3 standard deviations
gauss_std = (N - 1) / 6
win = scipy.signal.windows.gaussian(N, gauss_std)
win = win / win.sum() # Normalize amplitude

# Convolve and multiply by sampling frequency to restore original units
vel = np.insert(scipy.signal.convolve(np.diff(pos), win, mode='same'), 0, 0) * freq
acc = np.insert(scipy.signal.convolve(np.diff(vel), win, mode='same'), 0, 0) * freq

return vel, acc


def last_movement_onset(t, vel, event_time):
"""
(DEPRECATED) Find the time at which movement started, given an event timestamp that occurred during the
movement.
Movement start is defined as the first sample after the velocity has been zero for at least 50ms.
Wheel inputs should be evenly sampled.
:param t: numpy array of wheel timestamps in seconds
:param vel: numpy array of wheel velocities
:param event_time: timestamp anywhere during movement of interest, e.g. peak velocity
:return: timestamp of movement onset
"""
for line in traceback.format_stack():
print(line.strip())

msg = 'brainbox.behavior.wheel.last_movement_onset has been deprecated. Use get_movement_onset instead.'
warnings.warn(msg, FutureWarning)
logging.getLogger(__name__).warning(msg)

# Look back from timestamp
threshold = 50e-3
mask = t < event_time
times = t[mask]
vel = vel[mask]
t = None # Initialize
for i, t in enumerate(times[::-1]):
i = times.size - i
idx = np.min(np.where((t - times) < threshold))
if np.max(np.abs(vel[idx:i])) < 0.5:
break

# Return timestamp
return t


def get_movement_onset(intervals, event_times):
"""
Find the time at which movement started, given an event timestamp that occurred during the
Expand Down
28 changes: 23 additions & 5 deletions brainbox/io/one.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import spikeglx

import ibldsp.voltage
from ibldsp.waveform_extraction import WaveformsLoader
from iblutil.util import Bunch
from iblatlas.atlas import AllenAtlas, BrainRegions
from iblatlas import atlas
Expand Down Expand Up @@ -975,6 +976,21 @@ def raw_electrophysiology(self, stream=True, band='ap', **kwargs):
if cbin_file is not None:
return spikeglx.Reader(cbin_file)

def download_raw_waveforms(self, **kwargs):
"""
Downloads raw waveforms extracted from sorting to local disk.
"""
_logger.debug(f"loading waveforms from {self.collection}")
return self.one.load_object(
self.eid, "waveforms",
attribute=["traces", "templates", "table", "channels"],
collection=self._get_spike_sorting_collection("pykilosort"), download_only=True, **kwargs
)

def raw_waveforms(self, **kwargs):
wf_paths = self.download_raw_waveforms(**kwargs)
return WaveformsLoader(wf_paths[0].parent, wfs_dtype=np.float16)

def load_channels(self, **kwargs):
"""
Loads channels
Expand Down Expand Up @@ -1318,6 +1334,7 @@ class SessionLoader:
one: One = None
session_path: Path = ''
eid: str = ''
revision: str = ''
data_info: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
trials: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
wheel: pd.DataFrame = field(default_factory=pd.DataFrame, repr=False)
Expand Down Expand Up @@ -1445,7 +1462,7 @@ def load_trials(self, collection=None):
# itiDuration frequently has a mismatched dimension, and we don't need it, exclude using regex
self.one.wildcards = False
self.trials = self.one.load_object(
self.eid, 'trials', collection=collection, attribute=r'(?!itiDuration).*').to_df()
self.eid, 'trials', collection=collection, attribute=r'(?!itiDuration).*', revision=self.revision or None).to_df()
self.one.wildcards = True
self.data_info.loc[self.data_info['name'] == 'trials', 'is_loaded'] = True

Expand All @@ -1468,7 +1485,7 @@ def load_wheel(self, fs=1000, corner_frequency=20, order=8, collection=None):
"""
if not collection:
collection = self._find_behaviour_collection('wheel')
wheel_raw = self.one.load_object(self.eid, 'wheel', collection=collection)
wheel_raw = self.one.load_object(self.eid, 'wheel', collection=collection, revision=self.revision or None)
if wheel_raw['position'].shape[0] != wheel_raw['timestamps'].shape[0]:
raise ValueError("Length mismatch between 'wheel.position' and 'wheel.timestamps")
# resample the wheel position and compute velocity, acceleration
Expand Down Expand Up @@ -1498,7 +1515,7 @@ def load_pose(self, likelihood_thr=0.9, views=['left', 'right', 'body']):
# empty the dictionary so that if one loads only one view, after having loaded several, the others don't linger
self.pose = {}
for view in views:
pose_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['dlc', 'times'])
pose_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['dlc', 'times'], revision=self.revision or None)
# Double check if video timestamps are correct length or can be fixed
times_fixed, dlc = self._check_video_timestamps(view, pose_raw['times'], pose_raw['dlc'])
self.pose[f'{view}Camera'] = likelihood_threshold(dlc, likelihood_thr)
Expand All @@ -1525,7 +1542,8 @@ def load_motion_energy(self, views=['left', 'right', 'body']):
# empty the dictionary so that if one loads only one view, after having loaded several, the others don't linger
self.motion_energy = {}
for view in views:
me_raw = self.one.load_object(self.eid, f'{view}Camera', attribute=['ROIMotionEnergy', 'times'])
me_raw = self.one.load_object(
self.eid, f'{view}Camera', attribute=['ROIMotionEnergy', 'times'], revision=self.revision or None)
# Double check if video timestamps are correct length or can be fixed
times_fixed, motion_energy = self._check_video_timestamps(
view, me_raw['times'], me_raw['ROIMotionEnergy'])
Expand All @@ -1550,7 +1568,7 @@ def load_pupil(self, snr_thresh=5.):
will be considered unusable and will be discarded.
"""
# Try to load from features
feat_raw = self.one.load_object(self.eid, 'leftCamera', attribute=['times', 'features'])
feat_raw = self.one.load_object(self.eid, 'leftCamera', attribute=['times', 'features'], revision=self.revision or None)
if 'features' in feat_raw.keys():
times_fixed, feats = self._check_video_timestamps('left', feat_raw['times'], feat_raw['features'])
self.pupil = feats.copy()
Expand Down
6 changes: 0 additions & 6 deletions brainbox/tests/test_behavior.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,6 @@ def test_get_movement_onset(self):
with self.assertRaises(ValueError):
wheel.get_movement_onset(intervals, np.random.permutation(self.trials['feedback_times']))

def test_velocity_deprecation(self):
"""Ensure brainbox.behavior.wheel.velocity is removed."""
from datetime import datetime
self.assertTrue(datetime.today() < datetime(2024, 8, 1),
'remove brainbox.behavior.wheel.velocity, velocity_smoothed and last_movement_onset')


class TestTraining(unittest.TestCase):
def setUp(self):
Expand Down
21 changes: 11 additions & 10 deletions examples/data_release/data_release_behavior.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,16 @@
"source": [
"# Data Release - Behavior\n",
"\n",
"The International Brain Laboratory is a team of systems and computational neuroscientists, working collaboratively to understand the computations that support decision-making in the brain. To achieve this aim, we have developed a standardized decision-making task in mice in order that probes decision-making. The task requires mice to perform decisions by combining incoming visual evidence with internal beliefs about the dynamic structure of the environment. Please read our accompanying [paper (The International Brain Laboratory et al. 2020)](https://elifesciences.org/articles/63711) for details on the decision-making task and the experiment.\n"
"The International Brain Laboratory is a team of systems and computational neuroscientists, working collaboratively to understand the computations that support decision-making in the brain. To achieve this aim, we have developed a standardized decision-making task in mice in order that probes decision-making. The task requires mice to perform decisions by combining incoming visual evidence with internal beliefs about the dynamic structure of the environment. Please read our accompanying [paper (The International Brain Laboratory et al. 2020)]( https://doi.org/10.7554/eLife.63711) for details on the decision-making task and the experiment.\n"
]
},
{
"cell_type": "markdown",
"id": "dd157e91",
"metadata": {
"collapsed": false
},
"metadata": {},
"source": [
"## Overview of the Data\n",
"We have released behavioral data throughout learning from our standardized training pipeline, implemented across 9 labs in 7 institutions. Users can download behavioral data from mice throughout their training, and analyse the transition from novice to expert behavior unfold. The behavioral data is associated with 198 mice up until 2020-03-23, as used in [The International Brain Laboratory et al. 2020](https://elifesciences.org/articles/63711). This dataset contains notably information on the sensory stimuli presented to the mouse, as well as mouse decisions and response times.\n",
"We have released behavioral data throughout learning from our standardized training pipeline, implemented across 9 labs in 7 institutions. Users can download behavioral data from mice throughout their training, and analyse the transition from novice to expert behavior unfold. The behavioral data is associated with 198 mice up until 2020-03-23, as used in [The International Brain Laboratory et al. 2020](https://doi.org/10.7554/eLife.63711). This dataset contains notably information on the sensory stimuli presented to the mouse, as well as mouse decisions and response times.\n",
"\n",
"\n",
"## Data structure and download\n",
Expand All @@ -33,21 +31,24 @@
"* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_structure.html) to download an example dataset for one session, and get familiarised with the data structure\n",
"* [These instructions](https://int-brain-lab.github.io/iblenv/notebooks_external/data_download.html) to learn how to use the ONE-api to search and download the released datasets\n",
"* [These instructions](https://int-brain-lab.github.io/iblenv/loading_examples.html) to get familiarised with specific data loading functions\n",
"* [These instructions](https://int-brain-lab.github.io/iblenv/dj_docs/dj_public.html) for instructions on how to access this dataset via Datajoint. (warning: this method may be retired in 2023)\n",
"\n",
"\n",
"Note:\n",
"\n",
"* The tag associated to this release is `2021_Q1_IBL_et_al_Behaviour`"
"* The tag associated to this release is `2021_Q1_IBL_et_al_Behaviour`\n",
"\n",
"\n",
"## How to cite this dataset\n",
"If you are using this dataset for your research please cite the [Behavior Paper](https://doi.org/10.7554/eLife.63711) and see the **How to cite** section in the associated entry in the [AWS open data registry](https://registry.opendata.aws/ibl-behaviour/)."
]
}
],
"metadata": {
"celltoolbar": "Edit Metadata",
"kernelspec": {
"display_name": "Python [conda env:iblenv] *",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-iblenv-py"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -59,7 +60,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
"version": "3.9.16"
}
},
"nbformat": 4,
Expand Down
18 changes: 10 additions & 8 deletions examples/data_release/data_release_brainwidemap.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,16 @@
"source": [
"# Data Release - Brain Wide Map\n",
"\n",
"IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)."
"IBL aims to understand the neural basis of decision-making in the mouse by gathering a whole-brain activity map composed of electrophysiological recordings pooled from multiple laboratories. We have systematically recorded from nearly all major brain areas with Neuropixels probes, using a grid system for unbiased sampling and replicating each recording site in at least two laboratories. These data have been used to construct a brain-wide map of activity at single-spike cellular resolution during a [decision-making task]((https://elifesciences.org/articles/63711)). Please read the associated article [(IBL et al. 2023)](https://doi.org/10.1101/2023.07.04.547681). In addition to the map, this data set contains other information gathered during the task: sensory stimuli presented to the mouse; mouse decisions and response times; and mouse pose information from video recordings and DeepLabCut analysis. Please read our accompanying [technical paper](https://doi.org/10.6084/m9.figshare.21400815) for details on the experiment and data processing pipelines. To explore the data, visit [our vizualisation website](https://viz.internationalbrainlab.org/)."
]
},
{
"cell_type": "markdown",
"id": "e9be5894",
"metadata": {
"collapsed": false
},
"metadata": {},
"source": [
"## Overview of the Data\n",
"We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://www.biorxiv.org/content/10.1101/2023.07.04.547681v2).\n",
"We have released data from 459 Neuropixel recording sessions, which encompass 699 probe insertions, obtained in 139 subjects performing the IBL task across 12 different laboratories. As output of spike-sorting, there are 376730 units; of which 45085 are considered to be of good quality. In total, 138 brain regions were recorded in sufficient numbers for inclusion in IBL’s analyses [(IBL et al. 2023)](https://doi.org/10.1101/2023.07.04.547681).\n",
"\n",
"## Data structure and download\n",
"The organisation of the data follows the standard IBL data structure.\n",
Expand All @@ -38,6 +36,10 @@
"* The tag associated to this release is `Brainwidemap`\n",
"\n",
"\n",
"## How to cite this dataset\n",
"If you are using this dataset for your research please cite the [Brain Wide Map Paper](https://doi.org/10.1101/2023.07.04.547681) and see the **How to cite** section in the associated entry in the [AWS open data registry](https://registry.opendata.aws/ibl-brain-wide-map/).\n",
"\n",
"\n",
"## Updates on the data\n",
"Note: The section [Overview of the Data](#overview-of-the-data) contains the latest numbers released.\n",
"\n",
Expand All @@ -54,9 +56,9 @@
"metadata": {
"celltoolbar": "Edit Metadata",
"kernelspec": {
"display_name": "Python [conda env:iblenv] *",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-iblenv-py"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -68,7 +70,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
"version": "3.9.16"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit d15b12c

Please sign in to comment.