Merge pull request #39 from cortex-lab/tl-as-class_outputClasses

Timeline is now an object.

+dat/addLogEntry.m

@@ -1,4 +1,4 @@
-function e = addLogEntry(subject, timestamp, type, value, comments)
+function e = addLogEntry(subject, timestamp, type, value, comments, AlyxInstance)
 %DAT.ADDLOGENTRY Adds a new entry to the experiment log
 %   e = DAT.ADDLOGENTRY(subject, timestamp, type, value, comments) files a
 %   new log entry for 'subject' with the corresponding info.
@@ -26,6 +26,7 @@
 %% create and store entry
 e = entry(nextidx);
 log(nextidx) = e;
+if nargin > 5; e.AlyxInstance = AlyxInstance; end
 
 %% store updated log to *all* repos locations
 superSave(dat.logPath(subject, 'all'), struct('log', log));

+dat/listSubjects.m

@@ -1,17 +1,43 @@
-function subjects = listSubjects()
+function subjects = listSubjects(varargin)
 %DAT.LISTSUBJECTS Lists recorded subjects
-%   subjects = DAT.LISTSUBJECTS() Lists the experimental subjects present
+%   subjects = DAT.LISTSUBJECTS([alyxInstance]) Lists the experimental subjects present
 %   in experiment info repository ('expInfo').
 %
+% Optional input argument of an alyx instance will enable generating this
+% list from alyx rather than from the directory structure on zserver
+%
 % Part of Rigbox
 
 % 2013-03 CB created
+% 2018-01 NS added alyx compatibility
 
-% The master 'expInfo' repository is the reference for the existence of
-% experiments, as given by the folder structure
-expInfoPath = dat.reposPath('expInfo', 'master');
-
-dirs = file.list(expInfoPath, 'dirs');
-subjects = setdiff(dirs, {'misc'}); %exclude the misc directory
-
+if nargin>0 && ~isempty(varargin{1}) % user provided an alyx instance
+    ai = varargin{1}; % an alyx instance
+
+    % get list of all living, non-stock mice from alyx
+    s = alyx.getData(ai, 'subjects?stock=False&alive=True');
+
+    % determine the user for each mouse
+    respUser = cellfun(@(x)x.responsible_user, s, 'uni', false);
+
+    % get cell array of subject names
+    subjNames = cellfun(@(x)x.nickname, s, 'uni', false);
+
+    % determine which subjects belong to this user
+    thisUserSubs = sort(subjNames(strcmp(respUser, ai.username)));
+
+    % all the subjects
+    otherUserSubs = sort(subjNames(~strcmp(respUser, ai.username)));
+
+    % the full, ordered list
+    subjects = [{'default'}, thisUserSubs, otherUserSubs]';
+else
+
+    % The master 'expInfo' repository is the reference for the existence of
+    % experiments, as given by the folder structure
+    expInfoPath = dat.reposPath('expInfo', 'master');
+
+    dirs = file.list(expInfoPath, 'dirs');
+    subjects = setdiff(dirs, {'misc'}); %exclude the misc directory
+end
 end

+dat/loadBlock.m

@@ -1,5 +1,5 @@
 function block = loadBlock(varargin)
-%loadBlock Load the designated experiment block(s)
+%DAT.LOADBLOCK Load the designated experiment block(s)
 %   BLOCK = loadBlock(EXPREF, [EXPTYPE])
 %
 %   BLOCK = loadBlock(SUBJECTREF, EXPDATE, EXPSEQ, [EXPTYPE])

+dat/newExp.m

@@ -1,6 +1,20 @@
-function [expRef, expSeq] = newExp(subject, expDate, expParams)
+function [expRef, expSeq, url] = newExp(subject, expDate, expParams, AlyxInstance)
 %DAT.NEWEXP Create a new unique experiment in the database
-%   [ref, seq] = DAT.NEWEXP(subject, expDate, expParams) TODO
+%   [ref, seq, url] = DAT.NEWEXP(subject, expDate, expParams[, AlyxInstance])
+%   Create a new experiment by creating the relevant folder tree in the
+%   local and main data repositories in the following format:
+%
+%   subject/
+%          |_ YYYY-MM-DD/
+%                       |_ expSeq/
+%
+%   If experiment parameters are passed into the function, they are saved
+%   here, as a mat and in JSON (if possible).  If an instance of Alyx is
+%   passed and a base session for the experiment date is not found, one is
+%   created in the Alyx database. A corresponding subsession is also
+%   created and the parameters file is registered with the sub-session.
+%
+%   See also DAT.PATHS
 %
 % Part of Rigbox
 
@@ -16,6 +30,11 @@
   expParams = [];
 end
 
+if (nargin < 4 || isempty(AlyxInstance)) && ~strcmp(subject, 'default')
+  % no instance of Alyx, don't create session on Alyx
+  AlyxInstance = alyx.loginWindow;
+end
+
 if ischar(expDate)
   % if the passed expDate is a string, parse it into a datenum
   expDate = datenum(expDate, 'yyyy-mm-dd');
@@ -29,8 +48,7 @@
 [~, dateList, seqList] = dat.listExps(subject);
 
 % filter the list by expdate
-expDate = floor(expDate);
-filterIdx = dateList == expDate;
+filterIdx = dateList == floor(expDate);
 
 % find the next sequence number
 expSeq = max(seqList(filterIdx)) + 1;
@@ -40,14 +58,61 @@
 end
 
 % expInfo repository is the reference location for which experiments exist
-[expPath, expRef] = dat.expPath(subject, expDate, expSeq, 'expInfo');
+[expPath, expRef] = dat.expPath(subject, floor(expDate), expSeq, 'expInfo');
 % ensure nothing went wrong in making a "unique" ref and path to hold
 assert(~any(file.exists(expPath)), ...
   sprintf('Something went wrong as experiment folders already exist for "%s".', expRef));
 
 % now make the folder(s) to hold the new experiment
 assert(all(cellfun(@(p) mkdir(p), expPath)), 'Creating experiment directories failed');
 
+if ~strcmp(subject, 'default') % Ignore fake subject
+  % if the Alyx Instance is set, find or create BASE session
+  expDate = alyx.datestr(expDate); % date in Alyx format
+  % Get list of base sessions
+  sessions = alyx.getData(AlyxInstance,...
+    ['sessions?type=Base&subject=' subject]);
+
+  %If the date of this latest base session is not the same date as
+  %today, then create a new base session for today
+  if isempty(sessions) || ~strcmp(sessions{end}.start_time(1:10), expDate(1:10))
+    d = struct;
+    d.subject = subject;
+    d.procedures = {'Behavior training/tasks'};
+    d.narrative = 'auto-generated session';
+    d.start_time = expDate;
+    d.type = 'Base';
+    %       d.users = {AlyxInstance.username};
+
+    base_submit = alyx.postData(AlyxInstance, 'sessions', d);
+    assert(isfield(base_submit,'subject'),...
+      'Submitted base session did not return appropriate values');
+
+    %Now retrieve the sessions again
+    sessions = alyx.getData(AlyxInstance,...
+      ['sessions?type=Base&subject=' subject]);
+  end
+  latest_base = sessions{end};
+
+  %Now create a new SUBSESSION, using the same experiment number
+  d = struct;
+  d.subject = subject;
+  d.procedures = {'Behavior training/tasks'};
+  d.narrative = 'auto-generated session';
+  d.start_time = expDate;
+  d.type = 'Experiment';
+  d.parent_session = latest_base.url;
+  d.number = expSeq;
+  %   d.users = {AlyxInstance.username};
+
+  subsession = alyx.postData(AlyxInstance, 'sessions', d);
+  assert(isfield(subsession,'subject'),...
+    'Failed to create new sub-session in Alyx for %s', subject);
+  url = subsession.url;
+else
+  url = [];
+end
+
 % if the parameters had an experiment definition function, save a copy in
 % the experiment's folder
 if isfield(expParams, 'defFunction')
@@ -61,5 +126,29 @@
 % now save the experiment parameters variable
 superSave(dat.expFilePath(expRef, 'parameters'), struct('parameters', expParams));
 
-
+if ~isempty(expParams)
+  try  % save a copy of parameters in json
+    % First, change all functions to strings
+    f_idx = structfun(@(s)isa(s, 'function_handle'), expParams);
+    fields = fieldnames(expParams);
+    paramCell = struct2cell(expParams);
+    paramCell(f_idx) = cellfun(@func2str, paramCell(f_idx),'UniformOutput', false);
+    expParams = cell2struct(paramCell, fields);
+    % Generate JSON path and save
+    jsonPath = fullfile(fileparts(dat.expFilePath(expRef, 'parameters', 'master')),...
+      [expRef, '_parameters.json']);
+    savejson('parameters', expParams, jsonPath);
+    % Register our JSON parameter set to Alyx
+    if ~strcmp(subject, 'default')
+      alyx.registerFile(jsonPath, 'json', url, 'Parameters', [], AlyxInstance);
+    end
+  catch ex
+    warning(ex.identifier, 'Failed to save paramters as JSON: %s.\n Registering mat file instead', ex.message)
+    % Register our parameter set to Alyx
+    if ~strcmp(subject, 'default')
+      alyx.registerFile(dat.expFilePath(expRef, 'parameters', 'master'), 'mat',...
+        url, 'Parameters', [], AlyxInstance);
+    end
+  end
+end
 end

+dat/parseAlyxInstance.m

@@ -1,28 +1,42 @@
-function varargout = parseAlyxInstance(varargin)
-%DATA.PARSEALYXINSTANCE Converts input to string for UDP message and back
-%   [UDP_string] = DATA.PARSEALYXINSTANCE(AlyxInstance, ref)
+function [ref, AlyxInstance] = parseAlyxInstance(varargin)
+%DAT.PARSEALYXINSTANCE Converts input to string for UDP message and back
+%   [UDP_string] = DATA.PARSEALYXINSTANCE(ref, AlyxInstance)
+%   [ref, AlyxInstance] = DATA.PARSEALYXINSTANCE(UDP_string)
 %
 %   The pattern for 'ref' should be '{date}_{seq#}_{subject}', with two
 %   date formats accepted, either 'yyyy-mm-dd' or 'yyyymmdd'.
 %   
 %   AlyxInstance should be a struct with the following fields, all
-%   containing strings: 'baseURL', 'token', 'username'.
+%   containing strings: 'baseURL', 'token', 'username'[, 'subsessionURL'].
 %
 % Part of Rigbox
 
 % 2017-10 MW created
 
-if nargin > 1 % in [AlyxInstance, ref]
-  ai = varargin{1}; % extract AlyxInstance struct
-  ref = varargin(2); % extract expRef
+if nargin > 1 % in [ref, AlyxInstance]
+  ref = varargin{1}; % extract expRef
+  ai = varargin{2}; % extract AlyxInstance struct
   if isstruct(ai) % if there is an AlyxInstance
+    ai = orderfields(ai); % alphabetize fields
+    % remove water requirement remaining field
+    if isfield(ai, 'water_requirement_remaining')
+      ai = rmfield(ai, 'water_requirement_remaining');
+    end
+    fname = fieldnames(ai); % get fieldnames
+    emp = structfun(@isempty, ai); % find empty fields
+    if any(emp); ai = rmfield(ai, fname(emp)); end % remove the empty fields
     c = cellfun(@(fn) ai.(fn), fieldnames(ai), 'UniformOutput', false); % get fieldnames
-    varargout = strjoin([c; ref],'\'); % join into single string for UDP
-  else % otherwise just output the expRef
-    varargout = ref;
+    ref = strjoin([ref; c],'\'); % join into single string for UDP, otherwise just output the expRef
   end
 else % in [UDP_string]
   C = strsplit(varargin{1},'\'); % split string
-  varargout{1} = struct('baseURL', C{1}, 'token', C{2}, 'username', C{3}); % reconstruct AlyxInstance
-  varargout{2} = C{4}; % output expRef
+  ref = C{1}; % output expRef
+  if numel(C)>4 % if UDP string included AlyxInstance
+    AlyxInstance = struct('baseURL', C{2}, 'subsessionURL', C{3},...
+        'token', C{4}, 'username', C{5}); % reconstruct AlyxInstance
+  elseif numel(C)>1 % if AlyxInstance has no subsession set
+    AlyxInstance = struct('baseURL', C{2}, 'token', C{3}, 'username', C{4}); % reconstruct AlyxInstance
+  else
+    AlyxInstance = []; % if input was just an expRef, output empty AlyxInstance
+  end
 end

+dat/parseExpRef.m

@@ -1,5 +1,5 @@
 function [subjectRef, expDate, expSequence] = parseExpRef(ref)
-%DATA.PARSEEXPREF Extracts subject, date and seq from an experiment ref
+%DAT.PARSEEXPREF Extracts subject, date and seq from an experiment ref
 %   [subject, date, seq] = DATA.PARSEEXPREF(ref)
 %
 %   The pattern for 'ref' should be '{date}_{seq#}_{subject}', with two

+dat/paths.m

@@ -18,6 +18,8 @@
 % server3Name = '\\zserver3.cortexlab.net'; % 2017-02-18 MW - Currently
 % unused by Rigbox
 server4Name = '\\zserver4.cortexlab.net';
+basketName = '\\basket.cortexlab.net'; % for working analyses
+lugaroName = '\\lugaro.cortexlab.net'; % for tape backup
 
 %% defaults
 % path containing rigbox config folders
@@ -51,6 +53,18 @@
 % repository for all experiment definitions
 p.expDefinitions = fullfile(server1Name, 'Code', 'Rigging', 'ExpDefinitions');
 
+% repository for working analyses that are not meant to be stored
+% permanently
+p.workingAnalysisRepository = fullfile(basketName, 'data');
+
+% for tape backups, first files go here:
+p.tapeStagingRepository = fullfile(lugaroName, 'bigdrive', 'staging'); 
+
+% then they go here:
+p.tapeArchiveRepository = fullfile(lugaroName, 'bigdrive', 'toarchive');
+
+
+
 %% load rig-specific overrides from config file, if any  
 customPathsFile = fullfile(p.rigConfig, 'paths.mat');
 if file.exists(customPathsFile)

+dat/updateLogEntry.m

@@ -1,12 +1,23 @@
 function updateLogEntry(subject, id, newEntry)
 %DAT.UPDATELOGENTRY Updates an existing experiment log entry
-%   DAT.UPDATELOGENTRY(subject, id, newEntry)
-%   TODO
+%   DAT.UPDATELOGENTRY(subject, id, newEntry) The server copy of the log is
+%   loaded and the relevant record overwritten.  If an AlyxInstance is set,
+%   any session comments are saved in the session narrative in Alyx.
+%
+%   See also DAT.ADDLOGENTRY
 %
 % Part of Rigbox
 
 % 2013-03 CB created
 
+if isfield(newEntry, 'AlyxInstance')&&~isempty(newEntry.comments)
+  data = struct('subject', dat.parseExpRef(newEntry.value.ref),...
+      'narrative', strrep(mat2DStrTo1D(newEntry.comments),newline,'\n'));
+  alyx.putData(newEntry.AlyxInstance,...
+      newEntry.AlyxInstance.subsessionURL, data);
+  newEntry = rmfield(newEntry, 'AlyxInstance');
+end
+
 %load existing log from central repos
 log = pick(load(dat.logPath(subject, 'master')), 'log');
 %find the entry with specified id
@@ -17,5 +28,4 @@ function updateLogEntry(subject, id, newEntry)
 log(idx) = newEntry;
 %store new log to all repos locations
 superSave(dat.logPath(subject), struct('log', log));
-
 end