Merge pull request #195 from SainsburyWellcomeCentre/gl-issue-186-fil…

…tering-nonchunked-data

Ensure each chunk is compared against filter bounds

aeon/io/api.py

@@ -117,17 +117,16 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No
         return pd.concat(dataframes)
 
     if start is not None or end is not None:
-        chunkfilter = chunk_range(start, end)
-        files = list(filter(lambda item: item[0][1] in chunkfilter, files))
-    else:
-        chunkfilter = None
+        chunk_start = chunk(start) if start is not None else pd.Timestamp.min
+        chunk_end = chunk(end) if end is not None else pd.Timestamp.max
+        files = list(filter(lambda item: chunk_start <= chunk(item[0][1]) <= chunk_end, files))
 
     if len(files) == 0:
         return _empty(reader.columns)
 
     data = pd.concat([reader.read(file) for _, file in files])
     _set_index(data)
-    if chunkfilter is not None:
+    if start is not None or end is not None:
         try:
             return data.loc[start:end]
         except KeyError:
@@ -139,5 +138,4 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No
                 warnings.warn('data index for {0} contains duplicate keys!'.format(reader.pattern))
                 data = data[~data.index.duplicated(keep='first')]
             return data.loc[start:end]
-            return data
     return data

pyproject.toml

@@ -92,3 +92,8 @@ exclude = '''
 [tool.isort]
 profile = "black"
 color_output = false
+
+[tool.pytest.ini_options]
+markers = [
+    "api",
+]

tests/data/nonmonotonic/2022-06-06T09-24-28/Metadata.yml

@@ -0,0 +1,11 @@
+{
+  "Workflow": "Experiment0.2.bonsai",
+  "Commit": "249cdc654af63e6959e64f7ff2c21f219cc912ea",
+  "Devices": {
+    "VideoController": {
+      "PortName": "COM3",
+      "GlobalTriggerFrequency": "50",
+      "LocalTriggerFrequency": "125"
+    }
+  }
+}

tests/io/test_api.py

@@ -0,0 +1,42 @@
+import pytest
+from pytest import mark
+import aeon.io.api as aeon
+from aeon.schema.dataset import exp02
+import pandas as pd
+
+@mark.api
+def test_load_start_only():
+    data = aeon.load(
+        './tests/data/nonmonotonic',
+        exp02.Patch2.Encoder,
+        start=pd.Timestamp('2022-06-06T13:00:49'))
+    assert len(data) > 0
+
+@mark.api
+def test_load_end_only():
+    data = aeon.load(
+        './tests/data/nonmonotonic',
+        exp02.Patch2.Encoder,
+        end=pd.Timestamp('2022-06-06T13:00:49'))
+    assert len(data) > 0
+
+@mark.api
+def test_load_filter_nonchunked():
+    data = aeon.load(
+        './tests/data/nonmonotonic',
+        exp02.Metadata,
+        start=pd.Timestamp('2022-06-06T09:00:00'))
+    assert len(data) > 0
+
+@mark.api
+def test_load_monotonic():
+    data = aeon.load('./tests/data/monotonic', exp02.Patch2.Encoder)
+    assert data.index.is_monotonic_increasing
+
+@mark.api
+def test_load_nonmonotonic():
+    data = aeon.load('./tests/data/nonmonotonic', exp02.Patch2.Encoder)
+    assert not data.index.is_monotonic_increasing
+
+if __name__ == '__main__':
+    pytest.main()