Initial restructuring. (#82)

* Adding files from NIU cookiecutter.

* Replace CI workflow.

Co-authored-by: sfmig <[email protected]>

* Move source code files to package directory.

* Remove separate linting job.

* Gitignore and precommit from NIU template.

* Gitignore and precommit from NIU template.

* Move the tests to the NIU structure and fix imports.

* remove spurious index increase

* Remove egg!

* Add recursive include of .sh and .md, fix configure linters.

* Name the linting job.

* Make linters happy.

* Fix frame_extraction test data path.

* Drop python 3.11 for now.

* Remove pypi publishing actions step.

* Apply suggestions from code review

Co-authored-by: Sam Cunliffe <[email protected]>
Signed-off-by: sfmig <[email protected]>

* Name all jobs using imperatives. Use consistent order for job attributes.

* remove user-specific gitignore bits

* remove some commented bits

* rename to dense_optical_flow

* rename to opencv_notebooks

* rename to pose_estimation

* rename to stereo_calibration

* delete files with old names

* Update pyproject.toml

Co-authored-by: sfmig <[email protected]>
Signed-off-by: Sam Cunliffe <[email protected]>

---------

Signed-off-by: sfmig <[email protected]>
Signed-off-by: Sam Cunliffe <[email protected]>
Co-authored-by: sfmig <[email protected]>
Co-authored-by: sfmig <[email protected]>

.github/workflows/test_and_deploy.yml

@@ -0,0 +1,48 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+  pull_request:
+
+jobs:
+  linting:
+    name: Run pre-commit linters
+    runs-on: ubuntu-latest
+    steps:
+      - uses: neuroinformatics-unit/actions/lint@v2
+
+  manifest:
+    name: Check Manifest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: neuroinformatics-unit/actions/check_manifest@v2
+
+  test:
+    name: Run tests on ${{ matrix.os }} py${{ matrix.python-version }}
+    needs: [linting, manifest]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # Run all supported Python versions on linux
+        os: [ubuntu-latest]
+        python-version: ["3.9", "3.10"]
+        # Include one macos run
+        include:
+          - os: macos-latest
+            python-version: "3.10"
+    steps:
+      - uses: neuroinformatics-unit/actions/test@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+  build_sdist_wheels:
+    name: Build source distribution
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref_type == 'tag'
+    steps:
+      - uses: neuroinformatics-unit/actions/build_sdist_wheels@v2

.gitignore

@@ -1,19 +1,94 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask instance folder
+instance/
+
+# Sphinx documentation
+docs/_build/
+
+# MkDocs documentation
+/site/
+
+# PyBuilder
+target/
+
+# Pycharm and VSCode
+.idea/
+venv/
+.vscode/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# OS
 .DS_Store
-**/__pycache__/**
 
+# Video and image files
 *.MOV
-*.slp
-*.layout
-*.json
 *.png
 *.mp4
 
-/data/
-/bboxes_labelling/annotations
-/stereo calibration/check_chessboard_cells_size/*
-/pose estimation/animal_size_4k/*
-/pose estimation/top_down_model_4k_230713/*
+# Metadata and pose-estimation files
+*.slp
+*.json
 
-# include mp4 files in tests/data dir
+# Test data to includ
+# TODO: move this to GIN? (or at least the larger files)
 !/tests/data/**/*.mp4
 !/tests/data/**/*.json

.pre-commit-config.yaml

@@ -1,38 +1,36 @@
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.280
-    hooks:
-      - id: ruff
-        args: ["--config", "pyproject.toml"]
-  - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.1
-    hooks:
-      - id: forbid-tabs
-  - repo: https://github.com/pappasam/toml-sort
-    rev: v0.23.1
-    hooks:
-      - id: toml-sort-fix
-        exclude: "crabs-exploration/pyproject.toml"
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.4.1
-    hooks:
-      - id: mypy
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.0-alpha.9-for-vscode
     hooks:
       - id: prettier
-        exclude: "crabs-exploration/.github/workflows/tests.yml"
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
-      - id: check-case-conflict
       - id: check-docstring-first
+      # - id: check-executables-have-shebangs TODO: fix later
       - id: check-merge-conflict
+      - id: check-toml
       - id: end-of-file-fixer
       - id: mixed-line-ending
         args: [--fix=lf]
       - id: trailing-whitespace
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.280
+    hooks:
+      - id: ruff
   - repo: https://github.com/psf/black
     rev: 23.7.0
     hooks:
       - id: black
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.3.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - types-setuptools
+  - repo: https://github.com/mgedmin/check-manifest
+    rev: "0.49"
+    hooks:
+      - id: check-manifest
+        args: [--no-build-isolation]
+        additional_dependencies: [setuptools-scm]

MANIFEST.in

@@ -0,0 +1,12 @@
+include LICENSE
+include README.md
+
+recursive-include crabs *.md
+recursive-include crabs *.sh
+
+exclude .pre-commit-config.yaml
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+recursive-exclude docs *
+recursive-exclude tests *

README.md

@@ -1,3 +1,3 @@
 # crabs-exploration
 
-exploring the crabs field data
+A toolkit for detecting and tracking crabs in the field.

crabs/__init__.py

@@ -0,0 +1,7 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("crabs")
+except PackageNotFoundError:
+    # package is not installed
+    pass

bboxes_labelling/additional_channels_extraction.py → crabs/bboxes_labelling/additional_channels_extraction.py

@@ -4,9 +4,8 @@
 
 import cv2
 import numpy as np
-from PIL import Image
-
 from bboxes_labelling.annotations_utils import read_json_file
+from PIL import Image
 
 
 def apply_grayscale_and_blur(
@@ -88,7 +87,9 @@ def compute_mean_and_max_abs_blurred_frame(cap, kernel_size, sigmax):
         mean_blurred_frame += blurred_frame
 
         # accumulate max absolute values
-        max_abs_blurred_frame = np.maximum(max_abs_blurred_frame, abs(blurred_frame))
+        max_abs_blurred_frame = np.maximum(
+            max_abs_blurred_frame, abs(blurred_frame)
+        )
 
         # update frame counter
         frame_counter += 1
@@ -124,7 +125,9 @@ def compute_background_subtracted_frame(
         normalised difference between the blurred frame f and
         the mean blurred frame
     """
-    return (((blurred_frame - mean_blurred_frame) / max_abs_blurred_frame) + 1) / 2
+    return (
+        ((blurred_frame - mean_blurred_frame) / max_abs_blurred_frame) + 1
+    ) / 2
 
 
 def compute_motion_frame(
@@ -206,7 +209,9 @@ def compute_stacked_inputs(args: argparse.Namespace) -> None:
         (
             mean_blurred_frame,
             max_abs_blurred_frame,
-        ) = compute_mean_and_max_abs_blurred_frame(cap, args.kernel_size, args.sigmax)
+        ) = compute_mean_and_max_abs_blurred_frame(
+            cap, args.kernel_size, args.sigmax
+        )
 
         # save the mean
         cv2.imwrite(f"{Path(vid_file).stem}_mean.jpg", mean_blurred_frame)

bboxes_labelling/annotations_utils.py → crabs/bboxes_labelling/annotations_utils.py

@@ -114,11 +114,15 @@ def combine_multiple_via_jsons(
             raise ValueError(msg)
 
         # assign directory path to the VIA combined dictionary
-        via_data_combined["_via_settings"]["core"]["default_filepath"] = via_default_dir
+        via_data_combined["_via_settings"]["core"][
+            "default_filepath"
+        ] = via_default_dir
 
     # If required: change _via_settings > project > name
     if via_project_name:
-        via_data_combined["_via_settings"]["project"]["name"] = via_project_name
+        via_data_combined["_via_settings"]["project"][
+            "name"
+        ] = via_project_name
 
     # Save the VIA combined data as a new JSON file
     # if no output directory is passed, use the parent directory