diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 69168b8a..00000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Linting - -on: - push: - branches: - - main - pull_request: - -jobs: - linting: - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v3 - - - name: Cache pre-commit - uses: actions/cache@v3 - with: - path: ~/.cache/pre-commit - key: pre-commit-${{ '{{' }} hashFiles('.pre-commit-config.yaml') {{ '}}' }} - - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - # cache: "pip" - # cache-dependency-path: "pyproject.toml" - - - name: Install dependencies - run: |- - python -m pip install pre-commit - pre-commit install - - - name: Run pre-commit - run: pre-commit run --all-files --color always diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml new file mode 100644 index 00000000..d5b6445e --- /dev/null +++ b/.github/workflows/test_and_deploy.yml @@ -0,0 +1,48 @@ +name: tests + +on: + push: + branches: + - "*" + tags: + - "*" + pull_request: + +jobs: + linting: + name: Run pre-commit linters + runs-on: ubuntu-latest + steps: + - uses: neuroinformatics-unit/actions/lint@v2 + + manifest: + name: Check Manifest + runs-on: ubuntu-latest + steps: + - uses: neuroinformatics-unit/actions/check_manifest@v2 + + test: + name: Run tests on ${{ matrix.os }} py${{ matrix.python-version }} + needs: [linting, manifest] + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Run all supported Python versions on linux + os: [ubuntu-latest] + python-version: ["3.9", "3.10"] + # Include one macos run + include: + - os: macos-latest + python-version: "3.10" + steps: + - uses: neuroinformatics-unit/actions/test@v2 + with: + python-version: ${{ matrix.python-version }} + + build_sdist_wheels: + name: Build source distribution + needs: [test] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref_type == 'tag' + steps: + - uses: neuroinformatics-unit/actions/build_sdist_wheels@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index e5141d13..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Tests - -on: - push: - branches: - - main - pull_request: - paths-ignore: - - "**.md" - -jobs: - tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9"] - os: [ubuntu-latest] - # # Include one windows and macos run - # include: - # - os: macos-latest - # python-version: "3.9" - # - os: windows-latest - # python-version: "3.9" - - steps: - - name: Checkout source - uses: actions/checkout@v3 - - - name: Cache tox - uses: actions/cache@v3 - with: - path: .tox - key: tox-${{ '{{' }}hashFiles('pyproject.toml') {{ '}}' }} - - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: "3.9" # ${{ '{{' }} matrix.python-version {{ '}}' }} - cache: "pip" - cache-dependency-path: "pyproject.toml" - - - name: Install dependencies - run: python -m pip install tox tox-gh-actions - - - name: Test with tox - run: tox diff --git a/.gitignore b/.gitignore index 456f467b..0db92f31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,94 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Sphinx documentation +docs/_build/ + +# MkDocs documentation +/site/ + +# PyBuilder +target/ + +# Pycharm and VSCode +.idea/ +venv/ +.vscode/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# OS .DS_Store -**/__pycache__/** +# Video and image files *.MOV -*.slp -*.layout -*.json *.png *.mp4 -/data/ -/bboxes_labelling/annotations -/stereo calibration/check_chessboard_cells_size/* -/pose estimation/animal_size_4k/* -/pose estimation/top_down_model_4k_230713/* +# Metadata and pose-estimation files +*.slp +*.json -# include mp4 files in tests/data dir +# Test data to includ +# TODO: move this to GIN? (or at least the larger files) !/tests/data/**/*.mp4 !/tests/data/**/*.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a494c190..77f2dda6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,38 +1,36 @@ repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.280 - hooks: - - id: ruff - args: ["--config", "pyproject.toml"] - - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.1 - hooks: - - id: forbid-tabs - - repo: https://github.com/pappasam/toml-sort - rev: v0.23.1 - hooks: - - id: toml-sort-fix - exclude: "crabs-exploration/pyproject.toml" - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 - hooks: - - id: mypy - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.0-alpha.9-for-vscode hooks: - id: prettier - exclude: "crabs-exploration/.github/workflows/tests.yml" - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - - id: check-case-conflict - id: check-docstring-first + # - id: check-executables-have-shebangs TODO: fix later - id: check-merge-conflict + - id: check-toml - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] - id: trailing-whitespace + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.280 + hooks: + - id: ruff - repo: https://github.com/psf/black rev: 23.7.0 hooks: - id: black + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.3.0 + hooks: + - id: mypy + additional_dependencies: + - types-setuptools + - repo: https://github.com/mgedmin/check-manifest + rev: "0.49" + hooks: + - id: check-manifest + args: [--no-build-isolation] + additional_dependencies: [setuptools-scm] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..6bde708a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,12 @@ +include LICENSE +include README.md + +recursive-include crabs *.md +recursive-include crabs *.sh + +exclude .pre-commit-config.yaml + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude docs * +recursive-exclude tests * diff --git a/README.md b/README.md index 421d31ce..4a07e59c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # crabs-exploration -exploring the crabs field data +A toolkit for detecting and tracking crabs in the field. diff --git a/crabs/__init__.py b/crabs/__init__.py new file mode 100644 index 00000000..ece08197 --- /dev/null +++ b/crabs/__init__.py @@ -0,0 +1,7 @@ +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("crabs") +except PackageNotFoundError: + # package is not installed + pass diff --git a/bboxes_labelling/__init__.py b/crabs/bboxes_labelling/__init__.py similarity index 100% rename from bboxes_labelling/__init__.py rename to crabs/bboxes_labelling/__init__.py diff --git a/bboxes_labelling/additional_channels_extraction.py b/crabs/bboxes_labelling/additional_channels_extraction.py similarity index 96% rename from bboxes_labelling/additional_channels_extraction.py rename to crabs/bboxes_labelling/additional_channels_extraction.py index 9fbe6b67..58487dcb 100644 --- a/bboxes_labelling/additional_channels_extraction.py +++ b/crabs/bboxes_labelling/additional_channels_extraction.py @@ -4,9 +4,8 @@ import cv2 import numpy as np -from PIL import Image - from bboxes_labelling.annotations_utils import read_json_file +from PIL import Image def apply_grayscale_and_blur( @@ -88,7 +87,9 @@ def compute_mean_and_max_abs_blurred_frame(cap, kernel_size, sigmax): mean_blurred_frame += blurred_frame # accumulate max absolute values - max_abs_blurred_frame = np.maximum(max_abs_blurred_frame, abs(blurred_frame)) + max_abs_blurred_frame = np.maximum( + max_abs_blurred_frame, abs(blurred_frame) + ) # update frame counter frame_counter += 1 @@ -124,7 +125,9 @@ def compute_background_subtracted_frame( normalised difference between the blurred frame f and the mean blurred frame """ - return (((blurred_frame - mean_blurred_frame) / max_abs_blurred_frame) + 1) / 2 + return ( + ((blurred_frame - mean_blurred_frame) / max_abs_blurred_frame) + 1 + ) / 2 def compute_motion_frame( @@ -206,7 +209,9 @@ def compute_stacked_inputs(args: argparse.Namespace) -> None: ( mean_blurred_frame, max_abs_blurred_frame, - ) = compute_mean_and_max_abs_blurred_frame(cap, args.kernel_size, args.sigmax) + ) = compute_mean_and_max_abs_blurred_frame( + cap, args.kernel_size, args.sigmax + ) # save the mean cv2.imwrite(f"{Path(vid_file).stem}_mean.jpg", mean_blurred_frame) diff --git a/bboxes_labelling/annotations_utils.py b/crabs/bboxes_labelling/annotations_utils.py similarity index 97% rename from bboxes_labelling/annotations_utils.py rename to crabs/bboxes_labelling/annotations_utils.py index dcd10ac5..958b45d1 100644 --- a/bboxes_labelling/annotations_utils.py +++ b/crabs/bboxes_labelling/annotations_utils.py @@ -114,11 +114,15 @@ def combine_multiple_via_jsons( raise ValueError(msg) # assign directory path to the VIA combined dictionary - via_data_combined["_via_settings"]["core"]["default_filepath"] = via_default_dir + via_data_combined["_via_settings"]["core"][ + "default_filepath" + ] = via_default_dir # If required: change _via_settings > project > name if via_project_name: - via_data_combined["_via_settings"]["project"]["name"] = via_project_name + via_data_combined["_via_settings"]["project"][ + "name" + ] = via_project_name # Save the VIA combined data as a new JSON file # if no output directory is passed, use the parent directory diff --git a/bboxes_labelling/clip_video.py b/crabs/bboxes_labelling/clip_video.py similarity index 85% rename from bboxes_labelling/clip_video.py rename to crabs/bboxes_labelling/clip_video.py index cb03d7d8..aa08a0ac 100644 --- a/bboxes_labelling/clip_video.py +++ b/crabs/bboxes_labelling/clip_video.py @@ -1,7 +1,8 @@ import argparse -import cv2 -from pathlib import Path from datetime import datetime +from pathlib import Path + +import cv2 def real_time_to_frame_number( @@ -44,7 +45,11 @@ def create_clip( fourcc = cv2.VideoWriter_fourcc(*"avc1") out = cv2.VideoWriter( - output_file, fourcc, video_fps, (int(cap.get(3)), int(cap.get(4))), isColor=True + output_file, + fourcc, + video_fps, + (int(cap.get(3)), int(cap.get(4))), + isColor=True, ) cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) @@ -111,7 +116,10 @@ def argument_parser() -> argparse.Namespace: args = argument_parser() input_file = args.video_path - file_name = f"{Path(args.video_path).parent.stem}_" f"{Path(args.video_path).stem}_" + file_name = ( + f"{Path(args.video_path).parent.stem}_" + f"{Path(args.video_path).stem}_" + ) start_real_time = datetime.strptime(args.start_time, "%H:%M:%S") event_time = datetime.strptime(args.event_time, "%H:%M:%S") @@ -120,8 +128,12 @@ def argument_parser() -> argparse.Namespace: # Convert event times to frame numbers cap = cv2.VideoCapture(args.video_path) video_fps = cap.get(cv2.CAP_PROP_FPS) - start_frame = real_time_to_frame_number(start_real_time, video_fps, start_real_time) - event_frame = real_time_to_frame_number(event_time, video_fps, start_real_time) + start_frame = real_time_to_frame_number( + start_real_time, video_fps, start_real_time + ) + event_frame = real_time_to_frame_number( + event_time, video_fps, start_real_time + ) after_event_frame = real_time_to_frame_number( after_event_time, video_fps, start_real_time ) @@ -134,10 +146,14 @@ def argument_parser() -> argparse.Namespace: # Create event clip event_clip = f"{args.out_path}/{file_name}_event.mp4" - create_clip(args.video_path, event_frame, after_event_frame - 1, event_clip) + create_clip( + args.video_path, event_frame, after_event_frame - 1, event_clip + ) # Create post-event clip post_event_clip = f"{args.out_path}/{file_name}_post_event.mp4" - create_clip(args.video_path, after_event_frame, total_frames - 1, post_event_clip) + create_clip( + args.video_path, after_event_frame, total_frames - 1, post_event_clip + ) print("Clips created successfully!") diff --git a/bboxes_labelling/cluster_bash_scripts/run_additional_channel.sh b/crabs/bboxes_labelling/cluster_bash_scripts/run_additional_channel.sh similarity index 100% rename from bboxes_labelling/cluster_bash_scripts/run_additional_channel.sh rename to crabs/bboxes_labelling/cluster_bash_scripts/run_additional_channel.sh diff --git a/bboxes_labelling/cluster_bash_scripts/run_clip_video.sh b/crabs/bboxes_labelling/cluster_bash_scripts/run_clip_video.sh similarity index 100% rename from bboxes_labelling/cluster_bash_scripts/run_clip_video.sh rename to crabs/bboxes_labelling/cluster_bash_scripts/run_clip_video.sh diff --git a/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_array.sh b/crabs/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_array.sh similarity index 100% rename from bboxes_labelling/cluster_bash_scripts/run_frame_extraction_array.sh rename to crabs/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_array.sh diff --git a/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_local.sh b/crabs/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_local.sh similarity index 100% rename from bboxes_labelling/cluster_bash_scripts/run_frame_extraction_local.sh rename to crabs/bboxes_labelling/cluster_bash_scripts/run_frame_extraction_local.sh diff --git a/bboxes_labelling/combine_and_format_annotations.py b/crabs/bboxes_labelling/combine_and_format_annotations.py similarity index 90% rename from bboxes_labelling/combine_and_format_annotations.py rename to crabs/bboxes_labelling/combine_and_format_annotations.py index 29563d3f..47133601 100644 --- a/bboxes_labelling/combine_and_format_annotations.py +++ b/crabs/bboxes_labelling/combine_and_format_annotations.py @@ -1,6 +1,9 @@ from pathlib import Path -from annotations_utils import combine_multiple_via_jsons, convert_via_json_to_coco +from annotations_utils import ( + combine_multiple_via_jsons, + convert_via_json_to_coco, +) def main( diff --git a/bboxes_labelling/extract_frames_to_label_w_sleap.py b/crabs/bboxes_labelling/extract_frames_to_label_w_sleap.py similarity index 98% rename from bboxes_labelling/extract_frames_to_label_w_sleap.py rename to crabs/bboxes_labelling/extract_frames_to_label_w_sleap.py index 8951e4b2..fa0a32ed 100755 --- a/bboxes_labelling/extract_frames_to_label_w_sleap.py +++ b/crabs/bboxes_labelling/extract_frames_to_label_w_sleap.py @@ -101,7 +101,8 @@ def get_list_of_sleap_videos( cap.release() else: logging.warning( - f"Video at {vid_path!s} could not" " be opened by OpenCV. Skipping...", + f"Video at {vid_path!s} could not" + " be opened by OpenCV. Skipping...", ) # Print warning if list is empty @@ -302,7 +303,9 @@ def extract_frames_to_label_from_video( if flag_parent_dir_subdir_in_output: video_output_dir = ( output_subdir_path - / Path(vid_str).parent.stem # timestamp # parent dir of input video + / Path( + vid_str + ).parent.stem # timestamp # parent dir of input video ) video_output_dir.mkdir(parents=True, exist_ok=True) else: @@ -422,7 +425,9 @@ def compute_and_extract_frames_to_label(args): sort_keys=True, indent=4, ) - logging.info("New json file with extracted frames saved at {json_output_file}") + logging.info( + "New json file with extracted frames saved at {json_output_file}" + ) # Save suggested frames as png files # (extraction with opencv) diff --git a/bboxes_labelling/guides/ExtractFramesCluster.md b/crabs/bboxes_labelling/guides/ExtractFramesCluster.md similarity index 100% rename from bboxes_labelling/guides/ExtractFramesCluster.md rename to crabs/bboxes_labelling/guides/ExtractFramesCluster.md diff --git a/bboxes_labelling/guides/ManualLabellingSteps_dev.md b/crabs/bboxes_labelling/guides/ManualLabellingSteps_dev.md similarity index 100% rename from bboxes_labelling/guides/ManualLabellingSteps_dev.md rename to crabs/bboxes_labelling/guides/ManualLabellingSteps_dev.md diff --git a/bboxes_labelling/guides/ManualLabellingSteps_general.md b/crabs/bboxes_labelling/guides/ManualLabellingSteps_general.md similarity index 100% rename from bboxes_labelling/guides/ManualLabellingSteps_general.md rename to crabs/bboxes_labelling/guides/ManualLabellingSteps_general.md diff --git a/dense optical flow/estimate_optical_flow_on_video.py b/crabs/dense_optical _flow/estimate_optical_flow_on_video.py similarity index 93% rename from dense optical flow/estimate_optical_flow_on_video.py rename to crabs/dense_optical _flow/estimate_optical_flow_on_video.py index 4bdd60a7..602020b0 100644 --- a/dense optical flow/estimate_optical_flow_on_video.py +++ b/crabs/dense_optical _flow/estimate_optical_flow_on_video.py @@ -92,7 +92,9 @@ def run_model_on_video(args): output_dir = Path(args.output_dir) # create output dir if it doesnt exist output_dir.mkdir(parents=True, exist_ok=True) - videowriter_path = output_dir / Path(Path(input_file).stem + "_flow.mp4") + videowriter_path = output_dir / Path( + Path(input_file).stem + "_flow.mp4" + ) # initialise videowriter videowriter = cv2.VideoWriter( @@ -110,7 +112,9 @@ def run_model_on_video(args): # ensure we start capture at 0 if cap.get(cv2.CAP_PROP_POS_FRAMES) == 0: frame_idx = 0 - frame_idx_stop = int(nframes / args.step_frames) * args.step_frames + frame_idx_stop = ( + int(nframes / args.step_frames) * args.step_frames + ) while frame_idx < frame_idx_stop: # set 'index to read next' to the desired position @@ -124,7 +128,9 @@ def run_model_on_video(args): frame_idx = cap.get(cv2.CAP_PROP_POS_FRAMES) # read frame at f+n - cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx + args.step_frames) + cap.set( + cv2.CAP_PROP_POS_FRAMES, frame_idx + args.step_frames + ) success_frame_2, frame_2 = cap.read() # OJO index to read next is now at f+n+1, but we # will reset it to frame_idx at the start of @@ -150,7 +156,9 @@ def run_model_on_video(args): # compute flow # output has batch channel on the first dim # TODO: what is flow_low? (downsampled?) - flow_low, flow_up = model(image1, image2, iters=20, test_mode=True) + flow_low, flow_up = model( + image1, image2, iters=20, test_mode=True + ) # convert output to numpy array and reorder channels # first dim = batch size (1) diff --git a/dense optical flow/notebook_RAFT.py b/crabs/dense_optical _flow/notebook_RAFT.py similarity index 100% rename from dense optical flow/notebook_RAFT.py rename to crabs/dense_optical _flow/notebook_RAFT.py diff --git a/dense optical flow/run_RAFT_on_videos.sh b/crabs/dense_optical _flow/run_RAFT_on_videos.sh similarity index 100% rename from dense optical flow/run_RAFT_on_videos.sh rename to crabs/dense_optical _flow/run_RAFT_on_videos.sh diff --git a/opencv notebooks/notebook_clip_video.py b/crabs/opencv_notebooks/notebook_clip_video.py similarity index 94% rename from opencv notebooks/notebook_clip_video.py rename to crabs/opencv_notebooks/notebook_clip_video.py index f4d975e6..7cbc061d 100644 --- a/opencv notebooks/notebook_clip_video.py +++ b/crabs/opencv_notebooks/notebook_clip_video.py @@ -13,9 +13,7 @@ clip_suffix = "p1_05s" -output_dir = ( - "/Users/sofia/Documents_local/project_Zoo_crabs/crabs-exploration/tests/data" -) +output_dir = "/Users/sofia/Documents_local/project_Zoo_crabs/crabs-exploration/tests/data" # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # Get video params diff --git a/opencv notebooks/notebook_downsample.py b/crabs/opencv_notebooks/notebook_downsample.py similarity index 93% rename from opencv notebooks/notebook_downsample.py rename to crabs/opencv_notebooks/notebook_downsample.py index 3e9a7bff..abfb3e1e 100644 --- a/opencv notebooks/notebook_downsample.py +++ b/crabs/opencv_notebooks/notebook_downsample.py @@ -32,7 +32,9 @@ list_downsampled_imgs = [] for f in list_downsampled_factors: size_down = tuple(int(s * f) for s in size_og) - img_downsampled = cv2.resize(img_og, size_down, interpolation=cv2.INTER_NEAREST) + img_downsampled = cv2.resize( + img_og, size_down, interpolation=cv2.INTER_NEAREST + ) window_id = f"scale {f}" cv2.namedWindow(window_id, cv2.WINDOW_NORMAL) diff --git a/opencv notebooks/notebook_set_video_at_frame.py b/crabs/opencv_notebooks/notebook_set_video_at_frame.py similarity index 100% rename from opencv notebooks/notebook_set_video_at_frame.py rename to crabs/opencv_notebooks/notebook_set_video_at_frame.py diff --git a/opencv notebooks/notebook_video_params.py b/crabs/opencv_notebooks/notebook_video_params.py similarity index 97% rename from opencv notebooks/notebook_video_params.py rename to crabs/opencv_notebooks/notebook_video_params.py index 6d64f632..11a0874f 100644 --- a/opencv notebooks/notebook_video_params.py +++ b/crabs/opencv_notebooks/notebook_video_params.py @@ -25,7 +25,11 @@ list_paths = [] for typ in file_types: list_paths.extend( - [p for p in list(videos_parent_dir.glob(typ)) if not p.name.startswith("._")], + [ + p + for p in list(videos_parent_dir.glob(typ)) + if not p.name.startswith("._") + ], ) diff --git a/pose estimation/cluster bash scripts/howto_inference_cluster.md b/crabs/pose_estimation/cluster bash scripts/howto_inference_cluster.md similarity index 100% rename from pose estimation/cluster bash scripts/howto_inference_cluster.md rename to crabs/pose_estimation/cluster bash scripts/howto_inference_cluster.md diff --git a/pose estimation/cluster bash scripts/howto_training_cluster.md b/crabs/pose_estimation/cluster bash scripts/howto_training_cluster.md similarity index 100% rename from pose estimation/cluster bash scripts/howto_training_cluster.md rename to crabs/pose_estimation/cluster bash scripts/howto_training_cluster.md diff --git a/pose estimation/cluster bash scripts/slurm_inference.sh b/crabs/pose_estimation/cluster bash scripts/slurm_inference.sh similarity index 100% rename from pose estimation/cluster bash scripts/slurm_inference.sh rename to crabs/pose_estimation/cluster bash scripts/slurm_inference.sh diff --git a/pose estimation/cluster bash scripts/slurm_render.sh b/crabs/pose_estimation/cluster bash scripts/slurm_render.sh similarity index 100% rename from pose estimation/cluster bash scripts/slurm_render.sh rename to crabs/pose_estimation/cluster bash scripts/slurm_render.sh diff --git a/pose estimation/cluster bash scripts/slurm_train.sh b/crabs/pose_estimation/cluster bash scripts/slurm_train.sh similarity index 100% rename from pose estimation/cluster bash scripts/slurm_train.sh rename to crabs/pose_estimation/cluster bash scripts/slurm_train.sh diff --git a/stereo calibration/CalibrationDataCollection.md b/crabs/stereo_calibration/CalibrationDataCollection.md similarity index 100% rename from stereo calibration/CalibrationDataCollection.md rename to crabs/stereo_calibration/CalibrationDataCollection.md diff --git a/opencv notebooks/.DS_Store b/opencv notebooks/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/opencv notebooks/.DS_Store and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index af12544d..73e3410e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,126 +1,118 @@ -# [build-system] -# build-backend = "setuptools.build_meta" -# requires = [ -# "setuptools", -# "setuptools-scm", -# ] - [project] +name = "crabs" authors = [ {email = "n.aznan@ucl.ac.uk", name = "Nik Khadijah Nik Aznan"}, {email = "s.minano@ucl.ac.uk", name = "Sofia Minano"}, ] +description = "A toolkit for detecting and tracking crabs in the field." +readme = "README.md" +requires-python = ">=3.9.0" +dynamic = ["version"] + +license = {text = "BSD-3-Clause"} +urls.homepage = "https://github.com/SainsburyWellcomeCentre/crabs-exploration" + classifiers = [ - "Operating System :: POSIX", + "Development Status :: 2 - Pre-Alpha", + "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", - "Typing :: Typed", + "Programming Language :: Python :: 3.10", + "Operating System :: POSIX", + "License :: OSI Approved :: BSD License", ] dependencies = [ "opencv-python", "sleap[pypi]==1.3.3", ] -description = "exploring the crabs field data" -dynamic = [ - "version", + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "coverage", + "tox", # >=4 ? + "black", + "mypy", + "pre-commit", + "ruff", + "setuptools_scm", ] -keywords = [ + +[build-system] +requires = [ + "setuptools>=45", + "wheel", + "setuptools_scm[toml]>=6.2", ] -name = "crabs-exploration" -optional-dependencies = {dev = [ - "black", - "build", - "mypy", - "pre-commit", - "pytest", - "ruff", - "tox>=4", - "twine", -]} -readme = "README.md" -requires-python = ">=3.9.0" -license.file = "LICENCE.md" -urls.homepage = "https://github.com/sfmig/crabs-exploration" +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["crabs*"] +exclude = ["tests*"] [tool.pytest.ini_options] -addopts = "--color=yes -v" -testpaths = [ - "tests", +addopts = "--cov=crabs" + +[tool.black] +target-version = ['py39', 'py310'] +skip-string-normalization = false +line-length = 79 + +[tool.setuptools_scm] + +[tool.check-manifest] +ignore = [ + ".yaml", + "tox.ini", + "tests/", + "tests/test_unit/", + "tests/test_integration/", ] [tool.ruff] +line-length = 79 +exclude = ["__init__.py","build",".eggs"] +select = ["I", "E", "F"] fix = true -force-exclude = true ignore = [ - "D203", # no-blank-line-before-class - "D212", # multi-line-summary-first-line - "D417", # argument description in docstring (unreliable) + 'E501', # line too long: should be handled by black ] -per-file-ignores = {"test_*" = [ - "S101", -]} -# select = [ -# "A", -# "ARG", -# "B", -# "BLE", -# "C", -# "COM", -# "D", -# "DJ", -# "DTZ", -# "E", -# "EM", -# "ERA", -# "EXE", -# "F", -# "FBT", -# "I", -# "ICN", -# "ISC", -# "N", -# "NPY", -# "PD", -# "PGH", -# "PIE", -# "PL", -# "PT", -# "PTH", -# "PYI", -# "Q", -# "RET", -# "RSE", -# "RUF", -# "S", -# "SIM", -# "T", -# "TCH", -# "TID", -# "UP", -# "W", -# "YTT", +# force-exclude = true +# ignore = [ +# "D203", # no-blank-line-before-class +# "D212", # multi-line-summary-first-line +# "D417", # argument description in docstring (unreliable) # ] -target-version = "py39" +# per-file-ignores = {"test_*" = [ +# "S101", +# ]} +# [tool.tomlsort] +# all = true +# spaces_indent_inline_array = 4 +# trailing_comma_inline_array = true +# overrides."project.classifiers".inline_arrays = false +# overrides."tool.coverage.paths.source".inline_arrays = false + -[tool.tomlsort] -all = true -spaces_indent_inline_array = 4 -trailing_comma_inline_array = true -overrides."project.classifiers".inline_arrays = false -overrides."tool.coverage.paths.source".inline_arrays = false [tool.tox] legacy_tox_ini = """ - [gh-actions] - python = 3.9: py39 - - [testenv] - commands = - pytest --cov - deps = - pytest - pytest-cov - - [tox] - env_list = py39 +[tox] +envlist = py{39,310,311} +isolated_build = True + +[gh-actions] +python = + 3.9: py39 + 3.10: py310 + +[testenv] +extras = + dev +commands = + pytest -v --color=yes --cov=crabs --cov-report=xml """ diff --git a/tests/test_integration/__init__.py b/tests/test_integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_coco_and_via_conversions.py b/tests/test_integration/test_coco_and_via_conversions.py similarity index 90% rename from tests/test_coco_and_via_conversions.py rename to tests/test_integration/test_coco_and_via_conversions.py index 3cb1c6a3..bd851cb9 100644 --- a/tests/test_coco_and_via_conversions.py +++ b/tests/test_integration/test_coco_and_via_conversions.py @@ -4,7 +4,7 @@ import pytest -from bboxes_labelling.annotations_utils import ( +from crabs.bboxes_labelling.annotations_utils import ( combine_multiple_via_jsons, convert_via_json_to_coco, read_json_file, @@ -22,7 +22,9 @@ def via_json_1() -> str: """ # Return path to sample VIA (Visual Image Annotator) JSON file 1 return str( - Path(__file__).resolve().parent / "data" / "COCO_VIA_JSONS/VIA_JSON_1.json", + Path(__file__).resolve().parents[1] + / "data" + / "COCO_VIA_JSONS/VIA_JSON_1.json", ) @@ -37,7 +39,9 @@ def via_json_2() -> str: """ # Return path to sample VIA JSON file 2 return str( - Path(__file__).resolve().parent / "data" / "COCO_VIA_JSONS/VIA_JSON_2.json", + Path(__file__).resolve().parents[1] + / "data" + / "COCO_VIA_JSONS/VIA_JSON_2.json", ) @@ -81,10 +85,14 @@ def test_via_json_combine( # check values taken from the first file to combine: # VIA settings - assert via_json_combined_dict["_via_settings"] == via_json_1_dict["_via_settings"] + assert ( + via_json_combined_dict["_via_settings"] + == via_json_1_dict["_via_settings"] + ) # VIA attributes assert ( - via_json_combined_dict["_via_attributes"] == via_json_1_dict["_via_attributes"] + via_json_combined_dict["_via_attributes"] + == via_json_1_dict["_via_attributes"] ) # data format version assert ( @@ -100,7 +108,8 @@ def test_via_json_combine( ]: for ky in img_metadata_dict: assert ( - img_metadata_dict[ky] == via_json_combined_dict["_via_img_metadata"][ky] + img_metadata_dict[ky] + == via_json_combined_dict["_via_img_metadata"][ky] ) # Check the number of images in the combined JSON is the sum of the number @@ -115,7 +124,8 @@ def test_via_json_combine( via_json_2_dict["_via_image_id_list"], ]: assert all( - x in via_json_combined_dict["_via_image_id_list"] for x in img_id_list + x in via_json_combined_dict["_via_image_id_list"] + for x in img_id_list ) @@ -236,7 +246,8 @@ def test_via_json_combine_project_name( # check project name is as specified assert ( - via_json_combined_dict["_via_settings"]["project"]["name"] == via_project_name + via_json_combined_dict["_via_settings"]["project"]["name"] + == via_project_name ) @@ -282,7 +293,10 @@ def test_coco_generated_from_via_json( assert len(coco_json_dict["categories"]) == 1 assert coco_json_dict["categories"][0]["id"] == coco_category_ID assert coco_json_dict["categories"][0]["name"] == coco_category_name - assert coco_json_dict["categories"][0]["supercategory"] == coco_supercategory_name + assert ( + coco_json_dict["categories"][0]["supercategory"] + == coco_supercategory_name + ) # We assume keys in VIA's metadata list are sorted in increasing image ID # (ID starts from 1) @@ -296,12 +310,17 @@ def test_coco_generated_from_via_json( ky_in_via_img_metadata = list_keys_via_img_metadata[img_dict["id"] - 1] # Check key in VIA image metadata matches COCO filename - assert Path(img_dict["file_name"]).stem == Path(ky_in_via_img_metadata).stem + assert ( + Path(img_dict["file_name"]).stem + == Path(ky_in_via_img_metadata).stem + ) # Check VIA filename matches COCO filename assert ( img_dict["file_name"] - == via_json_dict["_via_img_metadata"][ky_in_via_img_metadata]["filename"] + == via_json_dict["_via_img_metadata"][ky_in_via_img_metadata][ + "filename" + ] ) # Compare annotations between VIA JSON and COCO file @@ -317,7 +336,9 @@ def test_coco_generated_from_via_json( # Get image info from VIA using the "image_id" from the COCO annotation ky_in_via_img_metadata = list_keys_via_img_metadata[img_id - 1] - img_dict_in_via = via_json_dict["_via_img_metadata"][ky_in_via_img_metadata] + img_dict_in_via = via_json_dict["_via_img_metadata"][ + ky_in_via_img_metadata + ] # Reset annotation index per image if image_id changes if img_id_previous_image != img_id: @@ -328,6 +349,7 @@ def test_coco_generated_from_via_json( reg = img_dict_in_via["regions"][ann_idx_per_img] w_from_via = reg["shape_attributes"]["width"] h_from_via = reg["shape_attributes"]["height"] + bbox_from_via = [ reg["shape_attributes"]["x"], reg["shape_attributes"]["y"], diff --git a/tests/test_frame_extraction.py b/tests/test_integration/test_frame_extraction.py similarity index 87% rename from tests/test_frame_extraction.py rename to tests/test_integration/test_frame_extraction.py index 8d5a6ac6..2cc21f87 100644 --- a/tests/test_frame_extraction.py +++ b/tests/test_integration/test_frame_extraction.py @@ -2,12 +2,14 @@ import pytest -from bboxes_labelling.extract_frames_to_label_w_sleap import get_list_of_sleap_videos +from crabs.bboxes_labelling.extract_frames_to_label_w_sleap import ( + get_list_of_sleap_videos, +) @pytest.fixture(autouse=True, scope="class") def input_video_dir(): - return Path(__file__).parent / "data" / "clips" + return Path(__file__).parents[1] / "data" / "clips" class TestsFrameExtraction: @@ -46,3 +48,4 @@ def test_extension_case_insensitive(self, input_video_dir): # check list of SLEAP videos matches the list of files assert len(list_sleap_videos) == len(list_files) + assert len(list_sleap_videos) == len(list_files) diff --git a/tests/test_unit/__init__.py b/tests/test_unit/__init__.py new file mode 100644 index 00000000..e69de29b