Automatically generate documentation pages for acquisition API (#43)

* Build acquisition API reference using docfx

* Add dotnet and docfx setup to CI workflow

* Ruff format and check acquisition build script

* Add instructions to build with docfx

* Reorganise API docs

* Fix missing ref

* Add "clean" actions to Windows make file

* Allow redirects to learn.microsoft.com/dotnet/api

* Temporarily ignore broken URL in acquisition API reference

* Fix redirected URLs

* Add `plotly` dependency

---------

Co-authored-by: lochhh <[email protected]>

.config/dotnet-tools.json

@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "docfx": {
+      "version": "2.77.0",
+      "commands": [
+        "docfx"
+      ],
+      "rollForward": false
+    }
+  }
+}

.github/workflows/docs_build_and_deploy.yml

@@ -31,6 +31,14 @@ jobs:
           git submodule update --init aeon_acquisition
           git submodule update --init aeon_analysis
           git submodule update --init aeon_mecha
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.x
+
+      - name: Setup DocFX
+        run: dotnet tool restore
 
       - name: Setup Python
         uses: actions/setup-python@v5

.gitignore

@@ -1,5 +1,7 @@
 # Sphinx documentation
 docs/
 src/reference/api/
-src/reference/api.rst
-src/user/**/*_copy*
+src/user/**/*_copy*
+
+# Python scripts
+__pycache__

Makefile

@@ -14,7 +14,8 @@ help:
 
 $(SOURCEDIR)/reference/api.rst:
 	@echo "Generating API documentation..."
-	@python make_api_doctree.py
+	@python make_mecha_doctree.py
+	@python make_acquisition_doctree.py
 
 copy-examples:
 	@echo "Copying example notebooks to 'src'..."
@@ -23,8 +24,7 @@ copy-examples:
 clean:
 	@echo "Removing auto-generated files under 'docs' and 'src'..."
 	@rm -rf $(BUILDDIR)
-	@rm -f $(SOURCEDIR)/reference/api.rst
-	@rm -rf $(SOURCEDIR)/reference/api
+	@rm -rf $(SOURCEDIR)/reference/api/
 
 .PHONY: help Makefile copy-examples
 

make.bat

@@ -14,11 +14,16 @@ if "%1" == "" goto help
 
 if not "%1" == "clean" (
 	@echo Building API docs...
-	python make_api_doctree.py
+	python make_mecha_doctree.py
+	python make_acquisition_doctree.py
 
 	@echo Copying example notebooks into 'src'...
 	python copy_examples_to_src.py
-) 
+) else (
+	@echo Removing auto-generated files under 'docs' and 'src'...
+	rmdir /S /Q %BUILDDIR%
+    rmdir /S /Q %SOURCEDIR%\reference\api\
+)
 
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (

make_acquisition_doctree.py

@@ -0,0 +1,63 @@
+import yaml
+import subprocess
+from pathlib import Path
+
+
+def make_acquisition_doctree():
+    """
+    Create a doctree of all namespaces in aeon_acquisition.
+    """
+    src_path = Path("src")
+    subprocess.run(["dotnet", "docfx", "metadata", "--outputFormat", "markdown"], cwd=src_path)
+    metadata_path = src_path.joinpath("reference", "api", "acquisition")
+
+    # get the acquisition doc header
+    with open(metadata_path.joinpath("toc.yml"), "r") as f:
+        acquisition_toc = yaml.safe_load(f)
+
+    # write file for acquisition doc with header + doctree
+    with open(metadata_path.with_suffix(".rst"), "w") as f:
+        f.write("..\n  This file is auto-generated.\n\n")
+        f.write(".. _target-acquisition-reference:\n\n")
+        f.write("``aeon_acquisition``\n")
+        f.write("=====================\n\n")
+        f.write(".. toctree::\n")
+        f.write("    :glob:\n\n")
+
+        # all namespaces are children of the root element
+        for namespace in acquisition_toc:
+            namespace_name = namespace["name"]
+            namespace_href = namespace.get("href")
+            namespace_items = namespace.get("items")
+            if namespace_href is None or namespace_items is None:
+                continue
+
+            # add namespace toctree entry
+            f.write(f"    {namespace_name} <acquisition/{namespace_name}>\n")
+
+            # generate toctree for each namespace document
+            namespace_path = metadata_path.joinpath(namespace_href)
+            with open(namespace_path, "r+", encoding="utf-8") as nsf:
+                content = nsf.read()
+                content += "```{toctree}\n"
+
+                # loop through all namespace items
+                for item in namespace_items:
+                    item_name = item["name"]
+                    item_href = item.get("href")
+                    if item_href is None:  # item is category, skip for now
+                        continue
+
+                    # item is type, add toctree entry
+                    item_path = metadata_path.joinpath(item_href)
+                    item_filename = item_path.with_suffix("").name
+                    content += f"\n{item_name} <{item_filename}>"
+
+                content += "\n```"
+                nsf.seek(0)
+                nsf.write(content)
+                nsf.truncate()
+
+
+if __name__ == "__main__":
+    make_acquisition_doctree()

make_api_doctree.py → make_mecha_doctree.py

@@ -7,7 +7,7 @@
 ]
 
 
-def make_api_doctree():
+def make_mecha_doctree():
     """
     Create a doctree of all modules in aeon_mecha/aeon.
     """
@@ -31,15 +31,18 @@ def make_api_doctree():
                     doctree += f"    {full}\n"
 
     # get the api doc header
-    with open("src/_templates/api_head.rst", "r") as f:
+    with open("src/_templates/api_mecha_head.rst", "r") as f:
         api_head = f.read()
 
     # write file for api doc with header + doctree
-    with open("./src/reference/api.rst", "w") as f:
+    output_dir = "./src/reference/api"
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(os.path.join(output_dir, "mecha.rst"), "w") as f:
         f.write("..\n  This file is auto-generated.\n\n")
         f.write(api_head)
         f.write(doctree)
 
 
 if __name__ == "__main__":
-    make_api_doctree()
+    make_mecha_doctree()

readme.md

@@ -6,6 +6,16 @@ The docs are built via [Sphinx](https://www.sphinx-doc.org/en/master/), and host
 
 
 ## Building the documentation locally
+Create a `conda` environment with the required dependencies and activate it:
+```bash
+conda create -n aeon_docs python dotnet -c conda-forge
+conda activate aeon_docs
+```
+
+Make the `docfx` tool available in the environment:
+```bash
+dotnet tool restore
+```
 
 From the root of the repository, install the requirements for building the documentation:
 ```bash

requirements.txt

@@ -3,6 +3,7 @@ dotmap
 matplotlib
 opencv-python
 pandas 
+plotly
 
 linkify-it-py
 myst-nb

src/_templates/api_mecha_head.rst

@@ -0,0 +1,12 @@
+.. _target-mecha-reference:
+
+``aeon_mecha``
+==============
+
+.. toctree::
+    :caption: aeon_mecha API
+    :maxdepth: 1
+
+.. autosummary::
+    :toctree: mecha
+

src/about/index.md

@@ -1,7 +1,7 @@
 (target-about)=
 # About
 
-The development of Aeon is funded by the [Gatsby Charitable Foundation](http://www.gatsby.org.uk/) and [Wellcome](https://wellcome.ac.uk/). This joint initiative involves people from the [Sainsbury Wellcome Centre](https://www.sainsburywellcome.org/web/) and the [Gatsby Computational Neuroscience Unit](https://www.ucl.ac.uk/gatsby/gatsby-computational-neuroscience-unit) at UCL, [NeuroGEARS Ltd](https://neurogears.org/), and [DataJoint Inc](https://www.datajoint.com/). For a full list of people involved, please refer to this [page](target-people).
+The development of Aeon is funded by the [Gatsby Charitable Foundation](https://www.gatsby.org.uk/) and [Wellcome](https://wellcome.org/). This joint initiative involves people from the [Sainsbury Wellcome Centre](https://www.sainsburywellcome.org/web/) and the [Gatsby Computational Neuroscience Unit](https://www.ucl.ac.uk/gatsby/gatsby-computational-neuroscience-unit) at UCL, [NeuroGEARS Ltd](https://neurogears.org/), and [DataJoint Inc](https://www.datajoint.com/). For a full list of people involved, please refer to this [page](target-people).
 
 The brain's computations are shaped by evolutionary pressures related to survival and the resulting behavioural responses. 
 Understanding the neural basis of natural behaviours is crucial for advancing our knowledge of the brain, as it sheds light on the functions of neural circuits in processing real-world stimuli. 

src/conf.py

@@ -148,13 +148,15 @@ def get_current_release_tag():
 # linkcheck will skip checking these URLs entirely
 linkcheck_ignore = [
     r"https://wiki\.ucl\.ac\.uk/.*",  # This is the UCL internal wiki
+    r"http://SubjectExpressionBuilder.Name",  # Broken URL
 ]
 
 # linkcheck will treat redirections from these source URI:canonical URI
 # mappings as "working".
 linkcheck_allowed_redirects = {
     r"https://doi\.org/10\.5281/zenodo\..*": r"https://zenodo\.org/records/.*",
     r"https://zenodo\.org/doi/.*": r"https://zenodo\.org/records/.*",
+    r"https://learn.microsoft.com/dotnet/api/.*": r"https://learn.microsoft.com/en-us/dotnet/api/.*",
 }
 
 myst_url_schemes = {

src/docfx.json

@@ -0,0 +1,41 @@
+{
+  "metadata": [
+    {
+      "src": [
+        {
+          "src": "../aeon_acquisition/src",
+          "files": ["**/*.csproj"],
+          "exclude": [ "**/*.Tests/**" ]
+        }
+      ],
+      "dest": "reference/api/acquisition"
+    }
+  ],
+  "build": {
+    "content": [
+      {
+        "files": [
+          "**/*.{md,yml}"
+        ],
+        "exclude": [
+          "_site/**"
+        ]
+      }
+    ],
+    "resource": [
+      {
+        "files": [
+          "images/**"
+        ]
+      }
+    ],
+
+    "xref": [
+      "https://horizongir.github.io/opencv.net/xrefmap.yml",
+      "https://horizongir.github.io/ZedGraph/xrefmap.yml",
+      "https://horizongir.github.io/opentk/xrefmap.yml",
+      "https://horizongir.github.io/reactive/xrefmap.yml",
+      "https://bonsai-rx.org/sleap/xrefmap.yml"
+    ]
+  }
+}

src/getting_started/installation.md

@@ -108,4 +108,4 @@ The next step is to configure your IDE to connect to the `swc-gateway` node via
 ::::
 :::::
 
-Within the `aeon` conda environment, Aeon data can be accessed directly from Ceph or the DataJoint database using the [Aeon API](target-api-reference). Examples for retrieving and visualizing the data can be found in the [User Guide](target-user-guide).
+Within the `aeon` conda environment, Aeon data can be accessed directly from Ceph or the DataJoint database using the [`aeon_mecha` API](target-mecha-reference). Examples for retrieving and visualizing the data can be found in the [User Guide](target-user-guide).

src/reference/api.md

@@ -0,0 +1,9 @@
+(target-api-reference)=
+# API Reference
+
+:::{toctree}
+:maxdepth: 2
+:hidden:
+
+api/mecha
+api/acquisition

src/reference/index.md

@@ -1,9 +1,11 @@
+(target-reference)=
 # Reference
 
 This reference manual details all software and hardware components, and a glossary of terms used in Project Aeon. It describes what these components are and how they function. For a comprehensive guide on how to use Project Aeon, please refer to the [complete documentation](target-home).
 
 :::{toctree}
 :maxdepth: 2
+:hidden:
 
 hardware
 api