Dockerized action runners and script updates

.github/workflows/unit_tests.yml

@@ -1,8 +1,6 @@
 name: 'Unit tests'
-
 on:
   pull_request:
-
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
@@ -15,7 +13,6 @@ jobs:
     steps:
       - name: 'Wipe workspace'
         run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -26,7 +23,6 @@ jobs:
         id: device
         run: |
           echo "flipper=/dev/ttyACM0" >> $GITHUB_OUTPUT
-
       - name: 'Flash unit tests firmware'
         id: flashing
         if: success()
@@ -42,7 +38,6 @@ jobs:
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
           python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
-
       - name: 'Copy assets and unit data, reboot and wait for flipper'
         id: copy
         if: steps.format_ext.outcome == 'success'
@@ -54,15 +49,13 @@ jobs:
           python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send build/latest/resources /ext
           python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
           python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
       - name: 'Run units and validate results'
         id: run_units
         if: steps.copy.outcome == 'success'
         timeout-minutes: 7
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testing/units.py ${{steps.device.outputs.flipper}}
-
       - name: 'Check GDB output'
         if: failure()
         run: |

.github/workflows/updater_test.yml

@@ -1,8 +1,6 @@
 name: 'Updater test'
-
 on:
   pull_request:
-
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
@@ -15,7 +13,6 @@ jobs:
     steps:
       - name: 'Wipe workspace'
         run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -28,15 +25,13 @@ jobs:
         run: |
           echo "flipper=Rekigyn" >> $GITHUB_OUTPUT
           echo "stlink=0F020D026415303030303032" >> $GITHUB_OUTPUT
-
       - name: 'Flashing target firmware'
         id: first_full_flash
         timeout-minutes: 10
         run: |
           source scripts/toolchain/fbtenv.sh
           ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
           python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
       - name: 'Validating updater'
         id: second_full_flash
         timeout-minutes: 10
@@ -45,13 +40,11 @@ jobs:
           source scripts/toolchain/fbtenv.sh
           ./fbt flash_usb PORT=${{steps.device.outputs.flipper}} FORCE=1
           python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
       - name: 'Get last release tag'
         id: release_tag
         if: failure()
         run: |
           echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
-
       - name: 'Wipe workspace'
         run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
 
@@ -66,7 +59,6 @@ jobs:
         if: failure()
         run: |
           ./fbt flash SWD_TRANSPORT_SERIAL=${{steps.device.outputs.stlink}} FORCE=1
-
       - name: 'Wait for flipper and format ext'
         if: failure()
         run: |

.github/workflows/workflow_testing.yml

@@ -0,0 +1,93 @@
+name: 'Testing workflows'
+
+on:
+  pull_request:
+
+env:
+  TARGETS: f7
+  DEFAULT_TARGET: f7
+  FBT_TOOLCHAIN_PATH: /opt
+  ACTIONS_RUNNER_DEBUG: true
+
+jobs:
+  unit_tests_dockerized:
+    runs-on: [self-hosted, FlipperZeroIntegrationTest]
+    steps:
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: 'Flash unit tests firmware'
+        id: flashing
+        if: success()
+        timeout-minutes: 10
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+          python3 scripts/testops.py await_flipper
+
+      - name: 'Copy assets and unit data, reboot and wait for flipper'
+        id: copy
+        timeout-minutes: 7
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          rm -rf build/latest/resources/dolphin
+          python3 scripts/storage.py -f send build/latest/resources /ext
+
+      - name: 'Run units and validate results'
+        id: run_units
+        timeout-minutes: 7
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testops.py run_units
+
+      - name: 'Check GDB output'
+        if: failure()
+        run: |
+          ./fbt gdb_trace_all FIRMWARE_APP_SET=unit_tests LIB_DEBUG=1 FORCE=1
+
+      - name: 'Create notification'
+        if: failure()
+        run: |
+          echo "::error ::Unit tests failed, check that step for details"
+
+
+  updater_tests_dockerized:
+    runs-on: [self-hosted, FlipperZeroIntegrationTest]
+    steps:
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          submodules: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: 'Flashing target firmware'
+        id: first_full_flash
+        timeout-minutes: 10
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt flash_usb_full FORCE=1
+          python3 scripts/testops.py await_flipper
+
+      - name: 'Validating updater'
+        id: second_full_flash
+        timeout-minutes: 5
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt flash_usb_full FORCE=1
+          python3 scripts/testops.py await_flipper
+
+      - name: 'Create notification'
+        if: failure()
+        run: |
+          echo "::error ::Flipper updater failure"
+

scripts/testops.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+import time
+from typing import Optional
+
+from flipper.app import App
+from flipper.storage import FlipperStorage
+from flipper.utils.cdc import resolve_port
+
+
+class Main(App):
+    # this is basic use without sub-commands, simply to reboot flipper / power it off, not meant as a full CLI wrapper
+    def init(self):
+        self.parser.add_argument("-p", "--port", help="CDC Port", default="auto")
+
+        self.subparsers = self.parser.add_subparsers(help="sub-command help")
+
+        self.parser_await_flipper = self.subparsers.add_parser(
+            "await_flipper", help="Wait for Flipper to connect or reconnect"
+        )
+        self.parser_await_flipper.set_defaults(func=self.await_flipper)
+
+        self.parser_run_units = self.subparsers.add_parser(
+            "run_units", help="Run unit tests and post result"
+        )
+        self.parser_run_units.set_defaults(func=self.run_units)
+
+    def _get_flipper(self, retry_count: Optional[int] = 1):
+        port = None
+        self.logger.info(f"Attempting to find flipper with {retry_count} attempts.")
+
+        for i in range(retry_count):
+            self.logger.info(f"Attempting to find flipper #{i}.")
+
+            if port := resolve_port(self.logger, self.args.port):
+                self.logger.info(f"Found flipper at {port}")
+                break
+            time.sleep(1)
+
+        if not port:
+            self.logger.info(f"Failed to find flipper")
+            return None
+
+        flipper = FlipperStorage(port)
+        flipper.start()
+        return flipper
+
+    def await_flipper(self):
+        if not (flipper := self._get_flipper(retry_count=90)):
+            return 1
+
+        self.logger.info("Flipper started")
+        flipper.stop()
+        return 0
+
+    def run_units(self):
+        if not (flipper := self._get_flipper(retry_count=10)):
+            return 1
+
+        self.logger.info("Running unit tests")
+        flipper.send("unit_tests" + "\r")
+        self.logger.info("Waiting for unit tests to complete")
+        data = flipper.read.until(">: ")
+        self.logger.info("Parsing result")
+
+        lines = data.decode().split("\r\n")
+
+        tests_re = r"Failed tests: \d{0,}"
+        time_re = r"Consumed: \d{0,}"
+        leak_re = r"Leaked: \d{0,}"
+        status_re = r"Status: \w{3,}"
+
+        tests_pattern = re.compile(tests_re)
+        time_pattern = re.compile(time_re)
+        leak_pattern = re.compile(leak_re)
+        status_pattern = re.compile(status_re)
+
+        tests, elapsed_time, leak, status = None, None, None, None
+        total = 0
+
+        for line in lines:
+            self.logger.info(line)
+            if "()" in line:
+                total += 1
+
+            if not tests:
+                tests = re.match(tests_pattern, line)
+            if not elapsed_time:
+                elapsed_time = re.match(time_pattern, line)
+            if not leak:
+                leak = re.match(leak_pattern, line)
+            if not status:
+                status = re.match(status_pattern, line)
+
+        if None in (tests, elapsed_time, leak, status):
+            self.logger.error(
+                f"Failed to parse output: {tests} {elapsed_time} {leak} {status}"
+            )
+            sys.exit(1)
+
+        leak = int(re.findall(r"[- ]\d+", leak.group(0))[0])
+        status = re.findall(r"\w+", status.group(0))[1]
+        tests = int(re.findall(r"\d+", tests.group(0))[0])
+        elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0])
+
+        if tests > 0 or status != "PASSED":
+            self.logger.error(f"Got {tests} failed tests.")
+            self.logger.error(f"Leaked (not failing on this stat): {leak}")
+            self.logger.error(f"Status: {status}")
+            self.logger.error(f"Time: {elapsed_time/1000} seconds")
+            flipper.stop()
+            return 1
+
+        self.logger.info(f"Leaked (not failing on this stat): {leak}")
+        self.logger.info(
+            f"Tests ran successfully! Time elapsed {elapsed_time/1000} seconds. Passed {total} tests."
+        )
+
+        flipper.stop()
+        return 0
+
+
+if __name__ == "__main__":
+    Main()()