zmk-docker/.github/workflows/containers.yml

name: Containers

env:
  zephyr-version: 2.4.0
  zephyr-sdk-version: 0.11.4
  run-unit-tests: ${{ secrets.RUN_UNIT_TESTS != null }}
  docker-hub-credentials: ${{ secrets.DOCKER_HUB_USERNAME != null && secrets.DOCKER_HUB_TOKEN != null }}
  ghcr-credentials: ${{ secrets.GHCR_USERNAME != null && secrets.GHCR_TOKEN != null }}
  docker-hub-namespace: ${{ secrets.DOCKER_HUB_NAMESPACE || github.repository_owner }}
  docker-hub-namespace-upstream: ${{ secrets.DOCKER_HUB_NAMESPACE_UPSTREAM || 'zmkfirmware' }}
  ghcr-namespace: ${{ github.repository_owner }}
  zmk-repository: ${{ secrets.ZMK_REPOSITORY || 'zmkfirmware/zmk' }}
  zmk-ref: ${{ secrets.ZMK_REF || 'main' }}

on:
  push:
  pull_request:
  workflow_dispatch:

concurrency: ${{ github.ref }}/${{ github.workflow }}

jobs:
  architectures:
    runs-on: ubuntu-latest
    outputs:
      json: ${{ steps.import.outputs.json }}
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Import from architectures.yml
        id: import
        shell: python
        run: |
          import yaml, json
          with open('architectures.yml', 'r') as file:
            architectures = yaml.safe_load(file)
            print('::set-output name=json::' + json.dumps(architectures))
  tags:
    runs-on: ubuntu-latest
    outputs:
      branch: ${{ steps.definitions.outputs.branch }}
      base: ${{ steps.definitions.outputs.base }}
      candidate: ${{ steps.definitions.outputs.candidate }}
      versions: ${{ steps.definitions.outputs.versions }} # e.g. 2.4.0-0.11.4
      major-minor: ${{ steps.definitions.outputs.major-minor }} # e.g. 2.4
      major-minor-branch: ${{ steps.definitions.outputs.major-minor-branch }} # e.g. 2.4-branch
      latest: ${{ steps.definitions.outputs.latest }}
      release-trigger: ${{ steps.definitions.outputs.release-trigger }}
    steps:
      - name: Definitions
        id: definitions
        env:
          SHA: ${{ github.sha }}
          ZEPHYR_VERSION: ${{ env.zephyr-version }}
          ZEPHYR_SDK_VERSION: ${{ env.zephyr-sdk-version }}
        run: |
          BRANCH=${GITHUB_REF#refs/heads/}
          BRANCH=${BRANCH//[^A-Za-z0-9_.-]/_} # Substitutes invalid Docker tag characters
          BASE=${GITHUB_BASE_REF//[^A-Za-z0-9_.-]/_} # Substitutes invalid Docker tag characters
          CANDIDATE=${SHA}
          VERSIONS=${ZEPHYR_VERSION}-${ZEPHYR_SDK_VERSION}
          MAJOR=$(echo ${ZEPHYR_VERSION} | cut -d'.' -f 1)
          MINOR=$(echo ${ZEPHYR_VERSION} | cut -d'.' -f 2)
          MAJOR_MINOR=${MAJOR}.${MINOR}
          MAJOR_MINOR_BRANCH=${MAJOR_MINOR}-branch
          LATEST=${MAJOR_MINOR}
          RELEASE_TRIGGER=${ZEPHYR_VERSION}-${ZEPHYR_SDK_VERSION}

          echo ::set-output name=branch::${BRANCH}
          echo ::set-output name=base::${BASE}
          echo ::set-output name=candidate::${CANDIDATE}
          echo ::set-output name=versions::${VERSIONS}
          echo ::set-output name=major-minor::${MAJOR_MINOR}
          echo ::set-output name=major-minor-branch::${MAJOR_MINOR_BRANCH}
          echo ::set-output name=latest::${LATEST}
          echo ::set-output name=release-trigger::${RELEASE_TRIGGER}
  dev-generic:
    needs:
      - tags
    if: ${{ !startsWith(github.ref, 'refs/tags') }}
    runs-on: ubuntu-latest
    steps:
      - name: Login to Docker Hub
        id: docker-hub-login
        if: ${{ env.docker-hub-credentials == 'true' }}
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}
      - name: Login to GitHub Container Registry
        id: ghcr-login
        if: ${{ env.ghcr-credentials == 'true' }}
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Define paths
        id: paths
        env:
          NS: ${{ env.docker-hub-namespace }}
          NSU: ${{ env.docker-hub-namespace-upstream }}
          REPOSITORY: zmk-dev-generic-cache
          BRANCH: ${{ needs.tags.outputs.branch }}
          BASE: ${{ needs.tags.outputs.base }}
          MAJOR_MINOR_BRANCH: ${{ needs.tags.outputs.major-minor-branch }}
        run: |
          echo ::set-output name=local::/tmp/.buildx/dev-generic
          echo ::set-output name=local-new::/tmp/.buildx/dev-generic-new
          echo ::set-output name=branch::docker.io/${NS}/${REPOSITORY}:${BRANCH}
          if [ ! -z "$BASE" ]; then
            echo ::set-output name=base::docker.io/${NS}/${REPOSITORY}:${BASE}
          fi
          echo ::set-output name=major-minor-branch::docker.io/${NS}/${REPOSITORY}:${MAJOR_MINOR_BRANCH}
          echo ::set-output name=branch-upstream::docker.io/${NSU}/${REPOSITORY}:${BRANCH}
          echo ::set-output name=major-minor-branch-upstream::docker.io/${NSU}/${REPOSITORY}:${MAJOR_MINOR_BRANCH}
      - name: Set up cache
        id: cache
        uses: actions/cache@v2
        env:
          cache-name: dev-generic
        with:
          path: ${{ steps.paths.outputs.local }}
          key: ${{ runner.os }}/${{ env.cache-name }}/${{ github.run_id }}
      - name: Rebuild cache?
        id: should-rebuild
        run: echo ::set-output name=value::${{ steps.cache.outputs.cache-hit != 'true' }}
      - name: Set up QEMU
        if: ${{ steps.should-rebuild.outputs.value == 'true' }}
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        if: ${{ steps.should-rebuild.outputs.value == 'true' }}
        uses: docker/setup-buildx-action@v1
      - name: Build to local cache
        if: ${{ steps.should-rebuild.outputs.value == 'true' }}
        uses: docker/build-push-action@v2
        with:
          target: dev-generic
          build-args: |
            ZEPHYR_VERSION=${{ env.zephyr-version }}
          cache-from: |
            type=registry,ref=${{ steps.paths.outputs.branch }}
            ${{ (steps.paths.outputs.base != '') && format('type=registry,ref={0}', steps.paths.outputs.base) || '' }}
            type=registry,ref=${{ steps.paths.outputs.major-minor-branch }}
            type=registry,ref=${{ steps.paths.outputs.branch-upstream }}
            type=registry,ref=${{ steps.paths.outputs.major-minor-branch-upstream }}
          cache-to: type=local,dest=${{ steps.paths.outputs.local-new }},mode=max
      - name: Push to registry cache
        if: ${{ (steps.should-rebuild.outputs.value == 'true') && (env.docker-hub-credentials == 'true') }}
        uses: docker/build-push-action@v2
        with:
          target: dev-generic
          build-args: |
            ZEPHYR_VERSION=${{ env.zephyr-version }}
          cache-from: type=local,src=${{ steps.paths.outputs.local-new }}
          cache-to: type=registry,ref=${{ steps.paths.outputs.branch }},mode=max
      # Workaround to stop the dev-generic cache ballooning ...
      # https://github.com/docker/build-push-action/issues/252
      # https://github.com/moby/buildkit/issues/1896
      - name: Switch local cache
        if: ${{ steps.should-rebuild.outputs.value == 'true' }}
        run: |
          rm -rf ${{ steps.paths.outputs.local }}
          mv ${{ steps.paths.outputs.local-new }} ${{ steps.paths.outputs.local }}
  candidates:
    needs:
      - architectures
      - tags
      - dev-generic
    if: ${{ !startsWith(github.ref, 'refs/tags') }}
    runs-on: ubuntu-latest
    env:
      docker-args: --rm --workdir /github/workspace -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/_temp:/home/runner/work/_temp -v /home/runner/work/_temp/_github_home:/github/home -v /home/runner/work/_temp/_github_workflow:/github/workflow -v /home/runner/work/_temp/_runner_file_commands:/github/file_commands -v ${{ github.workspace }}:/github/workspace
    defaults:
      run:
        shell: /usr/bin/docker exec candidate /bin/bash {0}
    strategy:
      max-parallel: 1 # takes advantage of caching between jobs
      matrix: 
        architecture: ${{ fromJSON(needs.architectures.outputs.json) }}
        include:
          - architecture: arm
            board: nice_nano
            shield: qaz
    steps:
      - name: Login to Docker Hub
        id: docker-hub-login
        if: ${{ env.docker-hub-credentials == 'true' }}
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}
      - name: Login to GitHub Container Registry
        id: ghcr-login
        if: ${{ env.ghcr-credentials == 'true' }}
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Define repositories
        id: repositories
        shell: bash
        run: |
          echo ::set-output name=build::zmk-build-${{ matrix.architecture }}
          echo ::set-output name=dev::zmk-dev-${{ matrix.architecture }}
      - name: Define paths
        id: paths
        shell: bash
        env:
          NS: ${{ env.docker-hub-namespace }}
          NSU: ${{ env.docker-hub-namespace-upstream }}
          BUILD: ${{ steps.repositories.outputs.build }}
          DEV: ${{ steps.repositories.outputs.dev }}
          CANDIDATE: ${{ needs.tags.outputs.candidate }}
          BRANCH: ${{ needs.tags.outputs.branch }}
          BASE: ${{ needs.tags.outputs.base }}
          MAJOR_MINOR_BRANCH: ${{ needs.tags.outputs.major-minor-branch }}
        run: |
          echo ::set-output name=dev-generic::/tmp/.buildx/dev-generic
          echo ::set-output name=build-candidate::docker.io/${NS}/${BUILD}:${CANDIDATE}
          echo ::set-output name=build-branch::docker.io/${NS}/${BUILD}:${BRANCH}
          if [ ! -z "$BASE" ]; then
            echo ::set-output name=build-base::docker.io/${NS}/${BUILD}:${BASE}
          fi
          echo ::set-output name=build-major-minor-branch::docker.io/${NS}/${BUILD}:${MAJOR_MINOR_BRANCH}
          echo ::set-output name=build-branch-upstream::docker.io/${NSU}/${BUILD}:${BRANCH}
          echo ::set-output name=build-major-minor-branch-upstream::docker.io/${NSU}/${BUILD}:${MAJOR_MINOR_BRANCH}
          echo ::set-output name=dev-candidate::docker.io/${NS}/${DEV}:${CANDIDATE}
          echo ::set-output name=dev-branch::docker.io/${NS}/${DEV}:${BRANCH}
          if [ ! -z "$BASE" ]; then
            echo ::set-output name=dev-base::docker.io/${NS}/${DEV}:${BASE}
          fi
          echo ::set-output name=dev-major-minor-branch::docker.io/${NS}/${DEV}:${MAJOR_MINOR_BRANCH}
          echo ::set-output name=dev-branch-upstream::docker.io/${NSU}/${DEV}:${BRANCH}
          echo ::set-output name=dev-major-minor-branch-upstream::docker.io/${NSU}/${DEV}:${MAJOR_MINOR_BRANCH}
      - name: Define build-args
        id: build-args
        shell: bash
        run: |
          LIST="
            ZEPHYR_VERSION=${{ env.zephyr-version }}
            ARCHITECTURE=${{ matrix.architecture }}
            ZEPHYR_SDK_VERSION=${{ env.zephyr-sdk-version }}
          "
          # Escapes %, \n and \r
          # See: https://github.community/t/set-output-truncates-multiline-strings/16852
          LIST="${LIST//'%'/'%25'}"
          LIST="${LIST//$'\n'/'%0A'}"
          LIST="${LIST//$'\r'/'%0D'}"
          echo ::set-output name=list::${LIST}
      - name: Define labels
        id: labels
        shell: bash
        run: |
          LIST="
            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
            org.opencontainers.image.revision=${{ github.sha }}
          "
          # Escapes %, \n and \r
          # See: https://github.community/t/set-output-truncates-multiline-strings/16852
          LIST="${LIST//'%'/'%25'}"
          LIST="${LIST//$'\n'/'%0A'}"
          LIST="${LIST//$'\r'/'%0D'}"
          echo ::set-output name=list::${LIST}
      - name: Set up dev-generic cache
        id: dev-generic-cache
        uses: actions/cache@v2
        env:
          cache-name: dev-generic
        with:
          path: ${{ steps.paths.outputs.dev-generic }}
          key: ${{ runner.os }}/${{ env.cache-name }}/${{ github.run_id }}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build and load 'build' candidate image
        uses: docker/build-push-action@v2
        with:
          target: build
          build-args: |
            ${{ steps.build-args.outputs.list }}
          labels: |
            ${{ steps.labels.outputs.list }}
          tags: |
            ${{ steps.paths.outputs.build-candidate }}
            ${{ steps.paths.outputs.build-branch }}
          cache-from: |
            type=local,src=${{ steps.paths.outputs.dev-generic }}
            type=registry,ref=${{ steps.paths.outputs.build-candidate }}
            type=registry,ref=${{ steps.paths.outputs.build-branch }}
            ${{ (steps.paths.outputs.build-base != '') && format('type=registry,ref={0}', steps.paths.outputs.build-base) || '' }}
            type=registry,ref=${{ steps.paths.outputs.build-major-minor-branch }}
            type=registry,ref=${{ steps.paths.outputs.build-branch-upstream }}
            type=registry,ref=${{ steps.paths.outputs.build-major-minor-branch-upstream }}
          cache-to: type=inline
          load: true
      - name: Build and load 'dev' candidate image
        uses: docker/build-push-action@v2
        with:
          target: dev
          build-args: |
            ${{ steps.build-args.outputs.list }}
          labels: |
            ${{ steps.labels.outputs.list }}
          tags: |
            ${{ steps.paths.outputs.dev-candidate }}
            ${{ steps.paths.outputs.dev-branch }}
          cache-from: |
            type=registry,ref=${{ steps.paths.outputs.build-candidate }}
            type=local,src=${{ steps.paths.outputs.dev-generic }}
            type=registry,ref=${{ steps.paths.outputs.dev-candidate }}
            type=registry,ref=${{ steps.paths.outputs.dev-branch }}
            ${{ (steps.paths.outputs.dev-base != '') && format('type=registry,ref={0}', steps.paths.outputs.dev-base) || '' }}
            type=registry,ref=${{ steps.paths.outputs.dev-major-minor-branch }}
            type=registry,ref=${{ steps.paths.outputs.dev-branch-upstream }}
            type=registry,ref=${{ steps.paths.outputs.dev-major-minor-branch-upstream }}
          cache-to: type=inline
          load: true
      - name: Checkout ZMK
        uses: actions/checkout@v2
        with:
          repository: ${{ env.zmk-repository }}
          ref: ${{ env.zmk-ref }}
      - name: Cache Zephyr modules
        uses: actions/cache@v2
        env:
          cache-name: zephyr-modules
        with:
          path: |
            modules/
            tools/
            zephyr/
            bootloader/
          key: ${{ runner.os }}/${{ env.cache-name }}/${{ hashFiles('app/west.yml') }}
          restore-keys: |
            ${{ runner.os }}/${{ env.cache-name }}/
      - name: Create and run container from 'build' candidate image
        shell: bash
        run: docker run -d -it --name candidate ${{ env.docker-args }} ${{ steps.paths.outputs.build-candidate }}
      - name: Test diff
        run: diff --version
      - name: Test west init
        run: west init -l app
      - name: Test west update
        run: west update
      - name: Test west zephyr-export
        run: west zephyr-export
      - name: Test board/shield (west build)
        if: ${{ matrix.board != null }}
        run: west build -s app -b ${{ matrix.board }} -- ${{ matrix.shield != null && format('-DSHIELD={0}', matrix.shield) || null }}
      - name: Test RAM report (west build)
        run: west build -t ram_report
      - name: Test ROM report (west build)
        run: west build -t rom_report
      - name: Test west test (single)
        run: west test tests/none/normal
      - name: Test west test (full)
        if: ${{ env.run-unit-tests == 'true' }}
        run: west test
      - name: Test clean (west build)
        run: west build -t clean
      - name: Stop container
        shell: bash
        run: docker stop candidate
      - name: Create and run container from 'dev' candidate image
        shell: bash
        run: docker run -d -it --name candidate ${{ env.docker-args }} ${{ steps.paths.outputs.dev-candidate }}
      - name: Test clang-format
        run: clang-format --version
      - name: Test node
        run: node --version
      - name: Test docs ci
        run: cd docs && npm ci
      - name: Test docs lint
        run: cd docs && npm run lint
      - name: Test docs prettier check
        run: cd docs && npm run prettier:check
      - name: Test docs start (webpack-dev-server)
        run: cd docs && timeout -s SIGINT 20 npm run start &
      - run: sleep 15
      - name: Test docs wget (webpack-dev-server)
        run: wget http://localhost:3000
      - run: sleep 10
      - name: Test docs build (webpack)
        run: cd docs && npm run build
      - name: Test docs serve (webpack)
        run: cd docs && timeout -s SIGINT 10 npm run serve &
      - run: sleep 5
      - name: Test docs wget (webpack)
        run: wget http://localhost:3000
      - name: Test ssh
        run: ssh -V
      - name: Stop container
        shell: bash
        run: docker stop candidate
      - name: Push candidate images to the registry
        if: ${{ steps.docker-hub-login.outcome == 'success' }}
        shell: bash
        run: |
          docker image push ${{ steps.paths.outputs.build-candidate }}
          docker image push ${{ steps.paths.outputs.build-branch }}
          docker image push ${{ steps.paths.outputs.dev-candidate }}
          docker image push ${{ steps.paths.outputs.dev-branch }}
  releases:
    needs:
      - architectures
      - tags
    if: ${{ github.ref == format('refs/tags/{0}', needs.tags.outputs.release-trigger) }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        architecture: ${{ fromJSON(needs.architectures.outputs.json) }}
        target:
          - build
          - dev
    steps:
      - name: Login to GitHub Container Registry
        id: ghcr-login
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Login to Docker Hub
        id: docker-hub-login
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}
      - name: Release (pull candidate, tag, push)
        env:
          DHNS: ${{ env.docker-hub-namespace }}
          GHCRNS: ${{ env.ghcr-namespace }}
          TARGET: ${{ matrix.target }}
          ARCHITECTURE: ${{ matrix.architecture }}
          CANDIDATE: ${{ needs.tags.outputs.candidate }}
          VERSIONS: ${{ needs.tags.outputs.versions }}
          LATEST: ${{ needs.tags.outputs.latest }}
        run: |
          REPOSITORY=zmk-${TARGET}-${ARCHITECTURE}
          
          docker pull docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE}
          docker tag docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE} docker.io/${DHNS}/${REPOSITORY}:${VERSIONS}
          docker tag docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE} docker.io/${DHNS}/${REPOSITORY}:${LATEST}
          docker tag docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE} ghcr.io/${GHCRNS}/${REPOSITORY}:${CANDIDATE}
          docker tag docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE} ghcr.io/${GHCRNS}/${REPOSITORY}:${VERSIONS}
          docker tag docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE} ghcr.io/${GHCRNS}/${REPOSITORY}:${LATEST}
          docker push docker.io/${DHNS}/${REPOSITORY}:${CANDIDATE}
          docker push docker.io/${DHNS}/${REPOSITORY}:${VERSIONS}
          docker push docker.io/${DHNS}/${REPOSITORY}:${LATEST}
          docker push ghcr.io/${GHCRNS}/${REPOSITORY}:${CANDIDATE}
          docker push ghcr.io/${GHCRNS}/${REPOSITORY}:${VERSIONS}
          docker push ghcr.io/${GHCRNS}/${REPOSITORY}:${LATEST}
  git-tag:
    needs:
    - tags
    - releases
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Tag
        env:
          TAG: ${{ needs.tags.outputs.major-minor }}
        run: |
          git tag ${TAG}
          git push -f origin ${TAG}