Some backports from work on #46 to greatly speed up Circle builds

This backports from topic-docker the base image, which we can now use in Circle to MASSIVELY speed up our CI situation. The base image can currently only be pushed by @klardotsh as nobody else has access to the Docker Hub org, but that can change in the future.
2018-10-11 17:44:02 -07:00
parent 3c5c95cfd7
commit 7d2d04f070
5 changed files with 55 additions and 85 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,134 +1,66 @@
 version: 2
 jobs:
-  lint:
-    docker:
-      - image: 'ubuntu:bionic'
-
-    environment:
-        KMK_TEST: 1
-        PIPENV_VENV_IN_PROJECT: 1
-
-    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh wget
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
-      - run: make lint
-
-      - save_cache:
-          key: v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-          paths:
-            - .venv
-
  test:
    docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'

    environment:
        KMK_TEST: 1
        PIPENV_VENV_IN_PROJECT: 1

    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
      - checkout
      - restore_cache:
          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}

-      - run: python3.7 -m pip install pipenv==2018.7.1
      - run: make test

  build_feather_m4_express:
    docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'

    environment:
        KMK_TEST: 1
        PIPENV_VENV_IN_PROJECT: 1

    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
      - checkout
      - restore_cache:
          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}

      - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-feather-m4-express

  build_itsybitsy_m4_express:
    docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'

    environment:
        KMK_TEST: 1
        PIPENV_VENV_IN_PROJECT: 1

    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
      - checkout
      - restore_cache:
          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}

      - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-itsybitsy-m4-express

  build_pyboard:
    docker:
-      - image: 'ubuntu:bionic'
+      - image: 'kmkfw/base'

    environment:
        KMK_TEST: 1
        PIPENV_VENV_IN_PROJECT: 1

    steps:
-      - run: apt-get update && apt-get install -y software-properties-common git ssh
-      - run: add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: add-apt-repository ppa:deadsnakes/ppa
-      - run: apt-get update && apt-get install -y gcc-arm-embedded gettext wget unzip rsync python3.7 python3.7-dev build-essential pkg-config libffi-dev
-      - run: wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py
-      - run: python3.7 /tmp/get-pip.py
-      - run: python3.7 -m pip install pip==18.0  # downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
-
      - checkout
      - restore_cache:
          keys:
-            - v1-kmk-venv-{{ checksum "Pipfile.lock" }}
-
-      - run: python3.7 -m pip install pipenv==2018.7.1
+            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}

      - run: make SKIP_KEYMAP_VALIDATION=1 USER_KEYMAP=user_keymaps/noop.py build-pyboard

@@ -136,20 +68,12 @@ workflows:
  version: 2
  build-deploy:
    jobs:
-      - lint:
-          filters:
-            branches:
-              only: /.*/
-            tags:
-              only: /.*/
      - test:
          filters:
            branches:
              only: /.*/
            tags:
              only: /.*/
-          requires:
-            - lint
      - build_feather_m4_express:
          filters:
            branches:
--- a/.dockerignore
+++ b/.dockerignore
@@ -0,0 +1,10 @@
+build
+vendor
+.devdeps
+.submodules
+.idea
+.micropython-deps
+.circuitpython-deps
+.ampy
+.venv
+venv
--- a/.gitignore
+++ b/.gitignore
@@ -108,6 +108,7 @@ venv.bak/
 .circuitpy-deps
 .micropython-deps
 .devdeps
+.docker_base

 # Pycharms cruft
 .idea
--- a/21
+++ b/21
@@ -0,0 +1,21 @@
+# vim: ft=dockerfile
+
+# Not using python:3.7 here because team-gcc-arm-embedded/ppa does not support
+# Ubuntu Cosmic or Debian Stretch, and Alpine, bizarrely, does not seem to
+# package GCC cross compilers
+FROM ubuntu:bionic
+
+# Set up PPAs we'll need for Python and for GCC ARM
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN add-apt-repository ppa:team-gcc-arm-embedded/ppa
+
+# Install Python
+RUN apt-get update && apt-get install -y python3.7 python3.7-dev build-essential pkg-config libffi-dev curl
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+# Downgrade pip to work around https://github.com/pypa/pipenv/issues/2924
+RUN python3.7 -m pip install pip==18.0
+RUN python3.7 -m pip install pipenv==2018.7.1
+
+# Install KMK CI and/or build-time dependencies
+RUN apt-get install -y gcc-arm-embedded gettext ssh wget unzip rsync git locales libusb-dev
--- a/16
+++ b/16
@@ -8,12 +8,26 @@
 	freeze-nrf-build-deps \
 	lint

+DOCKER_BASE_TAG ?= latest
+DOCKER_TAG ?= latest
+
 AMPY_PORT ?= /dev/ttyUSB0
 AMPY_BAUD ?= 115200
 AMPY_DELAY ?= 1.5
 ARDUINO ?= /usr/share/arduino
 PIPENV ?= $(shell which pipenv)

+.docker_base: Dockerfile_base
+	@echo "===> Building Docker base image kmkfw/base:${DOCKER_BASE_TAG}"
+	@docker build -f Dockerfile_base -t kmkfw/base:${DOCKER_BASE_TAG} .
+	@touch .docker_base
+
+docker-base: .docker_base
+
+docker-base-deploy: docker-base
+	@echo "===> Pushing Docker base image kmkfw/base:${DOCKER_BASE_TAG} to Docker Hub"
+	@docker push kmkfw/base:${DOCKER_BASE_TAG}
+
 .devdeps: Pipfile.lock
 	@echo "===> Installing dependencies with pipenv"
 	@$(PIPENV) install --dev --ignore-pipfile
@@ -41,7 +55,7 @@ powerwash: clean
 	@echo "===> Removing pipenv-managed virtual environment"
 	@$(PIPENV) --rm || true

-test: micropython-build-unix
+test: lint micropython-build-unix
 	@echo "===> Testing keymap_sanity_check.py script"
 	@echo "    --> Known good layout should pass..."
 	@MICROPYPATH=tests/test_data:./ ./bin/micropython.sh bin/keymap_sanity_check.py keymaps/known_good.py