From 39e3a659554d8044db963448df094755961a2266 Mon Sep 17 00:00:00 2001 From: Josh Klar Date: Wed, 24 Jul 2019 20:21:56 -0700 Subject: [PATCH] Full deployment workflow: push finished assets to DigitalOcean Spaces (cdn.kmkfw.io) --- .circleci/config.yml | 30 ++-------------- .s3cfg | 81 ++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 2 +- Makefile | 42 +++++++++++++++-------- Pipfile | 15 ++++---- Pipfile.lock | 36 ++++++++++++++------ 6 files changed, 144 insertions(+), 62 deletions(-) create mode 100644 .s3cfg diff --git a/.circleci/config.yml b/.circleci/config.yml index 056a3a8..c1958b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,44 +7,18 @@ jobs: environment: KMK_TEST: 1 - PIPENV_VENV_IN_PROJECT: 1 steps: - checkout - - restore_cache: - keys: - - v2-kmk-venv-{{ checksum "Pipfile.lock" }} - - - run: make dist - - test: - docker: - - image: 'kmkfw/base' - - environment: - KMK_TEST: 1 - PIPENV_VENV_IN_PROJECT: 1 - - steps: - - checkout - - restore_cache: - keys: - - v2-kmk-venv-{{ checksum "Pipfile.lock" }} - - run: make test + - run: make dist + - run: make dist-deploy workflows: version: 2 build-deploy: jobs: - build: - filters: - branches: - # only: /master/ - only: /.*/ - tags: - only: /.*/ - - test: filters: branches: only: /.*/ diff --git a/.s3cfg b/.s3cfg new file mode 100644 index 0000000..4ae2654 --- /dev/null +++ b/.s3cfg @@ -0,0 +1,81 @@ +[default] +access_key = +access_token = +add_encoding_exts = +add_headers = +bucket_location = US +ca_certs_file = +cache_file = +check_ssl_certificate = True +check_ssl_hostname = True +cloudfront_host = cloudfront.amazonaws.com +content_disposition = +content_type = +default_mime_type = binary/octet-stream +delay_updates = False +delete_after = False +delete_after_fetch = False +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +expiry_date = +expiry_days = +expiry_prefix = +follow_symlinks = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = sfo2.digitaloceanspaces.com +host_bucket = %(bucket)s.sfo2.digitaloceanspaces.com +human_readable_sizes = False +invalidate_default_index_on_cf = False +invalidate_default_index_root_on_cf = True +invalidate_on_cf = False +kms_key = +limit = -1 +limitrate = 0 +list_md5 = False +log_target_prefix = +long_listing = False +max_delete = -1 +mime_type = +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +preserve_attrs = True +progress_meter = True +proxy_host = +proxy_port = 0 +put_continue = False +recursive = False +recv_chunk = 65536 +reduced_redundancy = False +requester_pays = False +restore_days = 1 +restore_priority = Standard +secret_key = +send_chunk = 65536 +server_side_encryption = False +signature_v2 = False +signurl_use_https = False +simpledb_host = sdb.amazonaws.com +skip_existing = False +socket_timeout = 300 +stats = False +stop_on_error = False +storage_class = +throttle_max = 100 +upload_id = +urlencoding_mode = normal +use_http_expect = False +use_https = True +use_mime_magic = True +verbosity = WARNING +website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ +website_error = +website_index = index.html diff --git a/Dockerfile b/Dockerfile index d26d6a7..80d094f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.7-alpine RUN mkdir -p /app WORKDIR /app -RUN apk update && apk add alpine-sdk curl gettext git git-lfs openssh rsync wget zip +RUN apk update && apk add alpine-sdk coreutils curl gettext git git-lfs openssh rsync wget zip RUN pip install pipenv diff --git a/Makefile b/Makefile index b4cba53..1827dab 100644 --- a/Makefile +++ b/Makefile @@ -20,35 +20,51 @@ MPY_CROSS ?= $(shell which mpy-cross 2>/dev/null) MPY_FLAGS ?= '-O2' MPY_SOURCES = 'kmk/' MPY_TARGET_DIR ?= .compiled +PY_KMK_TREE = $(shell find $(MPY_SOURCES) -name "*.py") +DIST_DESCRIBE = $(shell $(DIST_DESCRIBE_CMD)) all: copy-kmk copy-bootpy copy-keymap compile: $(MPY_TARGET_DIR)/.mpy.compiled -$(MPY_TARGET_DIR)/.mpy.compiled: $(shell find $(MPY_SOURCES) -name "*.py") +$(MPY_TARGET_DIR)/.mpy.compiled: $(PY_KMK_TREE) ifeq ($(MPY_CROSS),) @echo "===> Could not find mpy-cross in PATH, exiting" @false endif + @echo "===> Compiling all py files to mpy with flags $(MPY_FLAGS)" @mkdir -p $(MPY_TARGET_DIR) @find $(MPY_SOURCES) -name "*.py" -exec sh -c 'mkdir -p $(MPY_TARGET_DIR)/$$(dirname {}) && mpy-cross $(MPY_FLAGS) {} -o $(MPY_TARGET_DIR)/$$(dirname {})/$$(basename -s .py {}).mpy' \; @touch $(MPY_TARGET_DIR)/.mpy.compiled -dist: dist/latest.zip dist/latest.unoptimized.zip dist/$(shell $(DIST_DESCRIBE_CMD)).zip dist/$(shell $(DIST_DESCRIBE_CMD)).unoptimized.zip +dist: dist/latest.zip dist/latest.unoptimized.zip dist/$(DIST_DESCRIBE).zip dist/$(DIST_DESCRIBE).unoptimized.zip + +dist-deploy: devdeps dist + @echo "===> Uploading artifacts to https://cdn.kmkfw.io/" + @$(PIPENV) run s3cmd -c .s3cfg put dist/$(DIST_DESCRIBE).zip dist/$(DIST_DESCRIBE).unoptimized.zip s3://kmk-releases/ >/dev/null + @[[ "$${CIRCLE_BRANCH}" == "master" ]] && echo "====> Uploading artifacts as 'latest' to https://cdn.kmkfw.io/" || true + @[[ "$${CIRCLE_BRANCH}" == "master" ]] && $(PIPENV) run s3cmd -c .s3cfg put dist/latest.zip dist/latest.unoptimized.zip s3://kmk-releases/ >/dev/null || true + @[[ -n "$${CIRCLE_TAG}" ]] && echo "====> Uploading artifacts as '$${CIRCLE_TAG}' to https://cdn.kmkfw.io/" || true + @[[ -n "$${CIRCLE_TAG}" ]] && $(PIPENV) run s3cmd -c .s3cfg put dist/latest.zip s3://kmk-releases/$${CIRCLE_TAG}.zip >/dev/null || true + @[[ -n "$${CIRCLE_TAG}" ]] && $(PIPENV) run s3cmd -c .s3cfg put dist/latest.unoptimized.zip s3://kmk-releases/$${CIRCLE_TAG}.unoptimized.zip >/dev/null || true dist/latest.zip: compile + @echo "===> Building optimized ZIP" @mkdir -p dist - @cd $(MPY_TARGET_DIR) && zip -r ../dist/latest.zip kmk + @cd $(MPY_TARGET_DIR) && zip -r ../dist/latest.zip kmk 2>&1 >/dev/null -dist/$(shell $(DIST_DESCRIBE_CMD)).zip: dist/latest.zip - @cp dist/latest.zip dist/$$($(DIST_DESCRIBE_CMD)).zip +dist/$(DIST_DESCRIBE).zip: dist/latest.zip + @echo "===> Aliasing optimized ZIP" + @cp dist/latest.zip dist/$(DIST_DESCRIBE).zip -dist/latest.unoptimized.zip: +dist/latest.unoptimized.zip: $(PY_KMK_TREE) + @echo "===> Building unoptimized ZIP" @mkdir -p dist - @zip -r dist/latest.unoptimized.zip kmk + @zip -r dist/latest.unoptimized.zip kmk 2>&1 >/dev/null -dist/$(shell $(DIST_DESCRIBE_CMD)).unoptimized.zip: dist/latest.unoptimized.zip - @cp dist/latest.unoptimized.zip dist/$$($(DIST_DESCRIBE_CMD)).unoptimized.zip +dist/$(DIST_DESCRIBE).unoptimized.zip: dist/latest.unoptimized.zip + @echo "===> Aliasing unoptimized ZIP" + @cp dist/latest.unoptimized.zip dist/$(DIST_DESCRIBE).unoptimized.zip .docker_base: Dockerfile @echo "===> Building Docker base image kmkfw/base:${DOCKER_BASE_TAG}" @@ -63,7 +79,7 @@ docker-base-deploy: docker-base .devdeps: Pipfile.lock @echo "===> Installing dependencies with pipenv" - @$(PIPENV) install --dev --ignore-pipfile + @$(PIPENV) sync --dev @touch .devdeps devdeps: .devdeps @@ -74,14 +90,10 @@ lint: devdeps fix-isort: devdeps @find kmk/ tests/ user_keymaps/ -name "*.py" | xargs $(PIPENV) run isort -clean: clean-build-log +clean: @echo "===> Cleaning build artifacts" @rm -rf .devdeps build dist $(MPY_TARGET_DIR) -clean-build-log: - @echo "===> Clearing previous .build.log" - @rm -rf .build.log - # This is mostly a leftover from the days we vendored stuff from # micropython-lib via submodules. Leaving this here mostly in case someone goes # exploring through the history of KMK's repo and manages to screw up their diff --git a/Pipfile b/Pipfile index bec5f19..e6eb58b 100644 --- a/Pipfile +++ b/Pipfile @@ -4,19 +4,20 @@ verify_ssl = true name = "pypi" [packages] -pydux = "*" [dev-packages] -adafruit-ampy = "*" "flake8" = "*" -"flake8-comprehensions" = "*" -ipython = "*" -ipdb = "*" "flake8-commas" = "*" -isort = "*" +"flake8-comprehensions" = "*" "flake8-isort" = "*" -neovim = "*" "flake8-per-file-ignores" = "*" +"python-magic" = "*" +adafruit-ampy = "*" +ipdb = "*" +ipython = "*" +isort = "*" +neovim = "*" +s3cmd = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 8764c36..93fb96e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "96625b372d35c7f5ed0fd3289ba61afd0bcc034ddad31feb958a107e2751fb0a" + "sha256": "8f51706fcfb278497a6d3083109744210150dc3c644ead11eda3e3cd806e2e14" }, "pipfile-spec": 6, "requires": { @@ -15,16 +15,7 @@ } ] }, - "default": { - "pydux": { - "hashes": [ - "sha256:5cb9217be9d8c7ff79b028f6f02597bbb055b107ce8eecbe5f631f3fc76d793f", - "sha256:bed123b5255d566f792b9ceebad87e3f9c1d2d85abed4b9a9475ffc831035879" - ], - "index": "pypi", - "version": "==0.2.2" - } - }, + "default": {}, "develop": { "adafruit-ampy": { "hashes": [ @@ -269,6 +260,13 @@ ], "version": "==3.4" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "version": "==2.8.0" + }, "python-dotenv": { "hashes": [ "sha256:debd928b49dbc2bf68040566f55cdb3252458036464806f4094487244e2a4093", @@ -276,6 +274,22 @@ ], "version": "==0.10.3" }, + "python-magic": { + "hashes": [ + "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375", + "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5" + ], + "index": "pypi", + "version": "==0.4.15" + }, + "s3cmd": { + "hashes": [ + "sha256:231fe00daedb13095ce38ccf264bbe81d133f6e82f1484ef2f5efdc3f21b9ee8", + "sha256:6d7a3a49a12048a6c8e5fbb5ef42a83101e2fc69f16013d292b7f37ecfc574a0" + ], + "index": "pypi", + "version": "==2.0.2" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",