From 20c98377706274953a2de3704b84f615027e2ac8 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Sun, 22 Mar 2020 15:52:56 -0300 Subject: [PATCH 01/29] Created the .github/workflows/windows.yml # Conflicts: # .github/scripts/trailing-newlines.sh # Makefile # qt/tools/typecheck-setup.sh # react/Makefile # rspy/Makefile # svelte/Makefile # tslib/Makefile --- .github/scripts/contrib.sh | 2 +- .github/scripts/trailing-newlines.sh | 2 +- .../{checks.yml => linux_checks.yml} | 2 +- .github/workflows/windows_checks.yml | 76 +++++++++++++++++++ .gitignore | 1 + Makefile | 58 +++++++++----- README.contributing | 2 +- README.development | 8 +- pylib/Makefile | 12 ++- qt/Makefile | 12 ++- qt/ftl/scripts/fetch-latest-translations | 2 + qt/ftl/scripts/upload-latest-templates | 2 + qt/po/scripts/build-mo-files | 3 +- qt/po/scripts/copy-qt-files | 7 +- qt/po/scripts/fetch-latest-translations | 2 + qt/po/scripts/update-po-template | 3 +- qt/po/scripts/upload-latest-template | 2 +- qt/tools/build_ui.sh | 2 +- qt/tools/typecheck-setup.sh | 18 ++++- rslib/Makefile | 11 ++- rslib/ftl/scripts/fetch-latest-translations | 2 + rslib/ftl/scripts/upload-latest-templates | 2 + rspy/Makefile | 35 ++++++--- run | 2 + 24 files changed, 212 insertions(+), 56 deletions(-) rename .github/workflows/{checks.yml => linux_checks.yml} (97%) create mode 100644 .github/workflows/windows_checks.yml diff --git a/.github/scripts/contrib.sh b/.github/scripts/contrib.sh index 0bcf6153e..46da0954e 100755 --- a/.github/scripts/contrib.sh +++ b/.github/scripts/contrib.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} antispam=", at the domain " diff --git a/.github/scripts/trailing-newlines.sh b/.github/scripts/trailing-newlines.sh index 332e65280..e310c6747 100755 --- a/.github/scripts/trailing-newlines.sh +++ b/.github/scripts/trailing-newlines.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -eu -o pipefail ${SHELLFLAGS} files=$(rg -l '[^\n]\z' -g '!*.{svg,scss,json,sql}' || true) if [ "$files" != "" ]; then diff --git a/.github/workflows/checks.yml b/.github/workflows/linux_checks.yml similarity index 97% rename from .github/workflows/checks.yml rename to .github/workflows/linux_checks.yml index 4871a7c18..92b7c855e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/linux_checks.yml @@ -1,4 +1,4 @@ -name: Checks +name: Linux Tests on: [push, pull_request] diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml new file mode 100644 index 000000000..e05d89404 --- /dev/null +++ b/.github/workflows/windows_checks.yml @@ -0,0 +1,76 @@ +name: Windows Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: windows-latest + steps: + - run: git config --global core.autocrlf false + - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") + $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") + $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") + echo "::set-env name=pacmanbin::pacman-5.1.1-3-x86_64.pkg.tar.xz" + echo "::set-env name=pacmanmirror::pacman-mirrors-20200307-1-any.pkg.tar.xz" + echo "::set-env name=pacmankeys::msys2-keyring-r9.397a52e-1-any.pkg.tar.xz" + echo "::set-env name=pyaudio::$pyaudio" + echo "::set-env name=PATH::$new_path" + echo "::set-env name=RUST_BACKTRACE::full" + echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" + echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" + echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" + - name: Set up pacman, pyaudio, rsync, rename + shell: cmd + run: | + echo on + curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename + :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz + 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + :: Manually Install Pacman Binaries + :: https://github.com/Alexpux/MSYS2-pacman/issues/50 + cd /d "%programfiles%\Git\usr\bin" + .\bash pacman-key --init + .\bash pacman-key --populate msys2 + .\bash pacman-key --refresh-keys + .\pacman -Tv + .\pacman -Syuv --overwrite='*' + :: We must install bash first, otherwise we will have bash fork errors: + :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true + :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly + .\pacman --version + .\pacman -Sv --noconfirm --overwrite='*' bash + .\pacman -Sv --noconfirm --overwrite='*' rsync + - name: Set up scoop, gettext, ripgrep + shell: cmd + run: | + echo on + powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" + scoop install -g gettext ripgrep + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: Arduino/actions/setup-protoc@master + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Run checks + shell: cmd + run: | + echo on + set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "BUILDFLAGS=" + :: set "SHELLFLAGS=-x" + make check build diff --git a/.gitignore b/.gitignore index 7e4935ee6..57ab44f16 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .build .coverage .DS_Store +rename dist pyenv .mypy_cache diff --git a/Makefile b/Makefile index fdfa2c182..83325e02a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,23 @@ SHELL := /bin/bash +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + ifndef OS OS := unknown endif -ifeq ($(OS),Windows_NT) +ifeq (${OS},Windows_NT) + # Windows terminal is confusing it with its `cmd` builtin `rename` command + ifndef RENAME_BIN + RENAME_BIN := perl rename + endif + ifndef ACTIVATE_SCRIPT ACTIVATE_SCRIPT := pyenv/Scripts/activate endif @@ -13,6 +26,10 @@ ifeq ($(OS),Windows_NT) PYTHON_BIN := python endif else + ifndef RENAME_BIN + RENAME_BIN := rename + endif + ifndef ACTIVATE_SCRIPT ACTIVATE_SCRIPT := pyenv/bin/activate endif @@ -26,10 +43,7 @@ ifndef ANKI_EXTRA_PIP ANKI_EXTRA_PIP := true endif -.SHELLFLAGS := -eu -o pipefail -c .DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules SUBMAKE := $(MAKE) --print-directory .SUFFIXES: @@ -46,7 +60,13 @@ all: run # - modern pip required for wheel # - add qt if missing pyenv: + "${PYTHON_BIN}" -m ensurepip && \ + "${PYTHON_BIN}" -m pip install virtualenv && \ "${PYTHON_BIN}" -m venv pyenv && \ + case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) \ + dos2unix "${ACTIVATE_SCRIPT}" && \ + sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$(shell pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" \ + ;; esac; \ . "${ACTIVATE_SCRIPT}" && \ python --version && \ python -m pip install --upgrade pip setuptools && \ @@ -56,25 +76,23 @@ pyenv: # update build hash .PHONY: buildhash buildhash: - @oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true); \ - newhash=$$(git rev-parse --short=8 HEAD || echo dev); \ + @oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true) && \ + newhash=$$(git rev-parse --short=8 HEAD || echo dev) && \ if [ "$$oldhash" != "$$newhash" ]; then \ echo $$newhash > meta/buildhash; \ fi .PHONY: develop develop: pyenv buildhash prepare - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ + @. "${ACTIVATE_SCRIPT}" && \ for dir in $(DEVEL); do \ $(SUBMAKE) -C $$dir develop DEVFLAGS="$(DEVFLAGS)"; \ done .PHONY: run run: develop - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ - echo "Starting Anki..."; \ + @. "${ACTIVATE_SCRIPT}" && \ + echo "Starting Anki..." && \ python qt/runanki $(RUNFLAGS) .PHONY: prepare @@ -109,8 +127,7 @@ build-qt: .PHONY: clean clean: clean-dist - @set -eo pipefail && \ - for dir in $(DEVEL); do \ + @for dir in $(DEVEL); do \ $(SUBMAKE) -C $$dir clean; \ done @@ -120,8 +137,7 @@ clean-dist: .PHONY: check check: pyenv buildhash prepare - @set -eo pipefail && \ - .github/scripts/trailing-newlines.sh && \ + @.github/scripts/trailing-newlines.sh && \ for dir in $(CHECKABLE_RS); do \ $(SUBMAKE) -C $$dir check; \ done && \ @@ -136,17 +152,19 @@ check: pyenv buildhash prepare .PHONY: fix fix: - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ + @. "${ACTIVATE_SCRIPT}" && \ for dir in $(CHECKABLE_RS) $(CHECKABLE_PY); do \ $(SUBMAKE) -C $$dir fix; \ done; \ .PHONY: add-buildhash add-buildhash: - @ver=$$(cat meta/version); \ - hash=$$(cat meta/buildhash); \ - rename "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-$$ver-* + @if [[ ! -f rename ]]; then \ + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename; \ + fi && \ + ver="$$(cat meta/version)" && \ + hash="$$(cat meta/buildhash)" && \ + ${RENAME_BIN} "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-$$ver-* .PHONY: pull-i18n diff --git a/README.contributing b/README.contributing index b04c808e0..975c90c17 100644 --- a/README.contributing +++ b/README.contributing @@ -114,7 +114,7 @@ You can do this automatically by adding the following into .git/hooks/pre-commit or .git/hooks/pre-push and making it executable. #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} make check You may need to adjust the PATH variable so that things like a local install diff --git a/README.development b/README.development index 76270769b..c6031425b 100644 --- a/README.development +++ b/README.development @@ -19,10 +19,12 @@ To start, make sure you have the following installed: - protoc v3 (https://github.com/protocolbuffers/protobuf/releases) - rustup (https://rustup.rs/) - gettext - - rename + - rename (from the perl script https://github.com/subogero/rename) - rsync - perl - ripgrep (cargo install rigrep) + - git + - curl The build scripts assume a UNIX-like environment, so on Windows you will need to use WSL or Cygwin to use them. @@ -60,6 +62,10 @@ folder. Use 'make clean' to remove some generated files. +To see all commands run by make or any shell script, export the environment +variable SHELLFLAGS with '-x' to tell shell to print all commands run by it. +For example, 'export SHELLFLAGS=-x' on Linux or 'set SHELLFLAGS=-x' on Windows. + PyQt ----- diff --git a/pylib/Makefile b/pylib/Makefile index c78af5bf4..d019999a2 100644 --- a/pylib/Makefile +++ b/pylib/Makefile @@ -1,10 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules + +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) + +.DELETE_ON_ERROR: RUNARGS := .SUFFIXES: BLACKARGS := -t py36 anki tests setup.py tools/*.py --exclude='_pb2|buildinfo' diff --git a/qt/Makefile b/qt/Makefile index 82cd91292..d92370c9d 100644 --- a/qt/Makefile +++ b/qt/Makefile @@ -1,10 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules + +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) + +.DELETE_ON_ERROR: .SUFFIXES: BLACKARGS := -t py36 aqt tests setup.py tools/*.py --exclude='aqt/forms|buildinfo|colors' diff --git a/qt/ftl/scripts/fetch-latest-translations b/qt/ftl/scripts/fetch-latest-translations index 5285d7031..108eed1f5 100755 --- a/qt/ftl/scripts/fetch-latest-translations +++ b/qt/ftl/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/qt/ftl/scripts/upload-latest-templates b/qt/ftl/scripts/upload-latest-templates index a60f3ff2b..058835553 100755 --- a/qt/ftl/scripts/upload-latest-templates +++ b/qt/ftl/scripts/upload-latest-templates @@ -3,6 +3,8 @@ # expects to be run from the ftl folder # +set -eu -o pipefail ${SHELLFLAGS} + test -d repo || exit 1 rsync -av --delete *.ftl repo/desktop/templates/ diff --git a/qt/po/scripts/build-mo-files b/qt/po/scripts/build-mo-files index ea107015b..9e04f6a34 100755 --- a/qt/po/scripts/build-mo-files +++ b/qt/po/scripts/build-mo-files @@ -2,7 +2,8 @@ # # build mo files # -set -eo pipefail + +set -eu -o pipefail ${SHELLFLAGS} targetDir="../aqt_data/locale/gettext" mkdir -p $targetDir diff --git a/qt/po/scripts/copy-qt-files b/qt/po/scripts/copy-qt-files index 0a8a7eb25..248ca491c 100755 --- a/qt/po/scripts/copy-qt-files +++ b/qt/po/scripts/copy-qt-files @@ -1,15 +1,14 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} out=../aqt_data/locale/qt mkdir -p "$out" qtTranslations="$(python -c "from PyQt5.QtCore import *; import sys; sys.stdout.write(QLibraryInfo.location(QLibraryInfo.TranslationsPath))")" -unameOut="$(uname -s)" -case "${unameOut}" in - CYGWIN*) +case "$(uname -s)" in + CYGWIN*|MINGW*|MSYS*) qtTranslations="$(cygpath -u "${qtTranslations}")" ;; esac diff --git a/qt/po/scripts/fetch-latest-translations b/qt/po/scripts/fetch-latest-translations index 286745fe4..9d0e04223 100755 --- a/qt/po/scripts/fetch-latest-translations +++ b/qt/po/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/qt/po/scripts/update-po-template b/qt/po/scripts/update-po-template index 223cada7a..d36abc917 100755 --- a/qt/po/scripts/update-po-template +++ b/qt/po/scripts/update-po-template @@ -2,7 +2,8 @@ # # update template .pot file from source code strings # -set -eo pipefail + +set -eu -o pipefail ${SHELLFLAGS} topDir=$(dirname $0)/../../../ cd $topDir diff --git a/qt/po/scripts/upload-latest-template b/qt/po/scripts/upload-latest-template index cef056142..5935a1f94 100755 --- a/qt/po/scripts/upload-latest-template +++ b/qt/po/scripts/upload-latest-template @@ -1,6 +1,6 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} scripts/update-po-template (cd repo && git add desktop; git commit -m update; git push) diff --git a/qt/tools/build_ui.sh b/qt/tools/build_ui.sh index 80e28d37a..7b5db20c5 100755 --- a/qt/tools/build_ui.sh +++ b/qt/tools/build_ui.sh @@ -4,7 +4,7 @@ # should be on the path. # -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} if [ ! -d "designer" ] then diff --git a/qt/tools/typecheck-setup.sh b/qt/tools/typecheck-setup.sh index 65311f07c..ee68cb2a2 100755 --- a/qt/tools/typecheck-setup.sh +++ b/qt/tools/typecheck-setup.sh @@ -6,8 +6,22 @@ # able to resolve. A solution that doesn't require modifying the python install # would be welcome! +set -eu -o pipefail ${SHELLFLAGS} + +# https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash +if [[ -z "${OS+x}" ]]; then + OS=unknown; +fi + TOOLS="$(cd "`dirname "$0"`"; pwd)" -modDir=$(python -c 'import PyQt5, sys, os; print(os.path.dirname(sys.modules["PyQt5"].__file__))') -cmd="rsync -a $TOOLS/stubs/PyQt5/* $modDir/" +modDir=$(python -c 'import PyQt5, sys, os; sys.stdout.write(os.path.dirname(sys.modules["PyQt5"].__file__))') + +case "$(uname -s)" in + CYGWIN*|MINGW*|MSYS*) + modDir="$(cygpath -u "${modDir}")" + ;; +esac + +cmd="rsync -a \"${TOOLS}/stubs/PyQt5/\" \"${modDir}/\"" $cmd > /dev/null 2>&1 || sudo $cmd diff --git a/rslib/Makefile b/rslib/Makefile index 2c38d2723..6551bbfe8 100644 --- a/rslib/Makefile +++ b/rslib/Makefile @@ -1,11 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) +.DELETE_ON_ERROR: + $(shell mkdir -p .build) .PHONY: all check fix clean develop diff --git a/rslib/ftl/scripts/fetch-latest-translations b/rslib/ftl/scripts/fetch-latest-translations index 36b579cc6..1156605a5 100755 --- a/rslib/ftl/scripts/fetch-latest-translations +++ b/rslib/ftl/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/rslib/ftl/scripts/upload-latest-templates b/rslib/ftl/scripts/upload-latest-templates index 0c4b98d7b..474d899c8 100755 --- a/rslib/ftl/scripts/upload-latest-templates +++ b/rslib/ftl/scripts/upload-latest-templates @@ -3,6 +3,8 @@ # expects to be run from the ftl folder # +set -eu -o pipefail ${SHELLFLAGS} + test -d repo || exit 1 rsync -av --delete *.ftl repo/core/templates/ diff --git a/rspy/Makefile b/rspy/Makefile index 798a67515..20362e170 100644 --- a/rspy/Makefile +++ b/rspy/Makefile @@ -1,25 +1,36 @@ SHELL := /bin/bash + +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) ifndef OS OS := unknown endif -ifeq ($(OS),Windows_NT) - ifndef PYTHON_BIN - PYTHON_BIN := python +ifeq (${OS},Windows_NT) + ifndef PYTHON_FILE + PYTHON_FILE := $(shell which python) + USE_CYGPATH_COMMAND := case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) printf "yes";; esac; + USE_CYGPATH := $(shell ${USE_CYGPATH_COMMAND}) + + ifneq (,${USE_CYGPATH}) + PYTHON_FILE := $(shell cygpath -w "${PYTHON_FILE}") + endif endif else - ifndef PYTHON_BIN - PYTHON_BIN := python3 + ifndef PYTHON_FILE + PYTHON_FILE := $(shell which python3) endif endif -.SHELLFLAGS := -eu -o pipefail -c .DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - $(shell mkdir -p .build ../dist) OUTDIR := ../dist @@ -41,7 +52,7 @@ DEPS := .build/tools .build/vernum ../meta/buildhash \ $(wildcard $(QT_FTL_LOCALES)/*/*.ftl) \ $(shell ${FIND} ../rslib/src -name '*.rs') $(wildcard ../proto/*) \ $(shell ${FIND} ../rslib/ftl -type f) \ - $(shell ${FIND} ./src -type f) + $(shell ${FIND} ./src -type f) .build/develop: $(DEPS) touch ../proto/backend.proto @@ -50,10 +61,10 @@ DEPS := .build/tools .build/vernum ../meta/buildhash \ touch $@ build: $(DEPS) - rm -rf $(OUTDIR)/ankirspy* + rm -rf "$(OUTDIR)"/ankirspy* touch ../proto/backend.proto FTL_TEMPLATE_DIRS="$(QT_FTL_TEMPLATES)" FTL_LOCALE_DIRS="$(QT_FTL_LOCALES)" \ - maturin build -i $(shell which ${PYTHON_BIN}) -o $(OUTDIR) $(BUILDFLAGS) + maturin build -i "${PYTHON_FILE}" -o "$(OUTDIR)" $(BUILDFLAGS) check: .build/check diff --git a/run b/run index 7e984857d..22099b665 100755 --- a/run +++ b/run @@ -1,4 +1,6 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + python --version make -C $(dirname $0) run RUNFLAGS="$*" From b5a1cf3a1472502b07eac1dcb1aab59de896784f Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Sun, 22 Mar 2020 21:10:16 -0300 Subject: [PATCH 02/29] Cache rust libraries --- .github/workflows/linux_checks.yml | 41 ++++++++++++++++++++++++---- .github/workflows/windows_checks.yml | 37 +++++++++++++++++++++---- Makefile | 1 + 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index 92b7c855e..1a79cc270 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -17,11 +17,42 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 - - name: Run checks + - name: Set up ripgrep, pyaudio, gettext, rename run: | - # add requirements sudo apt update; sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic - export CARGO_TARGET_DIR=~/target - export RSPY_TARGET_DIR=~/target - make check build BUILDFLAGS="" + # Necessary for now for the cargo cache: + # https://github.com/actions/cache/issues/133#issuecomment-599102035 + - name: Fix ~/.cargo directory permissions + run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo build + uses: actions/cache@v1 + with: + path: ~/target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Configure project variables + run: | + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::~/target" + echo "::set-env name=CARGO_TARGET_DIR::~/target" + - name: Run checks + run: | + make check + - name: Run build + run: | + make build diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index e05d89404..8061092af 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -47,7 +47,7 @@ jobs: :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly .\pacman --version - .\pacman -Sv --noconfirm --overwrite='*' bash + .\pacman -Sv --noconfirm --overwrite='*' bash .\pacman -Sv --noconfirm --overwrite='*' rsync - name: Set up scoop, gettext, ripgrep shell: cmd @@ -65,12 +65,37 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo build + uses: actions/cache@v1 + with: + path: ~\target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Configure project variables + run: | + echo "::set-env name=RSPY_TARGET_DIR::~\target" + echo "::set-env name=CARGO_TARGET_DIR::~\target" - name: Run checks shell: cmd run: | echo on - set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" - set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" - set "BUILDFLAGS=" - :: set "SHELLFLAGS=-x" - make check build + make check BUILDFLAGS="" + - name: Run build + shell: cmd + run: | + echo on + make build BUILDFLAGS="" diff --git a/Makefile b/Makefile index 83325e02a..246a60c6c 100644 --- a/Makefile +++ b/Makefile @@ -181,3 +181,4 @@ push-i18n-ftl: pull-i18n .PHONY: push-i18n-po push-i18n-po: pull-i18n (cd qt/po && scripts/upload-latest-template) + From f90856625210d9e643baaa5651520b44d4886095 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Sun, 22 Mar 2020 23:34:22 -0300 Subject: [PATCH 03/29] Created .github/workflows/mac_checks.yml --- .github/workflows/linux_checks.yml | 16 ++++---- .github/workflows/mac_checks.yml | 59 ++++++++++++++++++++++++++++ .github/workflows/windows_checks.yml | 12 +++--- Makefile | 10 +++-- README.development | 2 +- qt/tools/typecheck-setup.sh | 2 +- 6 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/mac_checks.yml diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index 1a79cc270..d631d3511 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -19,7 +19,8 @@ jobs: node-version: 12 - name: Set up ripgrep, pyaudio, gettext, rename run: | - sudo apt update; sudo apt install portaudio19-dev gettext rename + sudo apt update + sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 @@ -45,14 +46,13 @@ jobs: with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - name: Configure project variables - run: | - echo "::set-env name=BUILDFLAGS::" - echo "::set-env name=RSPY_TARGET_DIR::~/target" - echo "::set-env name=CARGO_TARGET_DIR::~/target" - name: Run checks run: | - make check + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make check BUILDFLAGS="" - name: Run build run: | - make build + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make build BUILDFLAGS="" diff --git a/.github/workflows/mac_checks.yml b/.github/workflows/mac_checks.yml new file mode 100644 index 000000000..59f4cbfa6 --- /dev/null +++ b/.github/workflows/mac_checks.yml @@ -0,0 +1,59 @@ +name: Mac Tests + +# on: [push, pull_request] +# Disables Mac OS builds until this gets fixed +# https://github.com/actions/virtual-environments/issues/602 +on: + push: + branches-ignore: + - '**' + +jobs: + test: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: Arduino/actions/setup-protoc@master + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Set up brew packages + run: | + brew install portaudio protobuf gettext rename ripgrep + brew link gettext --force + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo build + uses: actions/cache@v1 + with: + path: ~/target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Run checks + run: | + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make check BUILDFLAGS="" + - name: Run build + run: | + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make build BUILDFLAGS="" diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index 8061092af..52a632804 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -43,10 +43,10 @@ jobs: .\bash pacman-key --refresh-keys .\pacman -Tv .\pacman -Syuv --overwrite='*' + .\pacman --version :: We must install bash first, otherwise we will have bash fork errors: :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly - .\pacman --version .\pacman -Sv --noconfirm --overwrite='*' bash .\pacman -Sv --noconfirm --overwrite='*' rsync - name: Set up scoop, gettext, ripgrep @@ -78,24 +78,24 @@ jobs: - name: Cache cargo build uses: actions/cache@v1 with: - path: ~\target + path: ${{ github.workspace }}\target key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - name: Configure project variables - run: | - echo "::set-env name=RSPY_TARGET_DIR::~\target" - echo "::set-env name=CARGO_TARGET_DIR::~\target" - name: Run checks shell: cmd run: | echo on + set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" make check BUILDFLAGS="" - name: Run build shell: cmd run: | echo on + set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" make build BUILDFLAGS="" diff --git a/Makefile b/Makefile index 246a60c6c..0a0a06cba 100644 --- a/Makefile +++ b/Makefile @@ -65,8 +65,13 @@ pyenv: "${PYTHON_BIN}" -m venv pyenv && \ case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) \ dos2unix "${ACTIVATE_SCRIPT}" && \ - sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$(shell pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" \ - ;; esac; \ + # https://github.com/PyO3/maturin/issues/283 \ + # Expected `python` to be a python interpreter inside a virtualenv \ + VIRTUAL_ENV="$$(pwd)" && \ + VIRTUAL_ENV="$$(cygpath -m "$${VIRTUAL_ENV}")" && \ + sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$$(pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" && \ + sed -i -- "s@export PATH@export PATH; VIRTUAL_ENV=\"$${VIRTUAL_ENV}/pyenv\";@g" "${ACTIVATE_SCRIPT}" \ + ;; esac && \ . "${ACTIVATE_SCRIPT}" && \ python --version && \ python -m pip install --upgrade pip setuptools && \ @@ -181,4 +186,3 @@ push-i18n-ftl: pull-i18n .PHONY: push-i18n-po push-i18n-po: pull-i18n (cd qt/po && scripts/upload-latest-template) - diff --git a/README.development b/README.development index c6031425b..3cee93523 100644 --- a/README.development +++ b/README.development @@ -79,7 +79,7 @@ Mac users You can use homebrew to install some dependencies: -$ brew install python mpv lame portaudio protobuf npm rustup-init gettext rename +$ brew install python mpv lame portaudio protobuf npm rustup-init gettext rename ripgrep $ brew link gettext --force diff --git a/qt/tools/typecheck-setup.sh b/qt/tools/typecheck-setup.sh index ee68cb2a2..acda0b599 100755 --- a/qt/tools/typecheck-setup.sh +++ b/qt/tools/typecheck-setup.sh @@ -22,6 +22,6 @@ case "$(uname -s)" in ;; esac -cmd="rsync -a \"${TOOLS}/stubs/PyQt5/\" \"${modDir}/\"" +cmd="rsync -a ${TOOLS}/stubs/PyQt5/ ${modDir}/" $cmd > /dev/null 2>&1 || sudo $cmd From 4ff6e35a2c83d2c9489884faa4fd8ef6fdebbdb9 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 11:50:17 -0300 Subject: [PATCH 04/29] Cache pacman package manager and upload python wheels --- .github/workflows/linux_checks.yml | 15 +++++++++++++-- .github/workflows/mac_checks.yml | 11 +++++++++++ .github/workflows/windows_checks.yml | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index d631d3511..0f489ea71 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -22,10 +22,16 @@ jobs: sudo apt update sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - - name: Fix ~/.cargo directory permissions - run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ + - name: Fix ~/.cache permissions + run: | + sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - name: Cache cargo registry uses: actions/cache@v1 with: @@ -56,3 +62,8 @@ jobs: export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target make build BUILDFLAGS="" + - name: Upload python wheels + uses: actions/upload-artifact@v1 + with: + name: Linux python wheels + path: dist diff --git a/.github/workflows/mac_checks.yml b/.github/workflows/mac_checks.yml index 59f4cbfa6..5ffb83e06 100644 --- a/.github/workflows/mac_checks.yml +++ b/.github/workflows/mac_checks.yml @@ -27,6 +27,12 @@ jobs: run: | brew install portaudio protobuf gettext rename ripgrep brew link gettext --force + - name: Cache pyenv + id: cache-pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- - name: Cache cargo registry uses: actions/cache@v1 with: @@ -57,3 +63,8 @@ jobs: export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target make build BUILDFLAGS="" + - name: Upload python wheels + uses: actions/upload-artifact@v1 + with: + name: Mac OS python wheels + path: dist diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index 52a632804..dd82de5f8 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -22,7 +22,20 @@ jobs: echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" + - name: Cache pyenv + id: cache-pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + - name: Cache pacman + id: cache-pacman + uses: actions/cache@v1 + with: + path: C:\Program Files\Git + key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }} - name: Set up pacman, pyaudio, rsync, rename + if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd run: | echo on @@ -99,3 +112,8 @@ jobs: set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" make build BUILDFLAGS="" + - name: Upload python wheels + uses: actions/upload-artifact@v1 + with: + name: Windows python wheels + path: dist From 8e43016248ff8d2882b55a6ae28f1a5cf17dfadb Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 13:59:34 -0300 Subject: [PATCH 05/29] Renamed the mac_checks.yml until it get fixed --- .github/workflows/{mac_checks.yml => mac_checks.yml.off} | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) rename .github/workflows/{mac_checks.yml => mac_checks.yml.off} (94%) diff --git a/.github/workflows/mac_checks.yml b/.github/workflows/mac_checks.yml.off similarity index 94% rename from .github/workflows/mac_checks.yml rename to .github/workflows/mac_checks.yml.off index 5ffb83e06..0055144fe 100644 --- a/.github/workflows/mac_checks.yml +++ b/.github/workflows/mac_checks.yml.off @@ -1,12 +1,8 @@ name: Mac Tests -# on: [push, pull_request] -# Disables Mac OS builds until this gets fixed +# Disabled until Mac OS builds until this gets fixed: # https://github.com/actions/virtual-environments/issues/602 -on: - push: - branches-ignore: - - '**' +on: [push, pull_request] jobs: test: From 19dab43aae8d0b38ac1251cfab6163ddd6c54abd Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 14:13:27 -0300 Subject: [PATCH 06/29] Moved the cache actions to the top of the file --- .github/workflows/linux_checks.yml | 42 ++++++++++++------------- .github/workflows/mac_checks.yml.off | 41 ++++++++++++------------ .github/workflows/windows_checks.yml | 47 ++++++++++++++-------------- 3 files changed, 64 insertions(+), 66 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index 0f489ea71..e93163e39 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -7,26 +7,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: Arduino/actions/setup-protoc@master - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Set up ripgrep, pyaudio, gettext, rename - run: | - sudo apt update - sudo apt install portaudio19-dev gettext rename - sudo snap install ripgrep --classic - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - name: Fix ~/.cache permissions @@ -52,6 +32,26 @@ jobs: with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: Arduino/actions/setup-protoc@master + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Set up ripgrep, pyaudio, gettext, rename + run: | + sudo apt update + sudo apt install portaudio19-dev gettext rename + sudo snap install ripgrep --classic - name: Run checks run: | export RSPY_TARGET_DIR=~/target @@ -65,5 +65,5 @@ jobs: - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: Linux python wheels + name: linux_python_wheels path: dist diff --git a/.github/workflows/mac_checks.yml.off b/.github/workflows/mac_checks.yml.off index 0055144fe..b5d324475 100644 --- a/.github/workflows/mac_checks.yml.off +++ b/.github/workflows/mac_checks.yml.off @@ -9,26 +9,6 @@ jobs: runs-on: macOS-latest steps: - uses: actions/checkout@v2 - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: Arduino/actions/setup-protoc@master - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Set up brew packages - run: | - brew install portaudio protobuf gettext rename ripgrep - brew link gettext --force - - name: Cache pyenv - id: cache-pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- - name: Cache cargo registry uses: actions/cache@v1 with: @@ -49,6 +29,25 @@ jobs: with: path: ~/Library/Caches/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: Arduino/actions/setup-protoc@master + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Set up brew packages + run: | + brew install portaudio protobuf gettext rename ripgrep + brew link gettext --force - name: Run checks run: | export RSPY_TARGET_DIR=~/target @@ -62,5 +61,5 @@ jobs: - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: Mac OS python wheels + name: mac_os_python_wheels path: dist diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index dd82de5f8..0eac083c3 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -23,14 +23,33 @@ jobs: echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" - name: Cache pyenv - id: cache-pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- - - name: Cache pacman - id: cache-pacman + - name: Cache cargo registry uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + - name: Cache cargo build + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - name: Cache pacman + uses: actions/cache@v1 + id: cache-pacman with: path: C:\Program Files\Git key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }} @@ -78,26 +97,6 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} - - name: Cache cargo build - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - name: Run checks shell: cmd run: | @@ -115,5 +114,5 @@ jobs: - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: Windows python wheels + name: windows_python_wheels path: dist From cfbd7ba1081678589e0113af80f9755cd7014e17 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 16:13:00 -0300 Subject: [PATCH 07/29] Added the develop rule to run first and install things before running tests and the build to properly measure the total time of each stage. --- .github/workflows/linux_checks.yml | 15 ++++++++++----- .github/workflows/mac_checks.yml.off | 15 ++++++++++----- .github/workflows/windows_checks.yml | 17 ++++++++++++----- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index e93163e39..6e6a685da 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -22,11 +22,11 @@ jobs: with: path: ~/.cargo/git key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} - - name: Cache cargo build + - name: Cache cargo target uses: actions/cache@v1 with: path: ~/target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} - name: Cache pip wheels uses: actions/cache@v1 with: @@ -52,18 +52,23 @@ jobs: sudo apt update sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic - - name: Run checks + - name: Run develop run: | export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target - make check BUILDFLAGS="" + make develop BUILDFLAGS="" - name: Run build run: | export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target make build BUILDFLAGS="" + - name: Run check + run: | + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make check BUILDFLAGS="" - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: linux_python_wheels + name: anki_linux_python_wheels path: dist diff --git a/.github/workflows/mac_checks.yml.off b/.github/workflows/mac_checks.yml.off index b5d324475..a50b918cd 100644 --- a/.github/workflows/mac_checks.yml.off +++ b/.github/workflows/mac_checks.yml.off @@ -19,11 +19,11 @@ jobs: with: path: ~/.cargo/git key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} - - name: Cache cargo build + - name: Cache cargo target uses: actions/cache@v1 with: path: ~/target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} - name: Cache pip wheels uses: actions/cache@v1 with: @@ -48,18 +48,23 @@ jobs: run: | brew install portaudio protobuf gettext rename ripgrep brew link gettext --force - - name: Run checks + - name: Run develop run: | export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target - make check BUILDFLAGS="" + make develop BUILDFLAGS="" - name: Run build run: | export RSPY_TARGET_DIR=~/target export CARGO_TARGET_DIR=~/target make build BUILDFLAGS="" + - name: Run check + run: | + export RSPY_TARGET_DIR=~/target + export CARGO_TARGET_DIR=~/target + make check BUILDFLAGS="" - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: mac_os_python_wheels + name: anki_mac_os_python_wheels path: dist diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index 0eac083c3..68917a3f5 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -37,11 +37,11 @@ jobs: with: path: C:\Rust\.cargo\git key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} - - name: Cache cargo build + - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} - name: Cache pip wheels uses: actions/cache@v1 with: @@ -97,13 +97,13 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 - - name: Run checks + - name: Run develop shell: cmd run: | echo on set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" - make check BUILDFLAGS="" + make develop BUILDFLAGS="" - name: Run build shell: cmd run: | @@ -111,8 +111,15 @@ jobs: set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" make build BUILDFLAGS="" + - name: Run check + shell: cmd + run: | + echo on + set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" + make check BUILDFLAGS="" - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: windows_python_wheels + name: anki_windows_python_wheels path: dist From 508f8df47f5fa9419f8902c00a74ad6e20b1a64e Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 19:55:50 -0300 Subject: [PATCH 08/29] Fixed pyaudio and rename not being setup when pacman cache is used --- .github/workflows/windows_checks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index 68917a3f5..a67d93bd8 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -58,11 +58,9 @@ jobs: shell: cmd run: | echo on - curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% - curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" @@ -81,6 +79,12 @@ jobs: :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly .\pacman -Sv --noconfirm --overwrite='*' bash .\pacman -Sv --noconfirm --overwrite='*' rsync + - name: Set up pyaudio, rename + shell: cmd + run: | + echo on + curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename - name: Set up scoop, gettext, ripgrep shell: cmd run: | From 131aee1a825efdfd38d1b7b12684ef832a9e9dcd Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 20:12:14 -0300 Subject: [PATCH 09/29] Appended a number by default on every cache key to easily force its regeneration by hand. --- .github/workflows/linux_checks.yml | 10 +++++----- .github/workflows/mac_checks.yml.off | 10 +++++----- .github/workflows/windows_checks.yml | 12 ++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index 6e6a685da..c792b0ff3 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -16,27 +16,27 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo target uses: actions/cache@v1 with: path: ~/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 - name: Set up python uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac_checks.yml.off b/.github/workflows/mac_checks.yml.off index a50b918cd..02d285d9a 100644 --- a/.github/workflows/mac_checks.yml.off +++ b/.github/workflows/mac_checks.yml.off @@ -13,27 +13,27 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo target uses: actions/cache@v1 with: path: ~/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 - name: Set up python uses: actions/setup-python@v1 with: diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index a67d93bd8..6d03dd484 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -26,33 +26,33 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 - name: Cache cargo registry uses: actions/cache@v1 with: path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo index uses: actions/cache@v1 with: path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }} + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 - name: Cache pacman uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }} + key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-1 - name: Set up pacman, pyaudio, rsync, rename if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd From 136cd4da7cd7cce49e6e865a336860dd6cfedd38 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Mon, 23 Mar 2020 23:39:05 -0300 Subject: [PATCH 10/29] Fixed not all rust libraries being cached --- .github/workflows/linux_checks.yml | 42 +++++++++++++++++---------- .github/workflows/mac_checks.yml.off | 43 +++++++++++++++++++--------- .github/workflows/windows_checks.yml | 40 ++++++++++++++++---------- Makefile | 2 +- 4 files changed, 83 insertions(+), 44 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index c792b0ff3..328cecd58 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -7,6 +7,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" + echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - name: Fix ~/.cache permissions @@ -16,27 +21,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo target uses: actions/cache@v1 with: - path: ~/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 + path: ${{ github.workspace }}/target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rslib/target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rspy/target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- - name: Set up python uses: actions/setup-python@v1 with: @@ -54,19 +69,16 @@ jobs: sudo snap install ripgrep --classic - name: Run develop run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target - make develop BUILDFLAGS="" + # export SHELLFLAGS=-x + make develop - name: Run build run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target - make build BUILDFLAGS="" + # export SHELLFLAGS=-x + make build - name: Run check run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target - make check BUILDFLAGS="" + # export SHELLFLAGS=-x + make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: diff --git a/.github/workflows/mac_checks.yml.off b/.github/workflows/mac_checks.yml.off index 02d285d9a..5e2f86d73 100644 --- a/.github/workflows/mac_checks.yml.off +++ b/.github/workflows/mac_checks.yml.off @@ -9,31 +9,51 @@ jobs: runs-on: macOS-latest steps: - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" + echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # Necessary for now for the cargo cache: + # https://github.com/actions/cache/issues/133#issuecomment-599102035 + - name: Fix ~/.cache permissions + run: | + sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - name: Cache cargo registry uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo target uses: actions/cache@v1 - with: - path: ~/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 + with: + path: ${{ github.workspace }}/target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rslib/target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rspy/target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- - name: Set up python uses: actions/setup-python@v1 with: @@ -50,18 +70,15 @@ jobs: brew link gettext --force - name: Run develop run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target + # export SHELLFLAGS=-x make develop BUILDFLAGS="" - name: Run build run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target + # export SHELLFLAGS=-x make build BUILDFLAGS="" - name: Run check run: | - export RSPY_TARGET_DIR=~/target - export CARGO_TARGET_DIR=~/target + # export SHELLFLAGS=-x make check BUILDFLAGS="" - name: Upload python wheels uses: actions/upload-artifact@v1 diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index 6d03dd484..c02fdc875 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -22,37 +22,50 @@ jobs: echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" + echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- - name: Cache cargo registry uses: actions/cache@v1 with: path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo index uses: actions/cache@v1 with: path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-1 + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\rslib\target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\rspy\target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-1 + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- - name: Cache pacman uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-1 + key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-3- - name: Set up pacman, pyaudio, rsync, rename if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd @@ -105,23 +118,20 @@ jobs: shell: cmd run: | echo on - set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" - set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" - make develop BUILDFLAGS="" + :: set "SHELLFLAGS=-x" + make develop - name: Run build shell: cmd run: | echo on - set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" - set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" - make build BUILDFLAGS="" + :: set "SHELLFLAGS=-x" + make build - name: Run check shell: cmd run: | echo on - set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" - set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" - make check BUILDFLAGS="" + :: set "SHELLFLAGS=-x" + make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: diff --git a/Makefile b/Makefile index 0a0a06cba..c40bdf2e5 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ pyenv: VIRTUAL_ENV="$$(pwd)" && \ VIRTUAL_ENV="$$(cygpath -m "$${VIRTUAL_ENV}")" && \ sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$$(pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" && \ - sed -i -- "s@export PATH@export PATH; VIRTUAL_ENV=\"$${VIRTUAL_ENV}/pyenv\";@g" "${ACTIVATE_SCRIPT}" \ + sed -i -- "s@export PATH@export PATH; VIRTUAL_ENV=\"$${VIRTUAL_ENV}/pyenv\";@g" "${ACTIVATE_SCRIPT}"; \ ;; esac && \ . "${ACTIVATE_SCRIPT}" && \ python --version && \ From 18aec3c713ff7d812d3d76489c232c3759ad8850 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 13:51:59 -0300 Subject: [PATCH 11/29] Re-enabled the mac_checks.yml after fixing the rate limit issue --- .github/workflows/linux_checks.yml | 4 +++- .../{mac_checks.yml.off => mac_checks.yml} | 16 +++++++++------- .github/workflows/windows_checks.yml | 9 ++++++--- Makefile | 9 +++++---- 4 files changed, 23 insertions(+), 15 deletions(-) rename .github/workflows/{mac_checks.yml.off => mac_checks.yml} (91%) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index 328cecd58..e3943c5b6 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -57,7 +57,9 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: Arduino/actions/setup-protoc@master + uses: arduino/setup-protoc@v1.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node uses: actions/setup-node@v1 with: diff --git a/.github/workflows/mac_checks.yml.off b/.github/workflows/mac_checks.yml similarity index 91% rename from .github/workflows/mac_checks.yml.off rename to .github/workflows/mac_checks.yml index 5e2f86d73..c767a8ea3 100644 --- a/.github/workflows/mac_checks.yml.off +++ b/.github/workflows/mac_checks.yml @@ -59,7 +59,9 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: Arduino/actions/setup-protoc@master + uses: arduino/setup-protoc@v1.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node uses: actions/setup-node@v1 with: @@ -70,16 +72,16 @@ jobs: brew link gettext --force - name: Run develop run: | - # export SHELLFLAGS=-x - make develop BUILDFLAGS="" + export SHELLFLAGS=-x + make develop - name: Run build run: | - # export SHELLFLAGS=-x - make build BUILDFLAGS="" + export SHELLFLAGS=-x + make build - name: Run check run: | - # export SHELLFLAGS=-x - make check BUILDFLAGS="" + export SHELLFLAGS=-x + make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index c02fdc875..ff66a91a1 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -66,7 +66,7 @@ jobs: with: path: C:\Program Files\Git key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-3- - - name: Set up pacman, pyaudio, rsync, rename + - name: Set up pacman, rsync if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd run: | @@ -85,7 +85,8 @@ jobs: .\bash pacman-key --populate msys2 .\bash pacman-key --refresh-keys .\pacman -Tv - .\pacman -Syuv --overwrite='*' + .\pacman -Syyuuv --overwrite='*' + .\pacman -Syuuv --overwrite='*' .\pacman --version :: We must install bash first, otherwise we will have bash fork errors: :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true @@ -109,7 +110,9 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: Arduino/actions/setup-protoc@master + uses: arduino/setup-protoc@v1.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node uses: actions/setup-node@v1 with: diff --git a/Makefile b/Makefile index c40bdf2e5..d9c1d921e 100644 --- a/Makefile +++ b/Makefile @@ -60,13 +60,12 @@ all: run # - modern pip required for wheel # - add qt if missing pyenv: - "${PYTHON_BIN}" -m ensurepip && \ + # https://github.com/PyO3/maturin/issues/283 + # Expected `python` to be a python interpreter inside a virtualenv "${PYTHON_BIN}" -m pip install virtualenv && \ "${PYTHON_BIN}" -m venv pyenv && \ case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) \ dos2unix "${ACTIVATE_SCRIPT}" && \ - # https://github.com/PyO3/maturin/issues/283 \ - # Expected `python` to be a python interpreter inside a virtualenv \ VIRTUAL_ENV="$$(pwd)" && \ VIRTUAL_ENV="$$(cygpath -m "$${VIRTUAL_ENV}")" && \ sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$$(pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" && \ @@ -76,7 +75,9 @@ pyenv: python --version && \ python -m pip install --upgrade pip setuptools && \ ${ANKI_EXTRA_PIP} && \ - python -c 'import PyQt5' 2>/dev/null || python -m pip install -r qt/requirements.qt + if ! python -c 'import PyQt5' 2>/dev/null; then \ + python -m pip install -r qt/requirements.qt; \ + fi; # update build hash .PHONY: buildhash From bcda7ecc2909fc6f5a6237c64582ad5848973c25 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 16:29:27 -0300 Subject: [PATCH 12/29] Fixed bash call by bash using other bashes on the system PATH --- .github/workflows/linux_checks.yml | 30 +++++--------- .github/workflows/mac_checks.yml | 32 +++++---------- .github/workflows/windows_checks.yml | 61 ++++++++++++---------------- 3 files changed, 47 insertions(+), 76 deletions(-) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/linux_checks.yml index e3943c5b6..7fc42352f 100644 --- a/.github/workflows/linux_checks.yml +++ b/.github/workflows/linux_checks.yml @@ -9,6 +9,7 @@ jobs: - uses: actions/checkout@v2 - name: Configure environment variables run: | + # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" @@ -21,37 +22,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- - name: Set up python uses: actions/setup-python@v1 with: @@ -69,18 +70,9 @@ jobs: sudo apt update sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic - - name: Run develop - run: | - # export SHELLFLAGS=-x - make develop - - name: Run build - run: | - # export SHELLFLAGS=-x - make build - - name: Run check - run: | - # export SHELLFLAGS=-x - make check + - run: make check + - run: make develop + - run: make build - name: Upload python wheels uses: actions/upload-artifact@v1 with: diff --git a/.github/workflows/mac_checks.yml b/.github/workflows/mac_checks.yml index c767a8ea3..6d91a8005 100644 --- a/.github/workflows/mac_checks.yml +++ b/.github/workflows/mac_checks.yml @@ -1,7 +1,5 @@ name: Mac Tests -# Disabled until Mac OS builds until this gets fixed: -# https://github.com/actions/virtual-environments/issues/602 on: [push, pull_request] jobs: @@ -11,6 +9,7 @@ jobs: - uses: actions/checkout@v2 - name: Configure environment variables run: | + # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" @@ -23,37 +22,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- - name: Set up python uses: actions/setup-python@v1 with: @@ -70,18 +69,9 @@ jobs: run: | brew install portaudio protobuf gettext rename ripgrep brew link gettext --force - - name: Run develop - run: | - export SHELLFLAGS=-x - make develop - - name: Run build - run: | - export SHELLFLAGS=-x - make build - - name: Run check - run: | - export SHELLFLAGS=-x - make check + - run: make check + - run: make develop + - run: make build - name: Upload python wheels uses: actions/upload-artifact@v1 with: diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml index ff66a91a1..1d296114f 100644 --- a/.github/workflows/windows_checks.yml +++ b/.github/workflows/windows_checks.yml @@ -10,6 +10,7 @@ jobs: - uses: actions/checkout@v2 - name: Configure environment variables run: | + # echo "::set-env name=SHELLFLAGS::-x" $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") @@ -29,43 +30,43 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- - name: Cache cargo registry uses: actions/cache@v1 with: path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo index uses: actions/cache@v1 with: path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}\rslib\target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}\rspy\target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-3- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-3- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- - name: Cache pacman uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-3- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-4- - name: Set up pacman, rsync if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd @@ -80,19 +81,22 @@ jobs: 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" :: Manually Install Pacman Binaries :: https://github.com/Alexpux/MSYS2-pacman/issues/50 - cd /d "%programfiles%\Git\usr\bin" - .\bash pacman-key --init - .\bash pacman-key --populate msys2 - .\bash pacman-key --refresh-keys - .\pacman -Tv - .\pacman -Syyuuv --overwrite='*' - .\pacman -Syuuv --overwrite='*' - .\pacman --version + set "PATH=%programfiles%\Git\usr\bin" + bash pacman-key --init + bash pacman-key --populate msys2 + bash pacman-key --refresh-keys + pacman -Tv + pacman -Syyuuv --overwrite='*' + pacman -Syuuv --overwrite='*' + pacman --version :: We must install bash first, otherwise we will have bash fork errors: :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly - .\pacman -Sv --noconfirm --overwrite='*' bash - .\pacman -Sv --noconfirm --overwrite='*' rsync + pacman -Sv --noconfirm --overwrite='*' bash + pacman -Sv --noconfirm --overwrite='*' rsync + :: clean all packages to decrease image size + pacman -Sccv --noconfirm + pacman -Qsv --noconfirm - name: Set up pyaudio, rename shell: cmd run: | @@ -117,24 +121,9 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 - - name: Run develop - shell: cmd - run: | - echo on - :: set "SHELLFLAGS=-x" - make develop - - name: Run build - shell: cmd - run: | - echo on - :: set "SHELLFLAGS=-x" - make build - - name: Run check - shell: cmd - run: | - echo on - :: set "SHELLFLAGS=-x" - make check + - run: make check + - run: make develop + - run: make build - name: Upload python wheels uses: actions/upload-artifact@v1 with: From 716989135cd0bf3ad0d24d672b97737d32a5fbde Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 21:58:39 -0300 Subject: [PATCH 13/29] Renamed linux_checks.yml to checks.yml --- .github/workflows/{linux_checks.yml => checks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{linux_checks.yml => checks.yml} (100%) diff --git a/.github/workflows/linux_checks.yml b/.github/workflows/checks.yml similarity index 100% rename from .github/workflows/linux_checks.yml rename to .github/workflows/checks.yml From d1337badab268882fa643978420bd3b1e429d730 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 21:59:20 -0300 Subject: [PATCH 14/29] Merged mac_checks.yml and windows_checks.yml into checks.yml --- .github/workflows/checks.yml | 208 ++++++++++++++++++++++++++- .github/workflows/mac_checks.yml | 79 ---------- .github/workflows/windows_checks.yml | 131 ----------------- 3 files changed, 206 insertions(+), 212 deletions(-) delete mode 100644 .github/workflows/mac_checks.yml delete mode 100644 .github/workflows/windows_checks.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7fc42352f..bb4381244 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,9 +1,9 @@ -name: Linux Tests +name: Checks on: [push, pull_request] jobs: - test: + linux_tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -78,3 +78,207 @@ jobs: with: name: anki_linux_python_wheels path: dist + + + mac_os_tests: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + # echo "::set-env name=SHELLFLAGS::-x" + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" + echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # Necessary for now for the cargo cache: + # https://github.com/actions/cache/issues/133#issuecomment-599102035 + - name: Fix ~/.cache permissions + run: | + sudo chown -R $(whoami):$(id -ng) ~/.cargo/ + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo target + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rslib/target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/rspy/target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}/pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: arduino/setup-protoc@v1.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Set up brew packages + run: | + brew install portaudio protobuf gettext rename ripgrep + brew link gettext --force + - run: make check + - run: make develop + - run: make build + - name: Upload python wheels + uses: actions/upload-artifact@v1 + with: + name: anki_mac_os_python_wheels + path: dist + + + windows_tests: + runs-on: windows-latest + steps: + - run: git config --global core.autocrlf false + - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + # echo "::set-env name=SHELLFLAGS::-x" + $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") + $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") + $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") + echo "::set-env name=pacmanbin::pacman-5.1.1-3-x86_64.pkg.tar.xz" + echo "::set-env name=pacmanmirror::pacman-mirrors-20200307-1-any.pkg.tar.xz" + echo "::set-env name=pacmankeys::msys2-keyring-r9.397a52e-1-any.pkg.tar.xz" + echo "::set-env name=pyaudio::$pyaudio" + echo "::set-env name=PATH::$new_path" + echo "::set-env name=RUST_BACKTRACE::full" + echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" + echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" + echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" + echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" + echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: C:\Rust\.cargo\git + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo target + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\rslib\target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}\rspy\target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- + - name: Cache pip wheels + uses: actions/cache@v1 + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- + - name: Cache pacman + uses: actions/cache@v1 + id: cache-pacman + with: + path: C:\Program Files\Git + key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-4- + - name: Set up pacman, rsync + if: steps.cache-pacman.outputs.cache-hit != 'true' + shell: cmd + run: | + echo on + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% + :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz + 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + :: Manually Install Pacman Binaries + :: https://github.com/Alexpux/MSYS2-pacman/issues/50 + set "PATH=%programfiles%\Git\usr\bin" + bash pacman-key --init + bash pacman-key --populate msys2 + bash pacman-key --refresh-keys + pacman -Tv + pacman -Syyuuv --overwrite='*' + pacman -Syuuv --overwrite='*' + pacman --version + :: We must install bash first, otherwise we will have bash fork errors: + :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true + :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly + pacman -Sv --noconfirm --overwrite='*' bash + pacman -Sv --noconfirm --overwrite='*' rsync + :: clean all packages to decrease image size + pacman -Sccv --noconfirm + pacman -Qsv --noconfirm + - name: Set up pyaudio, rename + shell: cmd + run: | + echo on + curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename + - name: Set up scoop, gettext, ripgrep + shell: cmd + run: | + echo on + powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" + scoop install -g gettext ripgrep + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: arduino/setup-protoc@v1.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - run: make check + - run: make develop + - run: make build + - name: Upload python wheels + uses: actions/upload-artifact@v1 + with: + name: anki_windows_python_wheels + path: dist diff --git a/.github/workflows/mac_checks.yml b/.github/workflows/mac_checks.yml deleted file mode 100644 index 6d91a8005..000000000 --- a/.github/workflows/mac_checks.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Mac Tests - -on: [push, pull_request] - -jobs: - test: - runs-on: macOS-latest - steps: - - uses: actions/checkout@v2 - - name: Configure environment variables - run: | - # echo "::set-env name=SHELLFLAGS::-x" - echo "::set-env name=BUILDFLAGS::" - echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" - echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" - # Necessary for now for the cargo cache: - # https://github.com/actions/cache/issues/133#issuecomment-599102035 - - name: Fix ~/.cache permissions - run: | - sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo target - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo rslib - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo rspy - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: arduino/setup-protoc@v1.1.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Set up brew packages - run: | - brew install portaudio protobuf gettext rename ripgrep - brew link gettext --force - - run: make check - - run: make develop - - run: make build - - name: Upload python wheels - uses: actions/upload-artifact@v1 - with: - name: anki_mac_os_python_wheels - path: dist diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml deleted file mode 100644 index 1d296114f..000000000 --- a/.github/workflows/windows_checks.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: Windows Tests - -on: [push, pull_request] - -jobs: - test: - runs-on: windows-latest - steps: - - run: git config --global core.autocrlf false - - uses: actions/checkout@v2 - - name: Configure environment variables - run: | - # echo "::set-env name=SHELLFLAGS::-x" - $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") - $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") - $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") - echo "::set-env name=pacmanbin::pacman-5.1.1-3-x86_64.pkg.tar.xz" - echo "::set-env name=pacmanmirror::pacman-mirrors-20200307-1-any.pkg.tar.xz" - echo "::set-env name=pacmankeys::msys2-keyring-r9.397a52e-1-any.pkg.tar.xz" - echo "::set-env name=pyaudio::$pyaudio" - echo "::set-env name=PATH::$new_path" - echo "::set-env name=RUST_BACKTRACE::full" - echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" - echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" - echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" - echo "::set-env name=BUILDFLAGS::" - echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" - echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo target - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo rslib - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\rslib\target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache cargo rspy - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\rspy\target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- - - name: Cache pacman - uses: actions/cache@v1 - id: cache-pacman - with: - path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-4- - - name: Set up pacman, rsync - if: steps.cache-pacman.outputs.cache-hit != 'true' - shell: cmd - run: | - echo on - curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% - curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% - curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% - :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz - 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" - 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" - 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" - :: Manually Install Pacman Binaries - :: https://github.com/Alexpux/MSYS2-pacman/issues/50 - set "PATH=%programfiles%\Git\usr\bin" - bash pacman-key --init - bash pacman-key --populate msys2 - bash pacman-key --refresh-keys - pacman -Tv - pacman -Syyuuv --overwrite='*' - pacman -Syuuv --overwrite='*' - pacman --version - :: We must install bash first, otherwise we will have bash fork errors: - :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true - :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly - pacman -Sv --noconfirm --overwrite='*' bash - pacman -Sv --noconfirm --overwrite='*' rsync - :: clean all packages to decrease image size - pacman -Sccv --noconfirm - pacman -Qsv --noconfirm - - name: Set up pyaudio, rename - shell: cmd - run: | - echo on - curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% - curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename - - name: Set up scoop, gettext, ripgrep - shell: cmd - run: | - echo on - powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" - scoop install -g gettext ripgrep - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: arduino/setup-protoc@v1.1.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - run: make check - - run: make develop - - run: make build - - name: Upload python wheels - uses: actions/upload-artifact@v1 - with: - name: anki_windows_python_wheels - path: dist From a0971628a6e20a2ee6bd686999d8751aad10c903 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 21:51:41 -0300 Subject: [PATCH 15/29] Set to run first develop build and check rules --- .github/workflows/checks.yml | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bb4381244..65bd73cf2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,37 +22,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- - name: Set up python uses: actions/setup-python@v1 with: @@ -70,9 +70,9 @@ jobs: sudo apt update sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic - - run: make check - run: make develop - run: make build + - run: make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: @@ -99,37 +99,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- - name: Set up python uses: actions/setup-python@v1 with: @@ -146,9 +146,9 @@ jobs: run: | brew install portaudio protobuf gettext rename ripgrep brew link gettext --force - - run: make check - run: make develop - run: make build + - run: make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: @@ -183,43 +183,43 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- - name: Cache cargo registry uses: actions/cache@v1 with: path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo index uses: actions/cache@v1 with: path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}\rslib\target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}\rspy\target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-4- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-4- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- - name: Cache pacman uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/windows_checks.yml') }}-4- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-5- - name: Set up pacman, rsync if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd @@ -274,9 +274,9 @@ jobs: uses: actions/setup-node@v1 with: node-version: 12 - - run: make check - run: make develop - run: make build + - run: make check - name: Upload python wheels uses: actions/upload-artifact@v1 with: From 206e167f85c63b4d00e5ccd83554cf128c22c0a0 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Tue, 24 Mar 2020 23:51:44 -0300 Subject: [PATCH 16/29] Set curl to run silently, added missing double quotes and removed trailing backslash on checks.yml --- .github/workflows/checks.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 65bd73cf2..fa51eea50 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -174,7 +174,7 @@ jobs: echo "::set-env name=PATH::$new_path" echo "::set-env name=RUST_BACKTRACE::full" echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" - echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" + echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" diff --git a/Makefile b/Makefile index d9c1d921e..7f80745a8 100644 --- a/Makefile +++ b/Makefile @@ -166,11 +166,11 @@ fix: .PHONY: add-buildhash add-buildhash: @if [[ ! -f rename ]]; then \ - curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename; \ + curl --silent -LO https://raw.githubusercontent.com/subogero/rename/master/rename; \ fi && \ ver="$$(cat meta/version)" && \ hash="$$(cat meta/buildhash)" && \ - ${RENAME_BIN} "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-$$ver-* + ${RENAME_BIN} "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-"$${ver}"-* .PHONY: pull-i18n From eab11622fd14c20f92a3978143c01db924267f39 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 01:38:37 -0300 Subject: [PATCH 17/29] Re-enabled the Makefile set -eo commands because by default Mac OS uses an old version of make which does not support the .SHELLFLAGS feature. --- Makefile | 73 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 7f80745a8..8360c87ac 100644 --- a/Makefile +++ b/Makefile @@ -62,19 +62,20 @@ all: run pyenv: # https://github.com/PyO3/maturin/issues/283 # Expected `python` to be a python interpreter inside a virtualenv - "${PYTHON_BIN}" -m pip install virtualenv && \ - "${PYTHON_BIN}" -m venv pyenv && \ + set -eu -o pipefail ${SHELLFLAGS}; \ + "${PYTHON_BIN}" -m pip install virtualenv; \ + "${PYTHON_BIN}" -m venv pyenv; \ case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) \ - dos2unix "${ACTIVATE_SCRIPT}" && \ - VIRTUAL_ENV="$$(pwd)" && \ - VIRTUAL_ENV="$$(cygpath -m "$${VIRTUAL_ENV}")" && \ - sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$$(pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" && \ + dos2unix "${ACTIVATE_SCRIPT}"; \ + VIRTUAL_ENV="$$(pwd)"; \ + VIRTUAL_ENV="$$(cygpath -m "$${VIRTUAL_ENV}")"; \ + sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$$(pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}"; \ sed -i -- "s@export PATH@export PATH; VIRTUAL_ENV=\"$${VIRTUAL_ENV}/pyenv\";@g" "${ACTIVATE_SCRIPT}"; \ - ;; esac && \ - . "${ACTIVATE_SCRIPT}" && \ - python --version && \ - python -m pip install --upgrade pip setuptools && \ - ${ANKI_EXTRA_PIP} && \ + ;; esac; \ + . "${ACTIVATE_SCRIPT}"; \ + python --version; \ + python -m pip install --upgrade pip setuptools; \ + ${ANKI_EXTRA_PIP}; \ if ! python -c 'import PyQt5' 2>/dev/null; then \ python -m pip install -r qt/requirements.qt; \ fi; @@ -82,23 +83,26 @@ pyenv: # update build hash .PHONY: buildhash buildhash: - @oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true) && \ - newhash=$$(git rev-parse --short=8 HEAD || echo dev) && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true); \ + newhash=$$(git rev-parse --short=8 HEAD || echo dev); \ if [ "$$oldhash" != "$$newhash" ]; then \ echo $$newhash > meta/buildhash; \ fi .PHONY: develop develop: pyenv buildhash prepare - @. "${ACTIVATE_SCRIPT}" && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ for dir in $(DEVEL); do \ $(SUBMAKE) -C $$dir develop DEVFLAGS="$(DEVFLAGS)"; \ done .PHONY: run run: develop - @. "${ACTIVATE_SCRIPT}" && \ - echo "Starting Anki..." && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ + echo "Starting Anki..."; \ python qt/runanki $(RUNFLAGS) .PHONY: prepare @@ -118,23 +122,27 @@ build: clean-dist build-rspy build-pylib build-qt add-buildhash .PHONY: build-rspy build-rspy: pyenv buildhash - @. "${ACTIVATE_SCRIPT}" && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ $(SUBMAKE) -C rspy build BUILDFLAGS="$(BUILDFLAGS)" .PHONY: build-pylib build-pylib: - @. "${ACTIVATE_SCRIPT}" && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ $(SUBMAKE) -C pylib build .PHONY: build-qt build-qt: - @. "${ACTIVATE_SCRIPT}" && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ $(SUBMAKE) -C qt build .PHONY: clean clean: clean-dist - @for dir in $(DEVEL); do \ - $(SUBMAKE) -C $$dir clean; \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + for dir in $(DEVEL); do \ + $(SUBMAKE) -C $$dir clean; \ done .PHONY: clean-dist @@ -143,13 +151,14 @@ clean-dist: .PHONY: check check: pyenv buildhash prepare - @.github/scripts/trailing-newlines.sh && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + .github/scripts/trailing-newlines.sh; \ for dir in $(CHECKABLE_RS); do \ $(SUBMAKE) -C $$dir check; \ - done && \ - . "${ACTIVATE_SCRIPT}" && \ - $(SUBMAKE) -C rspy develop && \ - $(SUBMAKE) -C pylib develop && \ + done; \ + . "${ACTIVATE_SCRIPT}"; \ + $(SUBMAKE) -C rspy develop; \ + $(SUBMAKE) -C pylib develop; \ for dir in $(CHECKABLE_PY); do \ $(SUBMAKE) -C $$dir check; \ done; @@ -158,18 +167,20 @@ check: pyenv buildhash prepare .PHONY: fix fix: - @. "${ACTIVATE_SCRIPT}" && \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + . "${ACTIVATE_SCRIPT}"; \ for dir in $(CHECKABLE_RS) $(CHECKABLE_PY); do \ $(SUBMAKE) -C $$dir fix; \ done; \ .PHONY: add-buildhash add-buildhash: - @if [[ ! -f rename ]]; then \ + @set -eu -o pipefail ${SHELLFLAGS}; \ + if [[ ! -f rename ]]; then \ curl --silent -LO https://raw.githubusercontent.com/subogero/rename/master/rename; \ - fi && \ - ver="$$(cat meta/version)" && \ - hash="$$(cat meta/buildhash)" && \ + fi; \ + ver="$$(cat meta/version)"; \ + hash="$$(cat meta/buildhash)"; \ ${RENAME_BIN} "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-"$${ver}"-* From 53b89adeebf149be644ab878f1d0843ea44417be Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 01:22:37 -0300 Subject: [PATCH 18/29] Fix for indentation not using tabs on the make file --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8360c87ac..c6fddd168 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ build-qt: clean: clean-dist @set -eu -o pipefail ${SHELLFLAGS}; \ for dir in $(DEVEL); do \ - $(SUBMAKE) -C $$dir clean; \ + $(SUBMAKE) -C $$dir clean; \ done .PHONY: clean-dist @@ -154,13 +154,13 @@ check: pyenv buildhash prepare @set -eu -o pipefail ${SHELLFLAGS}; \ .github/scripts/trailing-newlines.sh; \ for dir in $(CHECKABLE_RS); do \ - $(SUBMAKE) -C $$dir check; \ + $(SUBMAKE) -C $$dir check; \ done; \ . "${ACTIVATE_SCRIPT}"; \ $(SUBMAKE) -C rspy develop; \ $(SUBMAKE) -C pylib develop; \ for dir in $(CHECKABLE_PY); do \ - $(SUBMAKE) -C $$dir check; \ + $(SUBMAKE) -C $$dir check; \ done; @echo @echo "All checks passed!" @@ -170,7 +170,7 @@ fix: @set -eu -o pipefail ${SHELLFLAGS}; \ . "${ACTIVATE_SCRIPT}"; \ for dir in $(CHECKABLE_RS) $(CHECKABLE_PY); do \ - $(SUBMAKE) -C $$dir fix; \ + $(SUBMAKE) -C $$dir fix; \ done; \ .PHONY: add-buildhash From b77462eb8a2dfbdff43532c329c2daf1fc816887 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 01:57:03 -0300 Subject: [PATCH 19/29] Fixed lower case of runs-on: macos-latest Accordingly to the documentation the name is lower case: https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fa51eea50..fb6329601 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -81,7 +81,7 @@ jobs: mac_os_tests: - runs-on: macOS-latest + runs-on: macos-latest steps: - uses: actions/checkout@v2 - name: Configure environment variables From 1965cdb2b8d5bafec37600d776c7039d2a46d9a7 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 16:28:48 -0300 Subject: [PATCH 20/29] Set to use ankitects/setup-protoc@master instead of the original repository with a `repo-token` until know whether it is safe to pass the GitHub token around: https://github.com/actions/virtual-environments/issues 602 --- .github/workflows/checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fb6329601..529cfc4fc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -58,7 +58,7 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: arduino/setup-protoc@v1.1.0 + uses: ankitects/setup-protoc@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node @@ -135,7 +135,7 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: arduino/setup-protoc@v1.1.0 + uses: ankitects/setup-protoc@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node @@ -267,7 +267,7 @@ jobs: with: python-version: 3.7 - name: Set up protoc - uses: arduino/setup-protoc@v1.1.0 + uses: ankitects/setup-protoc@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up node From b94936a98d5795a364bfd545036417c6a0f87e6a Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 16:55:04 -0300 Subject: [PATCH 21/29] Update make to use the new `.SHELLFLAGS` feature https://github.com/ankitects/anki/pull 528 --- .github/workflows/checks.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 529cfc4fc..3607cb987 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -13,6 +13,9 @@ jobs: echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # Update make to use the new `.SHELLFLAGS` feature + # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac + echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - name: Fix ~/.cache permissions @@ -144,7 +147,7 @@ jobs: node-version: 12 - name: Set up brew packages run: | - brew install portaudio protobuf gettext rename ripgrep + brew install portaudio protobuf gettext rename ripgrep make brew link gettext --force - run: make develop - run: make build From c8e2c25e8343ff28d159537bba3ad52bdd9c6c99 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 25 Mar 2020 22:05:46 -0300 Subject: [PATCH 22/29] Fix Mac OS checks.yml caches being rebuilt by stop using the CARGO_TARGET_DIR and CARGO_INDEX_DIR variables: https://github.com/ankitects/anki/pull 528 --- .github/workflows/checks.yml | 50 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3607cb987..34c7f2458 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,37 +25,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - name: Set up python uses: actions/setup-python@v1 with: @@ -91,8 +91,10 @@ jobs: run: | # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" - echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" - echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # Disabled for Mac OS because it was not reusing the cached files + # https://github.com/ankitects/anki/pull/528 + # echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - name: Fix ~/.cache permissions @@ -102,37 +104,37 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - name: Set up python uses: actions/setup-python@v1 with: @@ -186,43 +188,43 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - name: Cache cargo registry uses: actions/cache@v1 with: path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo index uses: actions/cache@v1 with: path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}\rslib\target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}\rspy\target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-5- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- - name: Cache pip wheels uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-5- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- - name: Cache pacman uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-5- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-8- - name: Set up pacman, rsync if: steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd From e1252b785dfb2b6b2a11ce93560ac148f3797dde Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 01:20:00 -0300 Subject: [PATCH 23/29] Fixed checks.yml Mac OS gmake PATH being set on Ubuntu --- .github/workflows/checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 34c7f2458..b7334d16d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -13,9 +13,6 @@ jobs: echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" - # Update make to use the new `.SHELLFLAGS` feature - # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac - echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 - name: Fix ~/.cache permissions @@ -91,6 +88,9 @@ jobs: run: | # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" + # Update make to use the new `.SHELLFLAGS` feature + # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac + echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" # Disabled for Mac OS because it was not reusing the cached files # https://github.com/ankitects/anki/pull/528 # echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" From 96d572c3cc594df81a62820ed634f72adbc47899 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 01:40:53 -0300 Subject: [PATCH 24/29] Merged all workflows together using build matrixes --- .github/workflows/checks.yml | 298 ++++++++++++++--------------------- 1 file changed, 119 insertions(+), 179 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b7334d16d..366dccd12 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,175 +3,73 @@ name: Checks on: [push, pull_request] jobs: - linux_tests: - runs-on: ubuntu-latest + tests: + name: Test ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + include: + - os: ubuntu-latest + SEP: / + PIP_WHEELS_DIR: ~/.cache/pip + CARGO_INDEX_DIR: ~/.cargo/git + CARGO_REGISTRY_DIR: ~/.cargo/registry + ANKI_PYTHON_WHEELS: anki_linux_python_wheels + + - os: macos-latest + SEP: / + PIP_WHEELS_DIR: ~/Library/Caches/pip + CARGO_INDEX_DIR: ~/.cargo/git + CARGO_REGISTRY_DIR: ~/.cargo/registry + ANKI_PYTHON_WHEELS: anki_mac os_python_wheels + + - os: windows-latest + SEP: \ + PIP_WHEELS_DIR: ~\AppData\Local\pip\Cache + CARGO_INDEX_DIR: C:\Rust\.cargo\git + CARGO_REGISTRY_DIR: C:\Rust\.cargo\registry + ANKI_PYTHON_WHEELS: anki_windows_python_wheels + steps: + - run: git config --global core.autocrlf false + if: matrix.os == 'windows-latest' + - uses: actions/checkout@v2 - - name: Configure environment variables + + - name: Configure Linux environment variables + if: matrix.os == 'ubuntu-latest' run: | # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" - # Necessary for now for the cargo cache: - # https://github.com/actions/cache/issues/133#issuecomment-599102035 - - name: Fix ~/.cache permissions - run: | - sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo target - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rslib - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rspy - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: ankitects/setup-protoc@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Set up ripgrep, pyaudio, gettext, rename - run: | - sudo apt update - sudo apt install portaudio19-dev gettext rename - sudo snap install ripgrep --classic - - run: make develop - - run: make build - - run: make check - - name: Upload python wheels - uses: actions/upload-artifact@v1 - with: - name: anki_linux_python_wheels - path: dist - - mac_os_tests: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - name: Configure environment variables + - name: Configure Mac OS environment variables + if: matrix.os == 'macos-latest' run: | # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" + # Update make to use the new `.SHELLFLAGS` feature # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" + # Disabled for Mac OS because it was not reusing the cached files # https://github.com/ankitects/anki/pull/528 # echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" # echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" - # Necessary for now for the cargo cache: - # https://github.com/actions/cache/issues/133#issuecomment-599102035 - - name: Fix ~/.cache permissions - run: | - sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo target - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rslib - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rslib/target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rspy - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/rspy/target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}/pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - - name: Set up python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Set up protoc - uses: ankitects/setup-protoc@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Set up brew packages - run: | - brew install portaudio protobuf gettext rename ripgrep make - brew link gettext --force - - run: make develop - - run: make build - - run: make check - - name: Upload python wheels - uses: actions/upload-artifact@v1 - with: - name: anki_mac_os_python_wheels - path: dist - - windows_tests: - runs-on: windows-latest - steps: - - run: git config --global core.autocrlf false - - uses: actions/checkout@v2 - - name: Configure environment variables + - name: Configure Windows environment variables + if: matrix.os == 'windows-latest' run: | # echo "::set-env name=SHELLFLAGS::-x" + echo "::set-env name=BUILDFLAGS::" + $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") + echo "::set-env name=pacmanbin::pacman-5.1.1-3-x86_64.pkg.tar.xz" echo "::set-env name=pacmanmirror::pacman-mirrors-20200307-1-any.pkg.tar.xz" echo "::set-env name=pacmankeys::msys2-keyring-r9.397a52e-1-any.pkg.tar.xz" @@ -181,62 +79,80 @@ jobs: echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" - echo "::set-env name=BUILDFLAGS::" echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" - - name: Cache pyenv - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: C:\Rust\.cargo\git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo target - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rslib - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\rslib\target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- - - name: Cache cargo rspy - uses: actions/cache@v1 - with: - path: ${{ github.workspace }}\rspy\target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- + + # Necessary for now for the cargo cache: + # https://github.com/actions/cache/issues/133#issuecomment-599102035 + - name: Fix ~/.cache permissions + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + run: | + sudo chown -R $(whoami):$(id -ng) ~/.cargo/ + - name: Cache pip wheels uses: actions/cache@v1 with: - path: ~\AppData\Local\pip\Cache + path: ${{ matrix.PIP_WHEELS_DIR }} key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- + + - name: Cache pyenv + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}${{ matrix.SEP }}pyenv + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- + + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: ${{ matrix.CARGO_INDEX_DIR }} + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- + + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: ${{ matrix.CARGO_REGISTRY_DIR }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- + + - name: Cache cargo target + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}${{ matrix.SEP }}target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- + + - name: Cache cargo rslib + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}${{ matrix.SEP }}rslib${{ matrix.SEP }}target + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- + + - name: Cache cargo rspy + uses: actions/cache@v1 + with: + path: ${{ github.workspace }}${{ matrix.SEP }}rspy${{ matrix.SEP }}target + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- + - name: Cache pacman + if: matrix.os == 'windows-latest' uses: actions/cache@v1 id: cache-pacman with: path: C:\Program Files\Git key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-8- + - name: Set up pacman, rsync - if: steps.cache-pacman.outputs.cache-hit != 'true' + if: matrix.os == 'windows-latest' && steps.cache-pacman.outputs.cache-hit != 'true' shell: cmd run: | echo on curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% + :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + :: Manually Install Pacman Binaries :: https://github.com/Alexpux/MSYS2-pacman/issues/50 set "PATH=%programfiles%\Git\usr\bin" @@ -247,43 +163,67 @@ jobs: pacman -Syyuuv --overwrite='*' pacman -Syuuv --overwrite='*' pacman --version + :: We must install bash first, otherwise we will have bash fork errors: :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly pacman -Sv --noconfirm --overwrite='*' bash pacman -Sv --noconfirm --overwrite='*' rsync + :: clean all packages to decrease image size pacman -Sccv --noconfirm pacman -Qsv --noconfirm + - name: Set up pyaudio, rename + if: matrix.os == 'windows-latest' shell: cmd run: | echo on curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename + - name: Set up scoop, gettext, ripgrep + if: matrix.os == 'windows-latest' shell: cmd run: | echo on powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" scoop install -g gettext ripgrep + + - name: Set up ripgrep, pyaudio, gettext, rename + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install portaudio19-dev gettext rename + sudo snap install ripgrep --classic + + - name: Set up brew packages + if: matrix.os == 'macos-latest' + run: | + brew install portaudio protobuf gettext rename ripgrep make + brew link gettext --force + - name: Set up python uses: actions/setup-python@v1 with: python-version: 3.7 + - name: Set up protoc uses: ankitects/setup-protoc@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up node uses: actions/setup-node@v1 with: node-version: 12 + - run: make develop - run: make build - run: make check + - name: Upload python wheels uses: actions/upload-artifact@v1 with: - name: anki_windows_python_wheels + name: ${{ matrix.ANKI_PYTHON_WHEELS }} path: dist From 8b8be3e3f3ed17dd6be7f7ba18b3b68485021e9b Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 02:45:55 -0300 Subject: [PATCH 25/29] Fixed misspelling Mac OS with a space on the build matrix --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 366dccd12..05bb104c2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: PIP_WHEELS_DIR: ~/Library/Caches/pip CARGO_INDEX_DIR: ~/.cargo/git CARGO_REGISTRY_DIR: ~/.cargo/registry - ANKI_PYTHON_WHEELS: anki_mac os_python_wheels + ANKI_PYTHON_WHEELS: anki_macos_python_wheels - os: windows-latest SEP: \ From 1e4010332571c46187ebe2c7cf9ec192758194fb Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 03:04:47 -0300 Subject: [PATCH 26/29] Moved RSPY_TARGET_DIR and CARGO_TARGET_DIR to the top --- .github/workflows/checks.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 05bb104c2..0b89e4a6e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -51,20 +51,22 @@ jobs: # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" - # Update make to use the new `.SHELLFLAGS` feature - # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac - echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" - # Disabled for Mac OS because it was not reusing the cached files # https://github.com/ankitects/anki/pull/528 # echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target" # echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target" + # Update make to use the new `.SHELLFLAGS` feature + # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac + echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH" + - name: Configure Windows environment variables if: matrix.os == 'windows-latest' run: | # echo "::set-env name=SHELLFLAGS::-x" echo "::set-env name=BUILDFLAGS::" + echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" + echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") @@ -79,8 +81,6 @@ jobs: echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE" echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" - echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target" - echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target" # Necessary for now for the cargo cache: # https://github.com/actions/cache/issues/133#issuecomment-599102035 @@ -174,7 +174,7 @@ jobs: pacman -Sccv --noconfirm pacman -Qsv --noconfirm - - name: Set up pyaudio, rename + - name: Set up curl pyaudio, rename if: matrix.os == 'windows-latest' shell: cmd run: | @@ -182,7 +182,7 @@ jobs: curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename - - name: Set up scoop, gettext, ripgrep + - name: Set up Windows scoop, gettext, ripgrep if: matrix.os == 'windows-latest' shell: cmd run: | @@ -190,14 +190,14 @@ jobs: powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" scoop install -g gettext ripgrep - - name: Set up ripgrep, pyaudio, gettext, rename + - name: Set up Ubuntu ripgrep, pyaudio, gettext, rename if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install portaudio19-dev gettext rename sudo snap install ripgrep --classic - - name: Set up brew packages + - name: Set up brew ripgrep, pyaudio, gettext, rename if: matrix.os == 'macos-latest' run: | brew install portaudio protobuf gettext rename ripgrep make From 76bf3f8a03aeb3fb2327ca042457192a3045756e Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 16:39:34 -0300 Subject: [PATCH 27/29] Created a fast build matrix for quickly running the Unit Tests --- .github/workflows/checks.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0b89e4a6e..dae6c5d64 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -4,11 +4,12 @@ on: [push, pull_request] jobs: tests: - name: Test ${{ matrix.os }} + name: ${{ matrix.os }} ${{ matrix.BUILD_TYPE }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] + BUILD_TYPE: [check, build] include: - os: ubuntu-latest SEP: / @@ -31,6 +32,9 @@ jobs: CARGO_REGISTRY_DIR: C:\Rust\.cargo\registry ANKI_PYTHON_WHEELS: anki_windows_python_wheels + # Keep running all matrices if something fail + fail-fast: false + steps: - run: git config --global core.autocrlf false if: matrix.os == 'windows-latest' @@ -93,43 +97,43 @@ jobs: uses: actions/cache@v1 with: path: ${{ matrix.PIP_WHEELS_DIR }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-8- + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-11- - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-8- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-11- - name: Cache cargo index uses: actions/cache@v1 with: path: ${{ matrix.CARGO_INDEX_DIR }} - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-8- + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-11- - name: Cache cargo registry uses: actions/cache@v1 with: path: ${{ matrix.CARGO_REGISTRY_DIR }} - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-8- + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-11- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-8- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rslib${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-8- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rspy${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-8- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- - name: Cache pacman if: matrix.os == 'windows-latest' @@ -137,7 +141,7 @@ jobs: id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-8- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-11- - name: Set up pacman, rsync if: matrix.os == 'windows-latest' && steps.cache-pacman.outputs.cache-hit != 'true' @@ -219,10 +223,16 @@ jobs: node-version: 12 - run: make develop + if: matrix.BUILD_TYPE != 'check' + - run: make build + if: matrix.BUILD_TYPE != 'check' + - run: make check + if: matrix.BUILD_TYPE == 'check' - name: Upload python wheels + if: matrix.BUILD_TYPE != 'check' uses: actions/upload-artifact@v1 with: name: ${{ matrix.ANKI_PYTHON_WHEELS }} From 9bd28f151f94d1bc2d16e2d116f5276858715bc7 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Thu, 26 Mar 2020 21:55:03 -0300 Subject: [PATCH 28/29] Disabled the caches of cargo registry, index and python wheels --- .github/workflows/checks.yml | 44 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index dae6c5d64..92879dc30 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -93,47 +93,49 @@ jobs: run: | sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - name: Cache pip wheels - uses: actions/cache@v1 - with: - path: ${{ matrix.PIP_WHEELS_DIR }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-11- - - name: Cache pyenv uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-11- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-12- - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: ${{ matrix.CARGO_INDEX_DIR }} - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-11- + # Disabling these caches for now because they do not seem to be used/help + # https://github.com/ankitects/anki/pull/528 + # - name: Cache pip wheels + # uses: actions/cache@v1 + # with: + # path: ${{ matrix.PIP_WHEELS_DIR }} + # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-12- - - name: Cache cargo registry - uses: actions/cache@v1 - with: - path: ${{ matrix.CARGO_REGISTRY_DIR }} - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-11- + # - name: Cache cargo index + # uses: actions/cache@v1 + # with: + # path: ${{ matrix.CARGO_INDEX_DIR }} + # key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-12- + + # - name: Cache cargo registry + # uses: actions/cache@v1 + # with: + # path: ${{ matrix.CARGO_REGISTRY_DIR }} + # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-12- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rslib${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rspy${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-11- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- - name: Cache pacman if: matrix.os == 'windows-latest' @@ -141,7 +143,7 @@ jobs: id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-11- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-12- - name: Set up pacman, rsync if: matrix.os == 'windows-latest' && steps.cache-pacman.outputs.cache-hit != 'true' From 3f8cc60abc5be048b4eb34188242cce8fd852a5e Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Fri, 27 Mar 2020 02:17:25 -0300 Subject: [PATCH 29/29] Re-enabled the rust cargo index and registry caches because cargo was downloading them just for nothing. https://github.com/ankitects/anki/pull 528 --- .github/workflows/checks.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 92879dc30..cf8e02be1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -97,7 +97,7 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}pyenv - key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-12- + key: ${{ runner.os }}-pyenv-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/requirements.txt') }}-14- # Disabling these caches for now because they do not seem to be used/help # https://github.com/ankitects/anki/pull/528 @@ -105,37 +105,37 @@ jobs: # uses: actions/cache@v1 # with: # path: ${{ matrix.PIP_WHEELS_DIR }} - # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-12- + # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-14- - # - name: Cache cargo index - # uses: actions/cache@v1 - # with: - # path: ${{ matrix.CARGO_INDEX_DIR }} - # key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-12- + - name: Cache cargo index + uses: actions/cache@v1 + with: + path: ${{ matrix.CARGO_INDEX_DIR }} + key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-14- - # - name: Cache cargo registry - # uses: actions/cache@v1 - # with: - # path: ${{ matrix.CARGO_REGISTRY_DIR }} - # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-12- + - name: Cache cargo registry + uses: actions/cache@v1 + with: + path: ${{ matrix.CARGO_REGISTRY_DIR }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-14- - name: Cache cargo target uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- + key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-14- - name: Cache cargo rslib uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rslib${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- + key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-14- - name: Cache cargo rspy uses: actions/cache@v1 with: path: ${{ github.workspace }}${{ matrix.SEP }}rspy${{ matrix.SEP }}target - key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-$${{ matrix.BUILD_TYPE }}-12- + key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-14- - name: Cache pacman if: matrix.os == 'windows-latest' @@ -143,7 +143,7 @@ jobs: id: cache-pacman with: path: C:\Program Files\Git - key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-12- + key: ${{ runner.os }}-pacman-${{ hashFiles('**/checks.yml') }}-14- - name: Set up pacman, rsync if: matrix.os == 'windows-latest' && steps.cache-pacman.outputs.cache-hit != 'true'