anki/.github/workflows/checks.yml

name: Checks

on: [push, pull_request]

jobs:
  check_tag:
    name: Tag name
    outputs:
      tagged: ${{ steps.check_tagged.outputs.tagged }}
      matrix: ${{ steps.check_tagged.outputs.matrix }}
    runs-on: ubuntu-latest
    steps:
      - name: Check the ref
        id: check_tagged
        run: |
          set -x
          printf '%s' '#!/usr/bin/env python3
          # -*- coding: utf-8 -*-

          import json
          import argparse

          matrix = json.loads(r"""
          {
            "BUILD_TYPE": [
              "build"
            ],
            "os": [
              "ubuntu-latest",
              "macos-latest",
              "windows-latest"
            ],
            "python": [
              3.7
            ],
            "include": [
              {
                "os": "ubuntu-latest",
                "name": "Ubuntu",
                "SEP": "/",
                "PIP_WHEELS_DIR": "~/.cache/pip",
                "CARGO_INDEX_DIR": "~/.cargo/git",
                "CARGO_REGISTRY_DIR": "~/.cargo/registry",
                "ANKI_PYTHON_WHEELS": "anki_linux_python"
              },
              {
                "os": "macos-latest",
                "name": "Mac OS",
                "SEP": "/",
                "PIP_WHEELS_DIR": "~/Library/Caches/pip",
                "CARGO_INDEX_DIR": "~/.cargo/git",
                "CARGO_REGISTRY_DIR": "~/.cargo/registry",
                "ANKI_PYTHON_WHEELS": "anki_macos_python"
              },
              {
                "os": "windows-latest",
                "name": "Windows",
                "SEP": "\\",
                "PIP_WHEELS_DIR": "~\\AppData\\Local\\pip\\Cache",
                "CARGO_INDEX_DIR": "C:\\Rust\\.cargo\\git",
                "CARGO_REGISTRY_DIR": "C:\\Rust\\.cargo\\registry",
                "ANKI_PYTHON_WHEELS": "anki_windows_python"
              }
            ]
          }
          """)

          parser = argparse.ArgumentParser(description="Dynamically creates a build matrix for the GitHub Actions.")
          parser.add_argument("--tagged", action="store_true", default=False,
              help="Adds Python 3.8 builds into the matrix")

          args = parser.parse_args()

          if args.tagged:
            matrix["python"].append("3.8")
          else:
            matrix["BUILD_TYPE"].append("check")

          print(json.dumps(matrix))
          ' > buildmatrix.py;

          if [[ ${{ github.ref }} == refs/tags/* ]];
          then
            printf '::set-output name=tagged::%s\n' "tagged"
            printf '::set-output name=matrix::%s\n' "$(python3 buildmatrix.py --tagged)"
          else
            printf '::set-output name=tagged::%s\n' ""
            printf '::set-output name=matrix::%s\n' "$(python3 buildmatrix.py)"
          fi

  contrib:
    name: Author in CONTRIBUTORS
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v1
      - name: Check
        run: |
          set -x
          .github/scripts/contrib.sh

  tests:
    needs:
      - check_tag
    name: ${{ matrix.name }} ${{ needs.check_tag.outputs.tagged }} ${{ matrix.BUILD_TYPE }} ${{ matrix.python }}
    runs-on: ${{ matrix.os }}
    strategy:
      # https://github.com/ankitects/anki/pull/598
      matrix: ${{ fromJson( needs.check_tag.outputs.matrix ) }}

      # Keep all systems running if something fails
      fail-fast: false

    steps:
      - run: git config --global core.autocrlf false
        if: matrix.os == 'windows-latest'

      - uses: actions/checkout@v2

      - name: Configure Linux environment variables
        if: matrix.os == 'ubuntu-latest'
        run: |
          # echo "::set-env name=SHELLFLAGS::-x"
          echo "::set-env name=BUILDFLAGS::"
          echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target"
          echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target"

      - name: Configure Mac OS environment variables
        if: matrix.os == 'macos-latest'
        run: |
          # echo "::set-env name=SHELLFLAGS::-x"
          echo "::set-env name=BUILDFLAGS::"

          # Disabled for Mac OS because it was not reusing the cached files
          # https://github.com/ankitects/anki/pull/528
          # echo "::set-env name=RSPY_TARGET_DIR::${GITHUB_WORKSPACE}/target"
          # echo "::set-env name=CARGO_TARGET_DIR::${GITHUB_WORKSPACE}/target"

          # Update make to use the new `.SHELLFLAGS` feature
          # https://stackoverflow.com/questions/43175529/updating-make-version-4-1-on-mac
          echo "::set-env name=PATH::/usr/local/opt/make/libexec/gnubin:$PATH"

          # https://stackoverflow.com/questions/59644349/msgmerge-on-macos-catalina
          echo "::set-env name=PATH::/usr/local/opt/gettext/bin:$PATH"

      - name: Configure Windows environment variables
        if: matrix.os == 'windows-latest'
        run: |
          # echo "::set-env name=SHELLFLAGS::-x"
          echo "::set-env name=BUILDFLAGS::"
          echo "::set-env name=RSPY_TARGET_DIR::$env:GITHUB_WORKSPACE\target"
          echo "::set-env name=CARGO_TARGET_DIR::$env:GITHUB_WORKSPACE\target"

          # https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyaudio
          if( "3.7".equals( "${{ matrix.python }}" ) ) {
            $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl")
          }
          else {
            $pyaudio=("PyAudio-0.2.11-cp38-cp38-win_amd64.whl")
          }

          $new_path=("$env:GITHUB_WORKSPACE;$env:PATH")
          $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path")

          echo "::set-env name=rsyncbin::rsync-3.1.3-1-x86_64.pkg.tar.xz"
          echo "::set-env name=pyaudio::$pyaudio"
          echo "::set-env name=PATH::$new_path"
          echo "::set-env name=RUST_BACKTRACE::full"
          echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE"
          echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE"
          echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio"

      # Necessary for now for the cargo cache:
      # https://github.com/actions/cache/issues/133#issuecomment-599102035
      - name: Fix ~/.cache permissions
        if: matrix.os == 'ubuntu-latest' && matrix.python == '3.7'
        run: |
          sudo chown -R $(whoami):$(id -ng) ~/.cargo/

      # Disable it for macos because it pyenv cache is bugged (https://github.com/ankitects/anki/pull/563)
      - name: Cache pyenv
        if: matrix.os != 'macos-latest' && matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ github.workspace }}${{ matrix.SEP }}pyenv
          key: ${{ runner.os }}-pyenv-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-24-

      # # Disable it in attempt to reduce the overall cache size (https://github.com/ankitects/anki/pull/528)
      # - name: Cache pip wheels
      #   if: matrix.BUILD_TYPE == 'build' && matrix.python == '3.7'
      #   uses: actions/cache@v2
      #   with:
      #     path: ${{ matrix.PIP_WHEELS_DIR }}
      #     key: ${{ runner.os }}-pip-wheels-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/setup.py') }}-24-

      - name: Cache cargo index
        if: matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ matrix.CARGO_INDEX_DIR }}
          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-24-

      - name: Cache cargo registry
        if: matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ matrix.CARGO_REGISTRY_DIR }}
          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-24-

      - name: Cache cargo target
        if: matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ github.workspace }}${{ matrix.SEP }}target
          key: ${{ runner.os }}-cargo-target-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-24-

      - name: Cache cargo rslib
        if: matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ github.workspace }}${{ matrix.SEP }}rslib${{ matrix.SEP }}target
          key: ${{ runner.os }}-cargo-rslib-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-24-

      - name: Cache cargo rspy
        if: matrix.python == '3.7'
        uses: actions/cache@v2
        with:
          path: ${{ github.workspace }}${{ matrix.SEP }}rspy${{ matrix.SEP }}target
          key: ${{ runner.os }}-cargo-rspy-${{ hashFiles('**/requirements.*') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/Makefile') }}-${{ hashFiles('**/Cargo.toml') }}-${{ matrix.BUILD_TYPE }}-24-

      - name: Set up curl pyaudio, rsync
        if: matrix.os == 'windows-latest'
        shell: cmd
        run: |
          echo on
          curl --fail -LO https://github.com/ankitects/pyaudio-windows/raw/master/%pyaudio%
          curl --fail -LO http://repo.msys2.org/msys/x86_64/%rsyncbin%

          :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz
          7z x "%rsyncbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git"

      - name: Set up Windows scoop, gettext, ripgrep
        if: matrix.os == 'windows-latest'
        shell: cmd
        run: |
          echo on
          powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')"
          scoop install -g gettext ripgrep

      - name: Set up Ubuntu ripgrep, pyaudio, gettext
        if: matrix.os == 'ubuntu-latest'
        run: |
          set -x
          sudo apt update
          sudo apt install portaudio19-dev gettext
          # https://github.com/BurntSushi/ripgrep/issues/1232
          # sudo apt-get install ripgrep
          curl --fail -LO https://github.com/BurntSushi/ripgrep/releases/download/11.0.2/ripgrep_11.0.2_amd64.deb
          sudo dpkg -i ripgrep_11.0.2_amd64.deb

      - name: Set up brew ripgrep, pyaudio, gettext
        if: matrix.os == 'macos-latest'
        run: |
          set -x
          brew install portaudio protobuf gettext ripgrep make

      - name: Set up python
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python }}

      - name: Set up protoc
        uses: ankitects/setup-protoc@master
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up node
        uses: actions/setup-node@v1
        with:
          node-version: 12

      - run: make develop
        if: matrix.BUILD_TYPE == 'build'

      - run: make build
        if: matrix.BUILD_TYPE == 'build'

      - name: Check Windows wheels
        if: matrix.BUILD_TYPE == 'build' && matrix.os == 'windows-latest'
        shell: cmd
        run: |
          echo on
          copy %pyaudio% dist
          cd dist
          setlocal EnableDelayedExpansion
          cmd /C set "wheels=" && for /f "delims=" %%i in ('dir /b *.*') DO set "wheels="%%i" !wheels!"

          python -m pip install pyqtwebengine %wheels%
          python -c "import aqt; # aqt.run()"

      - name: Check Linux/Mac OS wheels
        if: matrix.BUILD_TYPE == 'build' && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' )
        run: |
          set -x
          cd dist
          python -m pip install pyqtwebengine pyaudio *.*
          python -c "import aqt; # aqt.run()"

      - run: make check
        if: matrix.BUILD_TYPE == 'check'

      - name: Upload python wheels
        if: matrix.BUILD_TYPE == 'build'
        uses: actions/upload-artifact@v2
        with:
          name: ${{ matrix.ANKI_PYTHON_WHEELS }}${{ matrix.python }}
          path: dist

      - name: Upload to PyPi Windows
        if: matrix.BUILD_TYPE == 'build' && startsWith(github.ref, 'refs/tags/') && matrix.os == 'windows-latest'
        shell: cmd
        env:
          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
          TWINE_USERNAME: __token__
        run: |
          echo on
          cd dist
          rm -f "%pyaudio%"
          setlocal EnableDelayedExpansion
          cmd /C set "wheels=" && for /f "delims=" %%i in ('dir /b *.*') DO set "wheels="%%i" !wheels!"

          :: https://github.com/ankitects/anki/pull/535
          perl ../scripts/rename -f "s@\+[\w\d]+-@-@g" %wheels%
          python -m pip install twine
          python -m twine upload --non-interactive --skip-existing --verbose ./*

      - name: Upload to PyPi Linux/Mac OS
        if: matrix.BUILD_TYPE == 'build' && startsWith(github.ref, 'refs/tags/') && ( matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' )
        env:
          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
          TWINE_USERNAME: __token__
        run: |
          set -x
          python -m pip install twine

          # https://github.com/ankitects/anki/pull/535
          perl scripts/rename -f "s@\+[\w\d]+-@-@g" dist/*
          python -m twine upload --non-interactive --skip-existing --verbose dist/*

      - name: macOS CI workaround test
        if: matrix.BUILD_TYPE == 'check' && matrix.os == 'macos-latest'
        run: sync && sleep 10