From 066986796444964a4be63f53ef3c8211b0c44378 Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sat, 26 Oct 2024 02:37:25 +0000 Subject: [PATCH] Refactor GitHub workflows and configuration for improved dependency management and consistency and fixing alpine release Signed-off-by: avifenesh --- .github/json_matrices/build-matrix.json | 48 +++++++++++++++---- .../install-shared-dependencies/action.yml | 2 +- .github/workflows/node.yml | 6 +-- .github/workflows/npm-cd.yml | 25 +++++----- .github/workflows/pypi-cd.yml | 16 ++----- .../workflows/setup-musl-on-linux/action.yml | 4 +- node/package.json | 10 ++-- 7 files changed, 70 insertions(+), 41 deletions(-) diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index 45ac7a57f3..fc02093b9f 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,15 +5,27 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm", "maven"] + "PACKAGE_MANAGERS": [ + "pypi", + "npm", + "maven" + ] }, { "OS": "ubuntu", "NAMED_OS": "linux", - "RUNNER": ["self-hosted", "Linux", "ARM64"], + "RUNNER": [ + "self-hosted", + "Linux", + "ARM64" + ], "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm", "maven"], + "PACKAGE_MANAGERS": [ + "pypi", + "npm", + "maven" + ], "CONTAINER": "2_28" }, { @@ -22,7 +34,11 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm", "maven"] + "PACKAGE_MANAGERS": [ + "pypi", + "npm", + "maven" + ] }, { "OS": "macos", @@ -30,17 +46,27 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm", "maven"] + "PACKAGE_MANAGERS": [ + "pypi", + "npm", + "maven" + ] }, { "OS": "ubuntu", "NAMED_OS": "linux", "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-musl", - "RUNNER": ["self-hosted", "Linux", "ARM64"], - "IMAGE": "node:alpine", + "RUNNER": [ + "self-hosted", + "Linux", + "ARM64" + ], + "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged --rm", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": [ + "npm" + ] }, { "OS": "ubuntu", @@ -48,8 +74,10 @@ "ARCH": "x64", "TARGET": "x86_64-unknown-linux-musl", "RUNNER": "ubuntu-latest", - "IMAGE": "node:alpine", + "IMAGE": "node:lts-alpine3.19", "CONTAINER_OPTIONS": "--user root --privileged", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": [ + "npm" + ] } ] diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index 1cb56e63f0..ed065e9840 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -39,7 +39,7 @@ runs: if: "${{ inputs.os == 'macos' }}" run: | brew update - brew install git gcc pkgconfig openssl coreutils + brew install git openssl coreutils - name: Install software dependencies for Ubuntu GNU shell: bash diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index c4c17a7e46..d8f690e560 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -70,7 +70,7 @@ jobs: - name: Use Node.js 16.x - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 16.x @@ -286,10 +286,10 @@ jobs: - uses: actions/checkout@v4 - - name: Use Node.js 18.x + - name: Use Node.js 16.x uses: actions/setup-node@v4 with: - node-version: 18.x + node-version: 16.x - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 3788d87e04..362117affb 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -119,10 +119,10 @@ jobs: INPUT_VERSION: ${{ github.event.inputs.version }} - name: Setup node - if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} - uses: actions/setup-node@v3 + if: ${{ !contains(matrix.build.TARGET, 'musl') }} + uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "latest" registry-url: "https://registry.npmjs.org" architecture: ${{ matrix.build.ARCH }} scope: "${{ vars.NPM_SCOPE }}" @@ -130,7 +130,7 @@ jobs: token: ${{ secrets.NPM_AUTH_TOKEN }} - name: Setup node for publishing - if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-musl' }} + if: ${{ !contains(matrix.build.TARGET, 'musl') }} working-directory: ./node run: | npm config set registry https://registry.npmjs.org/ @@ -185,7 +185,7 @@ jobs: # 2>&1 1>&3- redirects stderr to stdout and then redirects the original stdout to another file descriptor, # effectively separating stderr and stdout. The 3>&1 at the end redirects the original stdout back to the console. # https://github.com/npm/npm/issues/118#issuecomment-325440 - ignoring notice messages since currentlly they are directed to stderr - { npm_publish_err=$(npm publish --tag ${{ env.NPM_TAG }} --access public 2>&1 1>&3- | grep -v "notice") ;} 3>&1 + { npm_publish_err=$(npm publish --tag ${{ env.NPM_TAG }} --access public 2>&1 1>&3- | grep -Ev "notice|ExperimentalWarning") ;} 3>&1 if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]] then echo "Skipping publishing, package already published" @@ -203,8 +203,11 @@ jobs: if: ${{ matrix.build.ARCH == 'arm64' }} shell: bash run: | - git reset --hard + echo "Resetting repository" git clean -xdf + git reset --hard + git fetch + git checkout ${{ github.sha }} publish-base-to-npm: if: github.event_name != 'pull_request' @@ -218,9 +221,9 @@ jobs: submodules: "true" - name: Install node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "latest" registry-url: "https://registry.npmjs.org" scope: "${{ vars.NPM_SCOPE }}" always-auth: true @@ -336,10 +339,10 @@ jobs: arch: ${{ matrix.build.ARCH }} - name: Setup node - if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} - uses: actions/setup-node@v3 + if: ${{ !contains(matrix.build.TARGET, 'musl') }} + uses: actions/setup-node@v4 with: - node-version: "16" + node-version: "latest" registry-url: "https://registry.npmjs.org" architecture: ${{ matrix.build.ARCH }} scope: "${{ vars.NPM_SCOPE }}" diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index e69343f234..5d547ac7f5 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -112,16 +112,10 @@ jobs: if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }} uses: actions/setup-python@v5 with: - python-version: "3.10" - - - name: Set up Python older versions for MacOS - if: startsWith(matrix.build.NAMED_OS, 'darwin') - run: | - brew update - brew install python@3.8 python@3.9 + python-version: "3.12" - name: Setup Python for self-hosted Ubuntu runners - if: contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted') + if: contains(matrix.build.RUNNER, 'self-hosted') run: | sudo apt update -y sudo apt upgrade -y @@ -140,7 +134,7 @@ jobs: target: ${{ matrix.build.TARGET }} publish: "true" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: "7.2" - name: Include protobuf files in the package working-directory: ./python @@ -186,7 +180,7 @@ jobs: with: working-directory: ./python target: ${{ matrix.build.TARGET }} - args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.10' }} + args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.8 python3.9 python3.10 python3.11 python3.12' || 'python3.12' }} - name: Upload Python wheels if: github.event_name != 'pull_request' @@ -241,7 +235,7 @@ jobs: - name: Install ValKey uses: ./.github/workflows/install-valkey with: - version: "8.0.0" + version: "8.0" - name: Check if RC and set a distribution tag for the package shell: bash diff --git a/.github/workflows/setup-musl-on-linux/action.yml b/.github/workflows/setup-musl-on-linux/action.yml index ed4677b74a..f270c27507 100644 --- a/.github/workflows/setup-musl-on-linux/action.yml +++ b/.github/workflows/setup-musl-on-linux/action.yml @@ -42,10 +42,10 @@ runs: shell: bash run: | git config --global --add safe.directory "${{ inputs.workspace }}" + git fetch origin ${{ github.sha }} + git checkout ${{ github.sha }} git clean -xdf git reset --hard - git submodule sync - git submodule update --init --recursive - name: Set up access for musl on ARM shell: bash diff --git a/node/package.json b/node/package.json index 6cf654aa29..85d92b0476 100644 --- a/node/package.json +++ b/node/package.json @@ -46,7 +46,7 @@ }, "devDependencies": { "@jest/globals": "^29.7.0", - "@types/jest": "^29.5.12", + "@types/jest": "^29.5.14", "@types/minimist": "^1.2.5", "@types/redis-server": "^1.2.2", "@types/semver": "^7.5.8", @@ -71,10 +71,14 @@ "tests/", "rust-client/**", "!build-ts/**", - "babel.config.js", + ".prettierignore", "jest.config.js", "hybrid-node-tests/**", - "docs/" + "docs/", + "DEVELOPER.md", + ".ort.yml", + "tsconfig.json", + "THIRD_PARTY_LICENSES_NODE" ] }, "//": [