feat: re-use packages in builds #19
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build all ports in the repository | |
| on: | |
| - push | |
| env: | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| jobs: | |
| list-ports: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| PORTS_TO_BUILD_JSON: ${{ steps.list_ports_to_build.outputs.PORTS_TO_BUILD_JSON }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Create a list of ports to build | |
| id: list_ports_to_build | |
| shell: sh | |
| run: | | |
| # find ports in the repository and make it a list of one line | |
| # separated by a space. | |
| PORTS_TO_BUILD=`find * -type d -maxdepth 1 -mindepth 1 | tr '\n' ' ' | sed -e 's/ $//'` | |
| # create a JSON from the list | |
| PORTS_TO_BUILD_JSON=`jq -n -c -M --arg V "${PORTS_TO_BUILD}" '{ PORT: ($V | split(" ")) }'` | |
| # pass the JSON to other jobs | |
| echo "PORTS_TO_BUILD_JSON=${PORTS_TO_BUILD_JSON}" >> "${GITHUB_OUTPUT}" | |
| cat "${GITHUB_OUTPUT}" | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - list-ports | |
| strategy: | |
| # use matrix so that the total time of build does not exceed max job (6 | |
| # hours) and workflow time (35 days). | |
| # https://docs.github.com/en/actions/administering-github-actions/usage-limits-billing-and-administration | |
| # | |
| # each job builds one port in the repository | |
| matrix: ${{ fromJSON(needs.list-ports.outputs.PORTS_TO_BUILD_JSON) }} | |
| # do dont cancel other jobs in the matrix when a job fails. | |
| fail-fast: false | |
| steps: | |
| - name: Maximize build space | |
| if: runner.os == 'Linux' | |
| uses: AdityaGarg8/remove-unwanted-software@8831c82abf29b34eb2caac48d5f999ecfc0d8eef | |
| with: | |
| remove-android: 'true' | |
| remove-dotnet: 'true' | |
| remove-haskell: 'true' | |
| remove-codeql: 'true' | |
| remove-docker-images: 'true' | |
| remove-large-packages: 'true' | |
| remove-cached-tools: 'true' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download packages | |
| id: download-packages | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| # download packages cache from previous workflow run | |
| workflow: poudriere.yml | |
| workflow_conclusion: completed | |
| name: packages-cache | |
| path: packages | |
| if_no_artifact_found: warn | |
| - name: poudriere bulk | |
| id: make | |
| uses: cross-platform-actions/action@v0.25.0 | |
| with: | |
| operating_system: freebsd | |
| version: '14.1' | |
| image_url: https://github.com/trombik/freebsd-builder-poudriere/releases/download/v1.0.1/freebsd-14.1-x86-64.qcow2 | |
| sync_files: true | |
| run: | | |
| set -x | |
| set -e | |
| df -h -tufs | |
| tree . | |
| poudriere ports -l | |
| poudriere jail -l | |
| JAIL_NAME_VERSION=`uname -r | sed -E -e 's/-(CURRENT|RELEASE).*//' -e 's/\.//'` | |
| JAIL_NAME_ARCH=`uname -m` | |
| JAIL_NAME="${JAIL_NAME_VERSION}${JAIL_NAME_ARCH}" | |
| PORTS_NAME="default" | |
| OVERLAY_DIR=`realpath .` | |
| PORT_TO_BUILD=${{ matrix.PORT }} | |
| # disable tmpfs to save memory | |
| echo USE_TMPFS=no | sudo tee -a /usr/local/etc/poudriere.conf | |
| # avoid crushing guest OS due to running out of disk space | |
| MAX_JOBS=1 | |
| # avoid "fatal: detected dubious ownership in repository at ..." | |
| sudo chown -R root:wheel "${OVERLAY_DIR}" | |
| # create an overlay | |
| sudo poudriere ports -c -p overlay -B "${{ env.BRANCH_NAME }}" -m git+file -U "${OVERLAY_DIR}" | |
| poudriere ports -l | |
| # build pkg, which creates data/packages/${JAIL_NAME}-${PORTS_NAME}/All | |
| # so that packages cache can be extracted under the directory. | |
| # the `All` directory is a symlink and computed by poudriere. | |
| sudo poudriere bulk -j "${JAIL_NAME}" -b latest -J "${MAX_JOBS}" pkg | |
| tree "/usr/local/poudriere/data/packages/${JAIL_NAME}-${PORTS_NAME}/All" | |
| # pre-fill packages cache | |
| if [ -d packages ]; then | |
| ls -al packages | |
| cp -R packages/* "/usr/local/poudriere/data/packages/${JAIL_NAME}-${PORTS_NAME}/All/" | |
| rm -rf packages | |
| fi | |
| # build the ports with the overlay (-o) and binary packages from | |
| # PACKAGE_FETCH_URL (-b). Test the port with Q/A checks (-Ct). use | |
| # `true` to prevent failure on build failures. the job will fail | |
| # later if any build fails. limit the number of job to 2. | |
| sudo poudriere bulk -j "${JAIL_NAME}" -O overlay -b latest -J "${MAX_JOBS}" -Ct ${PORT_TO_BUILD} || true | |
| # collect the logs. freebsd-vm will copy them back to the host OS. | |
| tree "/usr/local/poudriere/data/logs/bulk/${JAIL_NAME}-${PORTS_NAME}/latest/logs" | |
| sudo cp -R "/usr/local/poudriere/data/logs/bulk/${JAIL_NAME}-${PORTS_NAME}/latest/logs" ${OVERLAY_DIR}/logs | |
| # collect the built packages | |
| sudo cp -RL "/usr/local/poudriere/data/packages/${JAIL_NAME}-${PORTS_NAME}/All" "${OVERLAY_DIR}/packages" | |
| df -h -tufs | |
| - name: Create a safe port name from matrix.PORT | |
| shell: sh | |
| run: | | |
| SAFE_PORT_NAME=`echo -n "${{ matrix.PORT }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g'` | |
| echo "SAFE_PORT_NAME=${SAFE_PORT_NAME}" >> "${GITHUB_ENV}" | |
| - name: upload logs | |
| id: logs-upload-step | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-${{ env.SAFE_PORT_NAME }} | |
| path: logs/ | |
| - name: upload packages | |
| id: packages-upload-step | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: packages-${{ env.SAFE_PORT_NAME }} | |
| path: packages/ | |
| - name: Fail if any build failed | |
| shell: sh | |
| run: | | |
| tree . | |
| # logs under `errors` are symlink, not file | |
| ERROR_LOG_FILES=`find logs/errors -type l -name '*.log'` | |
| echo "ERROR_LOG_FILES: ${ERROR_LOG_FILES}" | |
| FAILED_PACKAGES="" | |
| for F in ${ERROR_LOG_FILES}; do | |
| FAILED_PACKAGE=`basename "${F}" | sed -e 's/\.log//'` | |
| FAILED_PACKAGES="${FAILED_PACKAGES} ${FAILED_PACKAGE}" | |
| done | |
| echo "${FAILED_PACKAGE}" > failed_packages.txt | |
| echo "===> Failed package" | |
| cat failed_packages.txt | |
| echo "Logs are available in an artifact. ID: ${{ steps.logs-upload-step.outputs.artifact-id }}" | |
| echo "Packages are available in an artifact. ID: ${{ steps.packages-upload-step.outputs.artifact-id }}" | |
| EXIT_STATUS=0 | |
| if [ ! -z "${FAILED_PACKAGES}" ]; then | |
| EXIT_STATUS=1 | |
| fi | |
| exit "${EXIT_STATUS}" | |
| store-packages: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build | |
| steps: | |
| - name: Download built packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: packages-* | |
| merge-multiple: true | |
| - name: List package cache | |
| shell: sh | |
| run: | | |
| tree packages | |
| - name: Upload packages cache for future builds | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: packages-cache | |
| path: packages |