Skip to content

Merge branch 'update-unifrac' of github.com:qiime2/distributions into… #1

Merge branch 'update-unifrac' of github.com:qiime2/distributions into…

Merge branch 'update-unifrac' of github.com:qiime2/distributions into… #1

on:
workflow_call:
outputs:
epoch:
value: ${{ jobs.detect-vars.outputs.epoch }}
distro:
value: ${{ jobs.detect-vars.outputs.distro }}
is-release:
value: ${{ jobs.detect-vars.outputs.is-release }}
sibling-ref:
value: ${{ jobs.detect-vars.outputs.sibling-ref }}
matrix-key:
value: ${{ jobs.prepare.outputs.matrix-key }}
rev-deps-key:
value: ${{ jobs.prepare.outputs.rev-deps-key }}
generation-matrix:
value: ${{ jobs.prepare.outputs.generation-matrix }}
patched-channel-key:
value: ${{ jobs.prepare.outputs.patched-channel-key }}
rebuilt-channel-key:
value: ${{ jobs.prepare.outputs.rebuilt-channel-key }}
seed-environment-key:
value: ${{ jobs.prepare.outputs.seed-environment-key }}
solved-environment-key:
value: ${{ jobs.prepare.outputs.solved-environment-key }}
repodata-patches-key:
value: ${{ jobs.prepare.outputs.repodata-patches-key }}
env:
base_branch: 'dev'
matrix_key: '[GHA Matrix] Packages to rebuild and test'
matrix_path: './matrix'
partial_patched_channel_key: '[Conda Channel] Mirror with patches' # + (HASH)
patched_channel_path: './patched-channel'
rev_deps_key: '[Reverse Dependencies] Packages to rebuild and their reverse dependencies'
rev_deps_path: './rev-deps'
repodata_patches_key: '[Repodata Patches] patch_instructions.json'
repodata_patches_path: './repodata-patches'
rebuilt_channel_key: '[Conda Channel] Rebuilt packages'
rebuilt_channel_path: './rebuilt-channel'
seed_environment_key: '[Environment File] Metapackage seed environment'
seed_environment_path: './seed-env'
solved_environment_key: '[Environment File] Metapackage solved environment'
jobs:
detect-vars:
runs-on: ubuntu-latest # this job is always run on linux
outputs:
epoch: ${{ steps.set-vars.outputs.epoch }}
distro: ${{ steps.set-vars.outputs.distro }}
seed-env: ${{ steps.set-vars.outputs.seed-env }}
is-release: ${{ steps.set-vars.outputs.is-release }}
is-prepare: ${{ steps.set-vars.outputs.is-prepare }}
is-rebuild: ${{ steps.set-vars.outputs.is-rebuild }}
sibling-ref: ${{ steps.set-vars.outputs.sibling-ref }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 2
# TODO: fork this to prevent supply chain attack
- uses: tj-actions/changed-files@v35
id: changed
- name: set-vars
id: set-vars
run: |
# Parse the git changes to figure out which epoch triggered this workflow.
CHANGED="${{ steps.changed.outputs.all_changed_files }}"
EPOCH=$(python .github/workflows/bin/determine.py --epoch "$CHANGED")
DISTRO=$(python .github/workflows/bin/determine.py --distro "$CHANGED")
RELEASE=${{ startsWith(github.head_ref, 'Release-') }}
PREPARE=${{ startsWith(github.head_ref, 'Prepare-') }}
REBUILD=$([ $RELEASE = 'true' ] || [ $PREPARE = 'true' ] && echo 'true' || echo 'false')
SUBDIR=$([ $RELEASE = 'true' ] && echo 'released' || echo 'passed')
SIBLING_REF=$([ $RELEASE = 'true' ] && echo "$GITHUB_HEAD_REF" || echo "")
echo "epoch=$EPOCH" >> $GITHUB_OUTPUT
echo "distro=$DISTRO" >> $GITHUB_OUTPUT
echo "seed-env=$EPOCH/$DISTRO/$SUBDIR/seed-environment-conda.yml" >> $GITHUB_OUTPUT
echo "is-release=$RELEASE" >> $GITHUB_OUTPUT
echo "is-prepare=$PREPARE" >> $GITHUB_OUTPUT
echo "is-rebuild=$REBUILD" >> $GITHUB_OUTPUT
echo "sibling-ref=$SIBLING_REF" >> $GITHUB_OUTPUT
# this job will parse a diff from cbc, and use that to generate a DAG
prepare:
needs: detect-vars
runs-on: ubuntu-latest
outputs:
rev-deps-key: ${{ env.rev_deps_key }}
# patched-channel-key includes a hash which is calculated in this step
patched-channel-key: ${{ steps.patched-channel-restore.outputs.cache-primary-key }}
rebuilt-channel-key: ${{ env.rebuilt_channel_key }}
seed-environment-key: ${{ env.seed_environment_key }}
solved-environment-key: ${{ env.solved_environment_key }}
repodata-patches-key: ${{ env.repodata_patches_key }}
matrix-key: ${{ env.matrix_key }}
generation-matrix: ${{ steps.store-generations.outputs.generation-matrix }}
steps:
- name: checkout source
uses: actions/checkout@v3
with:
fetch-depth: '0'
- name: set up python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: install dependencies
run: |
pip install pyyaml jinja2 networkx conda-mirror ghapi
conda install -q conda-build
- name: make matrix dir
run: mkdir ${{ env.matrix_path }}
- name: make rev deps dir
run: mkdir ${{ env.rev_deps_path }}
- uses: ./.github/actions/find-distro-packages
id: distro-pkgs
with:
seed-env: ${{ needs.detect-vars.outputs.seed-env }}
- uses: ./.github/actions/find-sibling-branches
id: siblings
with:
repos: ${{ steps.distro-pkgs.outputs.repos }}
is-rebuild: ${{ needs.detect-vars.outputs.is-rebuild }}
- name: get diff
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' }}
run: |
git fetch origin ${{ env.base_branch }}:${{ env.base_branch }}
git show ${{ env.base_branch }}:${{ needs.detect-vars.outputs.seed-env }} > original-env.yml
- uses: ./.github/actions/find-changed-plugins
id: changed-plugins
with:
old-seed-env: ${{ needs.detect-vars.outputs.is-rebuild != 'true' && 'original-env.yml' || '' }}
new-seed-env: ${{ needs.detect-vars.outputs.seed-env }}
- uses: qiime2/action-library-packaging/make-conda-config@beta
id: find-channels
with:
seed-environment: ${{ needs.detect-vars.outputs.seed-env }}
conda-build-config: /dev/null
- uses: ./.github/actions/make-dag
with:
search-channels: ${{ steps.find-channels.outputs.channels }}
epoch: ${{ steps.detect-vars.outputs.epoch }}
distro: ${{ steps.detect-vars.outputs.distro }}
changed: ${{ steps.changed-plugins.outputs.all-changes }}
rebuild: ${{ steps.siblings.outputs.repos }}
distro-versions: ${{ steps.distro-pkgs.outputs.versions }}
env-versions: ${{ steps.distro-pkgs.outputs.all }}
matrix-path: ${{ env.matrix_path }}
rev-deps-path: ${{ env.rev_deps_path }}
- name: hash of distro
id: distro-hash
shell: bash
run: |
cat <<EOF > full_distro.json
${{ steps.distro-pkgs.outputs.versions }}
EOF
echo hash=$(
shasum -a 256 -U full_distro.json | cut -f1 -d ' '
) >> $GITHUB_OUTPUT
rm full_distro.json
- uses: actions/cache/restore@v3
id: patched-channel-restore
with:
key: "${{ env.partial_patched_channel_key }} (${{ steps.distro-hash.outputs.hash }})"
path: ${{ env.patched_channel_path }}
- name: setup patched channel
if: steps.patched-channel-restore.outputs.cache-hit != 'true'
shell: bash
run: |
mkdir ${{ env.patched_channel_path }}
conda index ${{ env.patched_channel_path }}
- name: mirror patched channel
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' && steps.patched-channel-restore.outputs.cache-hit != 'true' }}
uses: qiime2/action-library-packaging/mirror-channel@beta
with:
packages: ${{ steps.distro-pkgs.outputs.versions }}
local-channel: ${{ env.patched_channel_path }}
remote-channel: ${{ fromJSON(steps.find-channels.outputs.channels)[0] }}
- name: collect versions for patches
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' }}
uses: ./.github/actions/collect-versions
with:
packages: ${{ steps.distro-pkgs.outputs.all }}
changed: ${{ steps.changed-plugins.outputs.all-changes }}
versions-path: versions.txt
- name: apply patches
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' }}
uses: qiime2/action-library-packaging/patch-repodata@beta
with:
channel: ${{ env.patched_channel_path }}
rev-deps: ${{ env.rev_deps_path }}/rev_deps.json
versions-file: versions.txt
run: |
cat versions.txt
- name: upload reverse dependencies
uses: actions/upload-artifact@v3
with:
name: ${{ env.rev_deps_key }}
path: ${{ env.rev_deps_path }}
- name: upload patched channel
uses: actions/upload-artifact@v3
with:
name: ${{ steps.patched-channel-restore.outputs.cache-primary-key }}
path: ${{ env.patched_channel_path }}
- uses: actions/cache/save@v3
id: patched-channel-save
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' && steps.patched-channel-restore.outputs.cache-hit != 'true' }}
with:
key: ${{ steps.patched-channel-restore.outputs.cache-primary-key }}
path: ${{ env.patched_channel_path }}
- name: store matrices
uses: actions/upload-artifact@v3
with:
name: ${{ env.matrix_key }}
path: ${{ env.matrix_path }}
- name: store generations
id: store-generations
run: |
echo "generation-matrix=$(cat ${{ env.matrix_path }}/rebuild_matrix.json)" >> $GITHUB_OUTPUT
- name: copy repodata patches
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' }}
run: |
mkdir -p ${{ env.repodata_patches_path }}/linux-64
mkdir -p ${{ env.repodata_patches_path }}/osx-64
mkdir -p ${{ env.repodata_patches_path }}/noarch
cp ${{ env.patched_channel_path }}/linux-64/patch_instructions.json \
${{ env.repodata_patches_path }}/linux-64/patch_instructions.json
cp ${{ env.patched_channel_path }}/osx-64/patch_instructions.json \
${{ env.repodata_patches_path }}/osx-64/patch_instructions.json
cp ${{ env.patched_channel_path }}/noarch/patch_instructions.json \
${{ env.repodata_patches_path }}/noarch/patch_instructions.json
- name: store repodata patches
if: ${{ needs.detect-vars.outputs.is-rebuild != 'true' }}
uses: actions/upload-artifact@v3
with:
name: ${{ env.repodata_patches_key }}
path: ${{ env.repodata_patches_path }}
- name: create rebuilt channel
id: rebuilt-channel
run: |
mkdir ${{ env.rebuilt_channel_path }}
conda index ${{ env.rebuilt_channel_path }}
- name: store rebuilt channel
uses: actions/upload-artifact@v3
with:
name: ${{ env.rebuilt_channel_key }}
path: ${{ env.rebuilt_channel_path }}
- name: create seed env dir
id: seed-env-dir
run: |
mkdir ${{ env.seed_environment_path }}
cp ${{ needs.detect-vars.outputs.seed-env }} ${{ env.seed_environment_path }}
- name: store seed env
uses: actions/upload-artifact@v3
with:
name: ${{ env.seed_environment_key }}
path: ${{ env.seed_environment_path }}