Refactor IR to allow transformed values on the LHS of filter operations. #1580

Workflow file for this run

	name: CI

	on:
	pull_request:
	push:
	branches:
	- main
	workflow_dispatch:
	# Needed so we can run it manually

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	RUST_BACKTRACE: 1
	CARGO_TERM_COLOR: always

	jobs:
	ci-everything:
	name: All CI stages
	runs-on: ubuntu-latest
	needs:
	- lint
	- rust-tests
	- rust-fuzz
	- rust-rustdoc-wasm
	- python-tests
	- wasm-tests
	- js-lint
	if: ${{ success() \|\| failure() }} # Run this job even if a dependency has failed.
	steps:
	- name: Job outcomes
	run: \|
	echo "lint: ${{ needs.lint.result }}"
	echo "rust-tests: ${{ needs.rust-tests.result }}"
	echo "rust-fuzz: ${{ needs.rust-fuzz.result }}"
	echo "python-tests: ${{ needs.python-tests.result }}"
	echo "wasm-tests: ${{ needs.wasm-tests.result }}"
	echo "js-lint: ${{ needs.js-lint.result }}"
	echo "rust-rustdoc-wasm: ${{ needs.rust-rustdoc-wasm.result }}"

	# Fail this required job if any of its dependent jobs have failed.
	#
	# Do not attempt to consolidate these steps into one step, it won't work.
	# Multi-line `if` clauses are not evaluated properly: see the intermediate commits in
	# https://github.com/obi1kenobi/cargo-semver-checks/pull/405
	- if: ${{ needs.lint.result != 'success' }}
	run: exit 1
	- if: ${{ needs.rust-tests.result != 'success' }}
	run: exit 1
	- if: ${{ needs.rust-fuzz.result != 'success' }}
	run: exit 1
	- if: ${{ needs.python-tests.result != 'success' }}
	run: exit 1
	- if: ${{ needs.wasm-tests.result != 'success' }}
	run: exit 1
	- if: ${{ needs.js-lint.result != 'success' }}
	run: exit 1
	- if: ${{ needs.rust-rustdoc-wasm.result != 'success' }}
	run: exit 1

	python-tests:
	name: Python tests and maturin build
	runs-on: ubuntu-latest
	needs:
	- rust-tests
	strategy:
	matrix:
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust + caching
	uses: actions-rust-lang/setup-rust-toolchain@v1

	- name: Set up python
	id: setup-python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	virtualenvs-create: true
	virtualenvs-in-project: true
	installer-parallel: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v4
	with:
	path: pytrustfall/.venv
	key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
	run: \|
	cd pytrustfall
	poetry install --no-interaction --no-root

	- name: maturin build and test
	run: \|
	cd pytrustfall
	source .venv/bin/activate
	maturin develop
	pytest
	maturin build --interpreter python -o target/wheels/

	- name: Set environment variables
	id: branch-info
	run: \|
	set -ex
	echo "COMMIT=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
	echo "TAG_NAME=$(.github/workflows/get_py_prerelease_name.sh)" >> $GITHUB_OUTPUT
	echo "PYTHON_BINDINGS_CHANGED=$(.github/workflows/python_bindings_changed.sh)" >> $GITHUB_OUTPUT

	- name: Upload artifact
	uses: actions/upload-artifact@v4
	if: steps.branch-info.outputs.PYTHON_BINDINGS_CHANGED != 0
	with:
	name: pytrustfall-wheels
	path: pytrustfall/target/wheels/
	retention-days: 1

	- name: Make a new prerelease
	uses: ncipollo/release-action@v1
	if: steps.branch-info.outputs.BRANCH_NAME == 'main' && steps.branch-info.outputs.PYTHON_BINDINGS_CHANGED != 0
	with:
	artifacts: "pytrustfall/target/wheels/.whl,pytrustfall/target/wheels/.tar.gz"
	commit: ${{ steps.branch-info.outputs.COMMIT }}
	tag: ${{ steps.branch-info.outputs.TAG_NAME }}
	generateReleaseNotes: true
	prerelease: true
	token: ${{ secrets.GITHUB_TOKEN }}

	lint:
	name: Check lint and rustfmt
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust + caching
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	components: rustfmt, clippy

	- name: cargo clippy
	run: cargo clippy --workspace --all-features --all-targets --no-deps -- -D warnings --allow deprecated

	- name: cargo fmt
	run: cargo fmt -- --check

	- name: cargo doc
	env:
	RUSTDOCFLAGS: -D warnings
	run: cargo doc --workspace --all-features --no-deps --document-private-items

	rust-tests:
	name: Run tests
	runs-on: ubuntu-latest
	env:
	CARGO_INCREMENTAL: 0
	SCCACHE_GHA_ENABLED: "true"
	RUSTC_WRAPPER: "sccache"
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: true

	- name: Install rust + caching
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	rustflags: ""

	- name: Set up sccache
	uses: mozilla-actions/sccache-action@v0.0.4

	- name: Clear sccache stats
	run: sccache --zero-stats

	# Test all crates except trustfall_stubgen,
	# which is only tested if it has changed since its tests are a bit long.
	#
	# `--all-targets` also tests example and bench targets, but explicitly excludes doctests.
	# We'll test doctests separately in their own step.
	# https://github.com/rust-lang/cargo/issues/6669
	- name: cargo test
	run: cargo test --workspace --all-targets --all-features --exclude trustfall_stubgen

	# Run doctests separately, since `--all-targets` above explicitly excludes doctests.
	# https://github.com/rust-lang/cargo/issues/6669
	- name: run doctests
	run: cargo test --workspace --all-features --doc

	- name: test trustfall_stubgen if it has changed
	run: \|
	git fetch origin main

	# `git diff --quiet` exits non-zero if there are changes
	git diff --quiet HEAD origin/main -- ./trustfall_stubgen \|\| (cd trustfall_stubgen/ && cargo test --all-features)

	- name: Show sccache stats
	run: sccache --show-stats

	rust-fuzz:
	name: Check fuzz targets
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache: false

	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: \|
	./trustfall_core/fuzz

	- name: cargo clippy
	run: \|
	cd trustfall_core/fuzz
	cargo clippy --workspace --all-features --all-targets --no-deps -- -D warnings --allow deprecated

	- name: cargo fmt
	run: \|
	cd trustfall_core/fuzz
	cargo fmt -- --check

	- name: cargo doc
	env:
	RUSTDOCFLAGS: -D warnings
	run: \|
	cd trustfall_core/fuzz
	cargo doc --workspace --all-features --no-deps --document-private-items

	rust-rustdoc-wasm:
	name: Check rustdoc WASM target
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache: false

	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: \|
	./experiments/trustfall_rustdoc

	- name: cargo clippy
	run: \|
	cd experiments/trustfall_rustdoc
	cargo clippy --all-features --all-targets --no-deps -- -D warnings --allow deprecated

	- name: cargo fmt
	run: \|
	cd experiments/trustfall_rustdoc
	cargo fmt -- --check

	- name: cargo doc
	env:
	RUSTDOCFLAGS: -D warnings
	run: \|
	cd experiments/trustfall_rustdoc
	cargo doc --all-features --no-deps --document-private-items

	wasm-tests:
	name: WASM tests
	runs-on: ubuntu-latest
	needs:
	- rust-tests
	permissions:
	contents: write
	env:
	CARGO_INCREMENTAL: 0
	SCCACHE_GHA_ENABLED: "true"
	RUSTC_WRAPPER: "sccache"
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust + caching
	uses: actions-rust-lang/setup-rust-toolchain@v1

	- name: Install wasm-pack
	run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf \| sh

	- name: Set up sccache
	uses: mozilla-actions/sccache-action@v0.0.4

	- name: Clear sccache stats
	run: sccache --zero-stats

	- name: Run cargo test and wasm-pack tests
	run: \|
	cd trustfall_wasm
	wasm-pack test --headless --firefox
	wasm-pack test --headless --chrome

	- name: Set environment variables
	id: branch-info
	run: \|
	set -ex
	echo "COMMIT=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
	echo "TAG_NAME=$(.github/workflows/get_wasm_prerelease_name.sh)" >> $GITHUB_OUTPUT
	echo "WASM_CHANGED=$(.github/workflows/wasm_pkg_changed.sh)" >> $GITHUB_OUTPUT

	- name: Build WASM module
	if: steps.branch-info.outputs.WASM_CHANGED != 0
	run: \|
	cd trustfall_wasm
	wasm-pack build --no-typescript
	cp src/trustfall_wasm.d.ts pkg/
	cd pkg/
	tar -czvf ../trustfall_wasm.tar.gz --exclude='.gitignore' .

	- name: Upload artifact
	uses: actions/upload-artifact@v4
	if: steps.branch-info.outputs.WASM_CHANGED != 0
	with:
	name: wasm-module
	path: trustfall_wasm/pkg/
	retention-days: 1

	- name: Make a new prerelease
	uses: ncipollo/release-action@v1
	if: steps.branch-info.outputs.BRANCH_NAME == 'main' && steps.branch-info.outputs.WASM_CHANGED != 0
	with:
	artifacts: "trustfall_wasm/trustfall_wasm.tar.gz"
	commit: ${{ steps.branch-info.outputs.COMMIT }}
	tag: ${{ steps.branch-info.outputs.TAG_NAME }}
	generateReleaseNotes: true
	prerelease: true
	token: ${{ secrets.GITHUB_TOKEN }}

	- name: Show sccache stats
	run: sccache --show-stats

	js-lint:
	name: Run JS linters
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: experiments/schemaless_wasm/www/
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust + caching # necessary for `npm run build:wasm` later
	uses: actions-rust-lang/setup-rust-toolchain@v1

	- uses: actions/setup-node@v4
	with:
	node-version: 20

	- name: Install dependencies
	run: npm ci

	- name: Build wasm package
	run: npm run build:wasm

	- name: Run linters
	run: npm run lint

	pre-publish-checks:
	name: pre-publish checks
	if: github.ref == 'refs/heads/main'
	needs:
	- ci-everything
	- check-trustfall-examples-and-expensive-tests
	runs-on: ubuntu-latest
	steps:
	- run: exit 0

	check-trustfall-examples-and-expensive-tests:
	name: pre-publish checks
	if: github.ref == 'refs/heads/main'
	needs:
	- rust-tests
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Install rust
	uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache: false
	rustflags: ""

	- name: Publishing?
	id: version
	run: \|
	set +e
	.github/workflows/is_crate_version_already_uploaded.sh trustfall
	export TRUSTFALL="$?"
	.github/workflows/is_crate_version_already_uploaded.sh trustfall_core
	export TRUSTFALL_CORE="$?"
	.github/workflows/is_crate_version_already_uploaded.sh trustfall_derive
	export TRUSTFALL_DERIVE="$?"
	if [[ "$TRUSTFALL" == "7" && "$TRUSTFALL_CORE" == "7" && "$TRUSTFALL_DERIVE" == "7" ]]; then
	echo 'is_new_version=no' >> $GITHUB_OUTPUT
	elif [[ "$TRUSTFALL" == "0" \|\| "$TRUSTFALL_CORE" == "0" \|\| "$TRUSTFALL_DERIVE" == "0" ]]; then
	echo 'is_new_version=yes' >> $GITHUB_OUTPUT
	else
	# Unexpected outcome, indicates a bug.
	exit 1
	fi

	- uses: Swatinem/rust-cache@v2
	if: steps.version.outputs.is_new_version == 'yes'

	- name: feeds example
	if: steps.version.outputs.is_new_version == 'yes'
	run: \|
	cd trustfall
	cargo run --example feeds refresh
	cargo run --example feeds query ./examples/feeds/example_queries/feed_content.ron
	cargo run --example feeds query ./examples/feeds/example_queries/feed_links.ron
	cargo run --example feeds query ./examples/feeds/example_queries/game_reviews.ron

	- name: hackernews example
	if: steps.version.outputs.is_new_version == 'yes'
	run: \|
	cd trustfall
	cargo run --example hackernews query ./examples/hackernews/example_queries/front_page_stories_with_links.ron 1
	cargo run --example hackernews query ./examples/hackernews/example_queries/latest_links_by_high_karma_users.ron 1

	- name: weather example
	if: steps.version.outputs.is_new_version == 'yes'
	run: \|
	cd trustfall
	cargo run --example weather refresh
	cargo run --example weather query ./examples/weather/example_queries/boston_weather.ron
	cargo run --example weather query ./examples/weather/example_queries/high_winds.ron

	- name: trustfall_stubgen tests
	if: steps.version.outputs.is_new_version == 'yes'
	run: \|
	cd trustfall_stubgen
	cargo test --all-features --all-targets

	attempt-publish-trustfall-filetests-macros:
	name: publish trustfall_filetests_macros
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	uses: ./.github/workflows/publish-rust.yml
	with:
	crate-name: trustfall_filetests_macros
	secrets:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

	attempt-publish-trustfall-core:
	name: publish trustfall_core
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	- attempt-publish-trustfall-filetests-macros
	uses: ./.github/workflows/publish-rust.yml
	with:
	crate-name: trustfall_core
	secrets:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

	attempt-publish-trustfall-derive:
	name: publish trustfall_derive
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	- attempt-publish-trustfall-core
	uses: ./.github/workflows/publish-rust.yml
	with:
	crate-name: trustfall_derive
	secrets:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

	attempt-publish-trustfall:
	name: publish trustfall
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	- attempt-publish-trustfall-core
	- attempt-publish-trustfall-derive
	uses: ./.github/workflows/publish-rust.yml
	with:
	crate-name: trustfall
	secrets:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

	attempt-publish-trustfall-stubgen:
	name: publish trustfall_stubgen
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	- attempt-publish-trustfall
	uses: ./.github/workflows/publish-rust.yml
	with:
	crate-name: trustfall_stubgen
	secrets:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

	attempt-publish-pytrustfall:
	name: publish python trustfall
	if: github.ref == 'refs/heads/main'
	needs:
	- pre-publish-checks
	- attempt-publish-trustfall-core
	uses: ./.github/workflows/publish-python.yml

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Refactor IR to allow transformed values on the LHS of filter operations. #1580

Workflow file

Refactor IR to allow transformed values on the LHS of filter operations. #1580

Jobs

Run details

Workflow file for this run