-
Notifications
You must be signed in to change notification settings - Fork 110
143 lines (123 loc) · 6.19 KB
/
ion-java-performance-regression-detector.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
name: Ion Java performance regression detector
on:
pull_request:
paths:
- 'src/**'
permissions:
contents: read
env:
data_size: '50000'
test_data_id: 'generated-test-data'
jobs:
generate-test-data:
name: Generate Test Data
runs-on: ubuntu-latest
steps:
- name: Checkout ion-data-generator
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: amazon-ion/ion-data-generator
ref: main
- name: Build ion-data-generator
run: mvn clean install
- name: Generate test data
env:
jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
schema_dir: tst/com/amazon/ion/workflow
run: |
mkdir -p testData
for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
do
java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
done
- name: Upload test Ion Data to artifacts
uses: actions/upload-artifact@v4
with:
name: ${{env.test_data_id}}
path: testData
select-runner-type:
# If you want to use codebuild runners for your personal fork, follow the instructions to set
# up a codebuild project. https://docs.aws.amazon.com/codebuild/latest/userguide/action-runner.html
# Then, create a repository variable for your fork named `CODEBUILD_PROJECT_NAME` with the name
# of the project you created.
name: Select Runner Type
runs-on: ubuntu-latest
# We want to run on external PRs, but not on internal ones as push automatically builds
# H/T: https://github.com/Dart-Code/Dart-Code/commit/612732d5879730608baa9622bf7f5e5b7b51ae65
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'amazon-ion/ion-java'
env:
use-codebuild: ${{ vars.CODEBUILD_PROJECT_NAME != '' || github.repository_owner == 'amazon-ion' }}
codebuild-project-name: ${{ vars.CODEBUILD_PROJECT_NAME != '' && vars.CODEBUILD_PROJECT_NAME || 'ion-java' }}
al2-arm: "codebuild-${{ vars.CODEBUILD_PROJECT_NAME != '' && vars.CODEBUILD_PROJECT_NAME || 'ion-java' }}-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large"
outputs:
runner-type: ${{ env.use-codebuild && env.al2_arm || 'ubuntu-latest' }}
steps:
- name: Dump Config
run: echo '${{ toJSON(env) }}'
detect-regression:
name: Detect Regression
runs-on: ${{ needs.select-runner-type.outputs.runner-type }}
needs: [ generate-test-data, select-runner-type ]
strategy:
matrix:
test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
usage-patterns: [' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming ',' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming --io-type buffer --ion-length-preallocation 1 ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom --io-type buffer --ion-length-preallocation 1 ']
fail-fast: false
steps:
- name: Set up JDK 17
uses: actions/setup-java@v4.2.2
with:
distribution: 'corretto'
java-version: 17
- name: Checkout ion-java-benchmark-cli
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: amazon-ion/ion-java-benchmark-cli
ref: master
path: ion-java-benchmark-cli
- name: Checkout ion-java from the previous commit
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ github.base_ref }}
submodules: recursive
path: baseline
- name: Checkout ion-java from the new commit.
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.head_ref }}
submodules: recursive
path: new
- name: Download test Ion Data from artifacts
uses: actions/download-artifact@v4
with:
name: generated-test-data
path: testData
# Get the benchmark results from the base commit
- name: Build ion-java from the base commit
working-directory: baseline
run: |
./gradlew clean publishToMavenLocal
- name: Benchmark ion-java from the base commit
working-directory: ion-java-benchmark-cli
run: |
mvn clean install
mkdir /home/runner/work/ion-java/ion-java/benchmarkresults
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n
- name: Build ion-java from the new commit
working-directory: new
run: |
./gradlew clean publishToMavenLocal
- name: Benchmark ion-java from the new commit
working-directory: ion-java-benchmark-cli
run: |
mvn clean install
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n
#Detect regression
- name: Detect regression
working-directory: ion-java-benchmark-cli
run: |
a=$(java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion --benchmark-result-new /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion)
if [ ! -z "$a" ]; then echo "${a}" >> $GITHUB_STEP_SUMMARY; exit 1; fi