More updates to the OSB build (see previous commit 6f0ae23).

Signed-off-by: Govind Kamat <govkamat@amazon.com>

.ci/build.sh

@@ -1,22 +1,20 @@
 #!/usr/bin/env bash
 
-pyenv_init() {
- PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin
-}
-
 function setup {
- export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}"
- # this value is in bytes, the default is 50kB. We increase it to 200kiB.
- export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800}
- # adjust the default log level from WARNING
- export THESPLOG_THRESHOLD="INFO"
+ export BENCHMARK_HOME=$GITHUB_WORKSPACE
+
+ export THESPLOG_FILE=$BENCHMARK_HOME/.benchmark/logs/actor-system-internal.log
+ export THESPLOG_FILE_MAXSIZE=204800 # default is 50 KiB
+ export THESPLOG_THRESHOLD=INFO # default log level is WARNING
 
- pyenv_init
  export TERM=dumb
  export LC_ALL=en_US.UTF-8
+
+ # Init pyenv.
+ PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin
 }
 
-function build {
+function build_and_unit_test {
  setup
 
  set -e
@@ -25,11 +23,9 @@ function build {
  make test
 }
 
-function build_it {
+function run_it {
  setup
 
- export BENCHMARK_HOME="$GITHUB_WORKSPACE"
-
  docker pull ubuntu/squid:latest
 
  # make it38, it39, etc. so they run as concurrent GHA jobs

.github/workflows/integ-test.yml

@@ -42,6 +42,4 @@ jobs:
  - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
 
  - name: Run the CI build script
- run: bash .ci/build.sh build_it ${{ matrix.python-version }}
- env:
- BENCHMARK_HOME: env.GITHUB_WORKSPACE
+ run: bash .ci/build.sh run_it ${{ matrix.python-version }}

.github/workflows/release-drafter.yml → .github/workflows/publish-release.yml

@@ -1,12 +1,12 @@
-name: Release drafter
+name: Publish Release to GitHub
 
 on:
  push:
  tags:
  - "*"
 
 jobs:
- draft-a-release:
+ publish-release:
  runs-on: ubuntu-latest
  steps:
  - name: Checkout Repository
@@ -22,21 +22,20 @@ jobs:
  issue-title: 'Release opensearch-benchmark'
  issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }} **Commit**: ${{ github.sha }}"
  exclude-workflow-initiator-as-approver: true
+
  - name: Set up Python 3
  uses: actions/setup-python@v3
  with:
  python-version: '3.x'
- - name: Install build tools
- run: |
- python -m pip install --upgrade build
+
  - name: Build project for distribution
  run: |
- python -m build
- tar -zvcf artifacts.tar.gz dist
- - name: Release
+ make build
+ tar zcvf artifacts.tar.gz dist
+
+ - name: Publish release
  uses: softprops/action-gh-release@v1
  with:
  draft: true
  generate_release_notes: true
- files: |
- artifacts.tar.gz
+ files: artifacts.tar.gz

.github/workflows/unit-test.yml

@@ -17,6 +17,4 @@ jobs:
  run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv
 
  - name: Run the CI build script
- run: bash .ci/build.sh build
- env:
- BENCHMARK_HOME: env.GITHUB_WORKSPACE
+ run: bash .ci/build.sh build_and_unit_test

DEVELOPER_GUIDE.md

@@ -23,16 +23,29 @@ This document will walk you through on what's needed to start contributing code
 
 ### Prerequisites
 
- - **Pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding.
- For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation).
+ - **pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding.
+ For more details please refer to the [pyenv installation instructions](https://github.com/pyenv/pyenv#installation).
 
- **Optional Step:** For Debian-based systems, install the following modules to continue with the next steps: 
+ `pyenv` requires that the C compiler and development libraries be installed, so that the specified Python versions can be build from source. The installation instructions vary from platform to platform.
+
+ For Debian-based systems, install the following modules to continue with the next steps: 
  ```
  sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
  libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
  xz-utils tk-dev libffi-dev liblzma-dev git
  ```
 
+ For Amazon Linux 2023, run the following command
+ ```
+ sudo yum -y install gcc openssl-devel bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel zlib-devel xz-devel
+ ```
+
+ On the Mac platform, XCode needs to be installed as well as some additional required libraries:
+ ```
+ xcode-select --install
+ brew install pyenv jq zlib xz
+ ```
+
  - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters. JDK version 17 is used to build the current version of OpenSearch. Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites).
  Note that the `javadoc` executable should be available in the JDK installation. An earlier version of the JDK can be used, but not all the integration tests will pass.
 

Makefile

@@ -16,11 +16,14 @@
 # under the License.
 
 SHELL = /bin/bash
+PYTHON = python3
 PIP = pip3
 VERSIONS = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d')
 VERSION38 = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d' | grep 3\.8)
 PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv and run \033[0;31meval \"\$$(pyenv init -)\"\033[0m.\n"
 
+all: install-devel
+
 pyinst:
  @which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; }
  @for i in $(VERSIONS); do pyenv install --skip-existing $$i; done
@@ -41,22 +44,21 @@ check-java:
  fi
 
 install-deps: pyinst38
- # @if test `uname` = Darwin -o `python3 --version | sed 's/.* 3.\([0-9]*\).*/3\1/'` -lt 38; then make pyinst38; fi
- $(PIP) install --upgrade pip setuptools wheel
+ $(PIP) install --upgrade pip setuptools
 
 install-user: install-deps
  PIP_ONLY_BINARY=h5py $(PIP) install -e .
 
 install-devel: install-deps
- $(PIP) install -e .[develop]
+ PIP_ONLY_BINARY=h5py $(PIP) install -e .[develop]
 
-wheel:
- $(PIP) install --upgrade pip setuptools wheel
- PIP_ONLY_BINARY=h5py $(PIP) wheel .
+build:
+ $(PIP) install --upgrade build
+ $(PYTHON) -m build
 
-install: wheel
- PIP_ONLY_BINARY=h5py $(PIP) install opensearch_benchmark-*.whl
- rm -r *.whl *.egg-info
+install: build
+ PIP_ONLY_BINARY=h5py $(PIP) install dist/opensearch_benchmark-*.whl
+ rm -rf dist
 
 clean:
  rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt
@@ -75,7 +77,7 @@ tox-env-clean:
 lint:
  @find osbenchmark benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} +
 
-test:
+test: install-devel
  pytest tests/
 
 it: pyinst check-java python-caches-clean tox-env-clean

docker/Dockerfile

@@ -14,7 +14,10 @@ RUN apt-get -y update && \
 
 COPY . opensearch-benchmark
 
-RUN cd opensearch-benchmark; make wheel
+# There is no binary package currently available for yappi on ARM.
+RUN cd opensearch-benchmark; \
+ make build; \
+ if test "`uname -m`" = aarch64; then pip install yappi && cp "`pip cache list --format abspath | grep yappi`" dist; fi
 
 
 #
@@ -30,15 +33,14 @@ RUN groupadd --gid 1000 opensearch-benchmark && \
 RUN mkdir -p /opensearch-benchmark/.benchmark && \
  chown -R 1000:0 /opensearch-benchmark/.benchmark
 
-COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/yappi-*.whl /opensearch-benchmark/opensearch-benchmark/opensearch_benchmark-*.whl ./
+COPY --from=build-stage /opensearch-benchmark/opensearch-benchmark/dist/*.whl ./
 
-# There is no binary package currently available for yappi on ARM.
 RUN set -ex; \
  apt-get -y update; \
  apt-get install -y git pbzip2; \
  apt-get -y upgrade; \
  rm -rf /var/lib/apt/lists/*; \
- PIP_ONLY_BINARY=h5py pip install yappi-*.whl opensearch_benchmark-*.whl; \
+ PIP_ONLY_BINARY=h5py pip install *.whl; \
  rm *.whl
 
 USER 1000