diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml new file mode 100644 index 000000000..c01da73d0 --- /dev/null +++ b/.github/workflows/github-ci.yml @@ -0,0 +1,63 @@ +# Copyright (C) 2021 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helib +on: + # By default this will run when the activity type is "opened", "synchronize", + # or "reopened". + pull_request: + branches: + - master + # Manually run this workflow on any specified branch. + workflow_dispatch: + + +################### +# Define env vars # +################### +env: + INSTALL_PREFIX: $GITHUB_WORKSPACE + +jobs: + library_build: + runs-on: '${{ matrix.os }}' + name: 'os=${{ matrix.os }} package_build=${{ matrix.package_build }} hexl=${{ matrix.hexl }} compiler=${{ matrix.cxx_compiler}}' + defaults: + run: + shell: bash + strategy: + matrix: + # TODO: shared builds + package_build: [ON, OFF] + hexl: [ON, OFF] + c_compiler: [gcc, clang] + os: [macos-latest, ubuntu-20.04] + include: # Use g++ with gcc only and clang++ with clang only + - c_compiler: gcc + cxx_compiler: g++ + - c_compiler: clang + cxx_compiler: clang++ + exclude: # Skip HEXL package build + - package_build: ON + hexl: ON + steps: + - uses: actions/checkout@v2 + - run: | + set -x + env + ./ci/install_deps.sh "${{ matrix.package_build }}" "${{ matrix.os }}" \ + "${{ matrix.c_compiler }}" "${{ matrix.cxx_compiler }}" "${{ matrix.hexl }}" + ./ci/build_install_lib.sh "${{ matrix.package_build }}" ${{ env.INSTALL_PREFIX }} \ + ${{ matrix.c_compiler }} ${{ matrix.cxx_compiler }} \ + ${{ matrix.hexl }} "./hexl/lib/cmake/hexl-1.2.1" + ./ci/test_lib.sh + ./ci/build_test_consumer.sh "examples" "${{ matrix.package_build }}" ${{ env.INSTALL_PREFIX }} + ./ci/build_test_consumer.sh "utils" "${{ matrix.package_build }}" ${{ env.INSTALL_PREFIX }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eb54bcc9e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (C) 2020-2021 IBM Corp. -# This program is Licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -language: cpp -notifications: - email: false - -branches: - only: - - master - -cache: ccache - -jobs: - include: - - name: "Ubuntu 18.04 lib build gcc" - os: linux - dist: bionic - compiler: gcc - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "Ubuntu 20.04 lib build gcc" - os: linux - dist: focal - compiler: gcc - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "Ubuntu 18.04 lib build clang" - os: linux - dist: bionic - compiler: clang - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "Ubuntu 20.04 lib build clang" - os: linux - dist: focal - compiler: clang - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "Ubuntu 18.04 package build gcc" - os: linux - dist: bionic - compiler: gcc - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - name: "Ubuntu 20.04 package build gcc" - os: linux - dist: focal - compiler: gcc - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - name: "Ubuntu 18.04 package build clang" - os: linux - dist: bionic - compiler: clang - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - name: "Ubuntu 20.04 package build clang" - os: linux - dist: focal - compiler: clang - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - name: "MacOS Mojave lib build" - os: osx - osx_image: xcode11.3 - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "MacOS Mojave package build" - os: osx - osx_image: xcode11.3 - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - name: "MacOS Catalina lib build" - os: osx - osx_image: xcode12 - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=OFF - - name: "MacOS Catalina package build" - os: osx - osx_image: xcode12 - env: INSTALL_PREFIX="~/helib_install" PACKAGE_BUILD=ON - - -addons: - apt: - packages: - - g++ - - clang - - patchelf - - cmake - - bats - update: true - homebrew: - packages: - - cmake - - bats-core - - ccache -# update: true -# Uncomment this line if the latest update of homebrew is required. This will -# increase the overall build time of each platform. - -# activating ccache on MacOS. -before_install: - - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - export PATH="/usr/local/opt/ccache/libexec:$PATH"; - fi - -install: - - ./ci/install_deps.sh - -script: - - ./ci/build_install_lib.sh "${PACKAGE_BUILD}" "${INSTALL_PREFIX}" - - ./ci/test_lib.sh - - ./ci/build_test_consumer.sh "examples" "${PACKAGE_BUILD}" "${INSTALL_PREFIX}" - - ./ci/build_test_consumer.sh "utils" "${PACKAGE_BUILD}" "${INSTALL_PREFIX}" - diff --git a/CHANGES.md b/CHANGES.md index 734e82883..07b01de30 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,15 @@ Release Changes =============== +HElib 2.2.1, October 2021 +========================= +(tagged as v2.2.1) + +* Improved NTT bechmark code +* CI update +* Improvements to partial match logic +* Bug fixes + HElib 2.2.0, September 2021 ========================= (tagged as v2.2.0) diff --git a/benchmarks/fft_bench.cpp b/benchmarks/fft_bench.cpp index ff0b0e331..c38377e51 100644 --- a/benchmarks/fft_bench.cpp +++ b/benchmarks/fft_bench.cpp @@ -34,14 +34,13 @@ static void helib_fft_forward(benchmark::State& state, Meta& meta) long q = prime_generator.next(); helib::Cmodulus cmod(zms, q, 0); - NTL::ZZX poly; - poly.SetLength(N); - for (long i = 0; i < N; ++i) - poly[i] = i; + NTL::zz_pX poly(N, 1); - NTL::vec_long transformed; - for (auto _ : state) + NTL::vec_long transformed(NTL::INIT_SIZE, N); + + for (auto _ : state) { cmod.FFT(transformed, poly); + } } static void helib_fft_inverse(benchmark::State& state, Meta& meta) diff --git a/ci/build_install_lib.sh b/ci/build_install_lib.sh index 83a9ae754..256d81a7e 100755 --- a/ci/build_install_lib.sh +++ b/ci/build_install_lib.sh @@ -11,21 +11,45 @@ # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. +# Copyright (C) 2021 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -xe -if [ "$#" -ne 2 ]; then - echo "Wrong parameter number. Usage ./${0} " +if [ "$#" -ne 6 ]; then + echo "Wrong parameter number. Usage ./${0} " exit 1 fi PACKAGE_BUILD="${1}" CMAKE_INSTALL_PREFIX="${2}" +C_COMPILER="${3}" +CXX_COMPILER="${4}" +USE_INTEL_HEXL="${5}" +INTEL_HEXL_DIR="${6}" ROOT_DIR="$(pwd)" mkdir build cd build # We assume PACKAGE_BUILD argument to be a valid cmake option -cmake -DPACKAGE_BUILD="${PACKAGE_BUILD}" -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" -DBUILD_SHARED=ON -DENABLE_TEST=ON -DTARGET_ARCHITECTURE=x86-64 .. +cmake -DPACKAGE_BUILD="${PACKAGE_BUILD}" \ + -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" \ + -DUSE_INTEL_HEXL="${USE_INTEL_HEXL}" \ + -DCMAKE_C_COMPILER="${C_COMPILER}" \ + -DCMAKE_CXX_COMPILER="${CXX_COMPILER}" \ + -DHEXL_DIR="${INTEL_HEXL_DIR}" \ + -DBUILD_SHARED=OFF \ + -DENABLE_TEST=ON \ + -DTARGET_ARCHITECTURE=x86-64 \ + .. make -j4 VERBOSE=1 make install cd "${ROOT_DIR}" diff --git a/ci/install_deps.sh b/ci/install_deps.sh index bfe44bbbb..2abeab343 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -11,33 +11,50 @@ # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. +# Copyright (C) 2021 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -xe +if [ "$#" -ne 5 ]; then + echo "Wrong parameter number. Usage ./${0} " + exit 1 +fi + +PACKAGE_BUILD="${1}" +RUNNER_OS="${2}" +C_COMPILER="${3}" +CXX_COMPILER="${4}" +USE_INTEL_HEXL="${5}" + +if [ "${USE_INTEL_HEXL}" == "ON" ]; then + git clone https://github.com/intel/hexl.git -b v1.2.1 + cd hexl + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=./ \ + -DHEXL_SHARED_LIB=OFF \ + -DCMAKE_C_COMPILER="${C_COMPILER}" \ + -DCMAKE_CXX_COMPILER="${CXX_COMPILER}" + cmake --build build -j4 --target install + cd ../ +fi + cd $HOME -if [ "${PACKAGE_BUILD}" == "OFF" ]; then - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - if [ "${TRAVIS_DIST}" == "bionic" ]; then - sudo apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install m4 libgmp-dev - curl -O "https://libntl.org/ntl-11.4.3.tar.gz" - tar --no-same-owner -xf ntl-11.4.3.tar.gz - cd "$HOME/ntl-11.4.3/src" - ./configure SHARED=on NTL_GMP_LIP=on NTL_THREADS=on NTL_THREAD_BOOST=on - make -j4 - sudo make install - else - sudo apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install libgmp-dev libntl-dev - fi - elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then - # GMP will be installed as a dependency to NTL (if it is not already present) - brew install ntl - fi -else - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - sudo apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install patchelf m4 - elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then - brew install m4 - fi +if [ "${RUNNER_OS}" == "ubuntu-20.04" ]; then + sudo apt-get -yq --no-install-suggests --no-install-recommends install libgmp-dev libntl-dev bats +fi + +if [ "${RUNNER_OS}" == "macos-latest" ]; then + brew install ntl fi cd "$HOME" diff --git a/include/helib/CModulus.h b/include/helib/CModulus.h index af86d602e..6693fea14 100644 --- a/include/helib/CModulus.h +++ b/include/helib/CModulus.h @@ -9,6 +9,20 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ + +/* Intel HEXL integration. + * Copyright (C) 2021 Intel Corporation + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef HELIB_CMODULUS_H #define HELIB_CMODULUS_H /** @@ -123,6 +137,9 @@ class Cmodulus void FFT(NTL::vec_long& y, const NTL::ZZX& x) const; // y = FFT(x) void FFT(NTL::vec_long& y, const zzX& x) const; + // y = FFT(x) + void FFT(NTL::vec_long& y, NTL::zz_pX& x) const; + // expects zp context to be set externally // x = FFT^{-1}(y) diff --git a/include/helib/partialMatch.h b/include/helib/partialMatch.h index 2279b18cd..2958af0b8 100644 --- a/include/helib/partialMatch.h +++ b/include/helib/partialMatch.h @@ -94,12 +94,24 @@ Matrix calculateMasks(const EncryptedArray& ea, mask = mask.columns(columns); mask.inPlaceTranspose(); - (mask -= database) + // FIXME: Avoid deep copy + // Ptxt Query + if constexpr (std::is_same_v>) { + auto tmp = database.deepCopy(); + (tmp -= mask) .apply([&](auto& entry) { mapTo01(ea, entry); }) .apply([](auto& entry) { entry.negate(); }) .apply([](auto& entry) { entry.addConstant(NTL::ZZX(1l)); }); - return mask; + return tmp; + } else { // Ctxt Query + (mask -= database) + .apply([&](auto& entry) { mapTo01(ea, entry); }) + .apply([](auto& entry) { entry.negate(); }) + .apply([](auto& entry) { entry.addConstant(NTL::ZZX(1l)); }); + + return mask; + } } /** @@ -569,8 +581,8 @@ class Database * match or no match respectively. **/ template - Matrix contains(const Query_t& lookup_query, - const Matrix& query_data) const; + auto contains(const Query_t& lookup_query, + const Matrix& query_data) const; // FIXME: Combination of TXT = ctxt and TXT2 = ptxt does not work /** @@ -583,8 +595,8 @@ class Database * @return A `Matrix` containing a score on weighted matches. **/ template - Matrix getScore(const Query_t& weighted_query, - const Matrix& query_data) const; + auto getScore(const Query_t& weighted_query, + const Matrix& query_data) const; // TODO - correct name? /** @@ -593,6 +605,8 @@ class Database **/ long columns() { return data.dims(1); } + Matrix& getData(); + private: Matrix data; std::shared_ptr context; @@ -600,7 +614,7 @@ class Database template template -inline Matrix Database::contains( +inline auto Database::contains( const Query_t& lookup_query, const Matrix& query_data) const { @@ -619,7 +633,7 @@ inline Matrix Database::contains( template template -inline Matrix Database::getScore( +inline auto Database::getScore( const Query_t& weighted_query, const Matrix& query_data) const { @@ -633,6 +647,12 @@ inline Matrix Database::getScore( return result; } +template +inline Matrix& Database::getData() +{ + return data; +} + } // namespace helib #endif diff --git a/src/CModulus.cpp b/src/CModulus.cpp index 9e8af5e96..a663bbaba 100644 --- a/src/CModulus.cpp +++ b/src/CModulus.cpp @@ -370,7 +370,7 @@ void Cmodulus::FFT_aux(NTL::vec_long& y, NTL::zz_pX& tmp) const y.SetLength(phim); long* yp = y.elts(); - NTL::zz_p* tmp_p = tmp.rep.elts(); + const NTL::zz_p* tmp_p = tmp.rep.elts(); #ifdef USE_INTEL_HEXL @@ -456,7 +456,7 @@ void Cmodulus::FFT(NTL::vec_long& y, const NTL::ZZX& x) const convert(tmp, x); // convert input to zpx format } - FFT_aux(y, tmp); + FFT(y, tmp); } void Cmodulus::FFT(NTL::vec_long& y, const zzX& x) const @@ -471,8 +471,16 @@ void Cmodulus::FFT(NTL::vec_long& y, const zzX& x) const HELIB_NTIMER_START(FFT_remainder); convert(tmp, x); // convert input to zpx format } + FFT(y, tmp); +} - FFT_aux(y, tmp); +void Cmodulus::FFT(NTL::vec_long& y, NTL::zz_pX& x) const +{ + HELIB_TIMER_START; + NTL::zz_pBak bak; + bak.save(); + context.restore(); + FFT_aux(y, x); } void Cmodulus::iFFT(NTL::zz_pX& x, const NTL::vec_long& y) const diff --git a/utils/common/common.h b/utils/common/common.h index 40bc50f68..6ca8a6568 100644 --- a/utils/common/common.h +++ b/utils/common/common.h @@ -18,13 +18,13 @@ #include -std::string stripExtension(const std::string& s) +inline std::string stripExtension(const std::string& s) { std::size_t dotPos = s.find_last_of("."); return (dotPos == std::string::npos) ? s : s.substr(0, dotPos); } -std::string readline(std::istream& is) +inline std::string readline(std::istream& is) { std::string s; getline(is, s);