From dd548550b31601033273d2bc21c2c1bfb35a8ff1 Mon Sep 17 00:00:00 2001 From: Mohammed Das Date: Fri, 17 Mar 2023 16:12:43 -0700 Subject: [PATCH] B&R for UDP encryption binary (#2200) Summary: Pull Request resolved: https://github.com/facebookresearch/fbpcs/pull/2200 As title Differential Revision: D44179453 fbshipit-source-id: 4c049a677fe3415ffcd9125b5aaa889e23d3f73d --- binaries_out_lists/emp_games.txt | 1 + docker/emp_games/CMakeLists.txt | 202 ++---------------- docker/emp_games/Dockerfile.ubuntu | 11 +- docker/emp_games/common.cmake | 7 +- extract-docker-binaries.sh | 1 + promote_scripts/promote_binaries.sh | 1 + upload-binaries-to-s3-test.sh | 2 + .../upload-binaries-using-onedocker.sh | 2 + 8 files changed, 44 insertions(+), 183 deletions(-) diff --git a/binaries_out_lists/emp_games.txt b/binaries_out_lists/emp_games.txt index c9e34995c..9112d9562 100644 --- a/binaries_out_lists/emp_games.txt +++ b/binaries_out_lists/emp_games.txt @@ -8,3 +8,4 @@ pcf2_attribution_calculator pcf2_aggregation_calculator shard_aggregator private_id_dfca_aggregator +udp_encryptor diff --git a/docker/emp_games/CMakeLists.txt b/docker/emp_games/CMakeLists.txt index 753e53b7e..982955792 100644 --- a/docker/emp_games/CMakeLists.txt +++ b/docker/emp_games/CMakeLists.txt @@ -3,201 +3,43 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -cmake_minimum_required(VERSION 3.10.1) +cmake_minimum_required(VERSION 3.16) project(game) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +add_library(range_v3 INTERFACE IMPORTED) +set_target_properties(range_v3 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES /root/build/range-v3/include) +SET (CMAKE_CXX_COMPILER "/usr/bin/clang++" CACHE STRING "C++ compiler" FORCE) + # Don't compile with AVX512 instructions since many of the AWS # instances won't have access to that instruction set. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-avx512f -march=haswell") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx512f -march=haswell") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx512f -march=haswell -fcoroutines") include("common.cmake") include("perf_tools.cmake") -# lift -add_executable( - lift_calculator - "fbpcs/emp_games/lift/calculator/main.cpp" - "fbpcs/emp_games/lift/calculator/OutputMetrics.hpp" - "fbpcs/emp_games/lift/common/GroupedLiftMetrics.h" - "fbpcs/emp_games/lift/common/GroupedLiftMetrics.cpp" - "fbpcs/emp_games/lift/common/LiftMetrics.h" - "fbpcs/emp_games/lift/common/LiftMetrics.cpp" - "fbpcs/emp_games/lift/calculator/CalculatorApp.h" - "fbpcs/emp_games/lift/calculator/CalculatorApp.cpp" - "fbpcs/emp_games/lift/calculator/CalculatorGame.h" - "fbpcs/emp_games/lift/calculator/OutputMetrics.h" - "fbpcs/emp_games/lift/calculator/InputData.cpp" - "fbpcs/emp_games/lift/calculator/InputData.h" - "fbpcs/emp_games/lift/calculator/CalculatorGameConfig.h" - "fbpcs/emp_games/lift/calculator/OutputMetricsData.h" - "fbpcs/emp_games/common/PrivateData.h" - "fbpcs/emp_games/common/SecretSharing.h" - "fbpcs/emp_games/common/EmpOperationUtil.h" - "fbpcs/emp_games/common/Csv.h" - "fbpcs/emp_games/common/Csv.cpp") -target_link_libraries( - lift_calculator - empgamecommon) -install(TARGETS lift_calculator DESTINATION bin) - -# generic shard_aggregator -file(GLOB shard_aggregator_src - "fbpcs/emp_games/attribution/shard_aggregator/AggMetrics.cpp", - "fbpcs/emp_games/attribution/shard_aggregator/AggMetricsThresholdCheckers.cpp", - "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorApp.cpp", - "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorValidation.cpp", - "fbpcs/emp_games/attribution/shard_aggregator/main.cpp" - "fbpcs/emp_games/attribution/Aggregator.h" - "fbpcs/emp_games/attribution/AttributionMetrics.h" - "fbpcs/emp_games/attribution/AttributionRule.h" - "fbpcs/emp_games/attribution/Constants.h" - "fbpcs/emp_games/attribution/Conversion.h" - "fbpcs/emp_games/attribution/Debug.h" - "fbpcs/emp_games/attribution/Touchpoint.h", - "fbpcs/emp_games/attribution/shard_aggregator/MainUtil.h", - "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorGame.h") -add_executable( - shard_aggregator - ${shard_aggregator_src}) -target_link_libraries( - shard_aggregator - empgamecommon - perftools) -install(TARGETS shard_aggregator DESTINATION bin) - -# decoupled_attribution -file(GLOB decoupled_attribution_src - "fbpcs/emp_games/attribution/decoupled_attribution/**.c" - "fbpcs/emp_games/attribution/decoupled_attribution/**.cpp" - "fbpcs/emp_games/attribution/decoupled_attribution/**.h" - "fbpcs/emp_games/attribution/decoupled_attribution/**.hpp") -list(FILTER decoupled_attribution_src EXCLUDE REGEX ".*Test.*") -add_executable( - decoupled_attribution_calculator - ${decoupled_attribution_src}) -target_link_libraries( - decoupled_attribution_calculator - empgamecommon - perftools) -install(TARGETS decoupled_attribution_calculator DESTINATION bin) - -# decoupled_aggregation -file(GLOB decoupled_aggregation_src - "fbpcs/emp_games/attribution/decoupled_aggregation/**.c" - "fbpcs/emp_games/attribution/decoupled_aggregation/**.cpp" - "fbpcs/emp_games/attribution/decoupled_aggregation/**.h" - "fbpcs/emp_games/attribution/decoupled_aggregation/metadata/**.h" - "fbpcs/emp_games/attribution/decoupled_aggregation/**.hpp") -list(FILTER decoupled_aggregation_src EXCLUDE REGEX ".*Test.*") -add_executable( - decoupled_aggregation_calculator - ${decoupled_aggregation_src}) -target_link_libraries( - decoupled_aggregation_calculator - empgamecommon - perftools) -install(TARGETS decoupled_aggregation_calculator DESTINATION bin) - - -# Find boost -find_package(Boost REQUIRED) -include_directories(${Boost_INCLUDE_DIRS}) - -# pcf2_attribution -file(GLOB pcf2_attribution_src - "fbpcs/emp_games/pcf2_attribution/**.c" - "fbpcs/emp_games/pcf2_attribution/**.cpp" - "fbpcs/emp_games/pcf2_attribution/**.h" - "fbpcs/emp_games/pcf2_attribution/**.hpp") -list(FILTER pcf2_attribution_src EXCLUDE REGEX ".*Test.*") -add_executable( - pcf2_attribution_calculator - ${pcf2_attribution_src}) -target_link_libraries( - pcf2_attribution_calculator - empgamecommon - perftools) -install(TARGETS pcf2_attribution_calculator DESTINATION bin) - -# pcf2_aggregation -file(GLOB pcf2_aggregation_src - "fbpcs/emp_games/pcf2_aggregation/**.c" - "fbpcs/emp_games/pcf2_aggregation/**.cpp" - "fbpcs/emp_games/pcf2_aggregation/**.h" - "fbpcs/emp_games/pcf2_aggregation/**.hpp") -list(FILTER pcf2_aggregation_src EXCLUDE REGEX ".*Test.*") -add_executable( - pcf2_aggregation_calculator - ${pcf2_aggregation_src}) -target_link_libraries( - pcf2_aggregation_calculator - empgamecommon - perftools) -install(TARGETS pcf2_aggregation_calculator DESTINATION bin) +message(STATUS "CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD}") +message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") +message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") +message(STATUS "CMAKE_LINKER: ${CMAKE_LINKER}") -# pcf2_lift metadata compaction -file(GLOB pcf2_lift_metadata_compaction_src - "fbpcs/emp_games/lift/metadata_compaction/**.cpp" - "fbpcs/emp_games/lift/metadata_compaction/**.h") -list(FILTER pcf2_lift_metadata_compaction_src EXCLUDE REGEX ".*Test.*") +# pcf2_udp encryption +file(GLOB udp_encryptor_src + "fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.cpp" + "fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.h") +list(FILTER udp_encryptor_src EXCLUDE REGEX ".*Test.*") add_executable( - pcf2_lift_metadata_compaction - ${pcf2_lift_metadata_compaction_src}) + udp_encryptor + ${udp_encryptor_src}) target_link_libraries( - pcf2_lift_metadata_compaction + udp_encryptor empgamecommon perftools - pcf2_lift_input_processing + ${FOLLY_LIBRARIES} ${Boost_LIBRARIES} + range_v3 ) -install(TARGETS pcf2_lift_metadata_compaction DESTINATION bin) - -# pcf2_lift -file(GLOB pcf2_lift_calculator_src - "fbpcs/emp_games/lift/common/**.cpp" - "fbpcs/emp_games/lift/common/**.h" - "fbpcs/emp_games/lift/pcf2_calculator/**.cpp" - "fbpcs/emp_games/lift/pcf2_calculator/**.h") -list(FILTER pcf2_lift_calculator_src EXCLUDE REGEX ".*Test.*") -add_executable( - pcf2_lift_calculator - ${pcf2_lift_calculator_src}) -target_link_libraries( - pcf2_lift_calculator - empgamecommon - perftools - pcf2_lift_input_processing) -install(TARGETS pcf2_lift_calculator DESTINATION bin) - -# pcf2_shard_combiner -file(GLOB pcf2_shard_combiner_src - "fbpcs/emp_games/pcf2_shard_combiner/**.cpp" - "fbpcs/emp_games/pcf2_shard_combiner/**.h" - "fbpcs/emp_games/pcf2_shard_combiner/util/**.h") -list(FILTER pcf2_shard_combiner_src EXCLUDE REGEX ".*Test.*") -add_executable( - pcf2_shard_combiner - ${pcf2_shard_combiner_src}) -target_link_libraries( - pcf2_shard_combiner - empgamecommon - perftools) -install(TARGETS pcf2_shard_combiner DESTINATION bin) - -# private_id_dfca_aggregator -file(GLOB private_id_dfca_aggregator_src - "fbpcs/emp_games/private_id_dfca_aggregator/**.cpp" - "fbpcs/emp_games/private_id_dfca_aggregator/**.h" - "fbpcs/emp_games/private_id_dfca_aggregator/util/**.h") -list(FILTER private_id_dfca_aggregator_src EXCLUDE REGEX ".*Test.*") -add_executable( - private_id_dfca_aggregator - ${private_id_dfca_aggregator_src}) -target_link_libraries( - private_id_dfca_aggregator - empgamecommon - perftools) -install(TARGETS private_id_dfca_aggregator DESTINATION bin) +install(TARGETS udp_encryptor DESTINATION bin) diff --git a/docker/emp_games/Dockerfile.ubuntu b/docker/emp_games/Dockerfile.ubuntu index d7ba37c06..8ab715041 100644 --- a/docker/emp_games/Dockerfile.ubuntu +++ b/docker/emp_games/Dockerfile.ubuntu @@ -7,6 +7,11 @@ ARG fbpcf_image FROM ${fbpcf_image} as dev +WORKDIR /root/build +RUN git clone https://github.com/ericniebler/range-v3 + + + RUN mkdir -p /root/build/emp_game WORKDIR /root/build/emp_game @@ -25,6 +30,9 @@ COPY fbpcs/emp_games/pcf2_shard_combiner/ ./fbpcs/emp_games/pcf2_shard_combiner COPY fbpcs/emp_games/private_id_dfca_aggregator/ ./fbpcs/emp_games/private_id_dfca_aggregator COPY fbpcs/emp_games/lift/ ./fbpcs/emp_games/lift COPY fbpcs/emp_games/common/ ./fbpcs/emp_games/common +COPY fbpcs/emp_games/data_processing/global_parameters ./fbpcs/emp_games/data_processing/global_parameters +COPY fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor +COPY fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor RUN cmake . -DTHREADING=ON -DUSE_RANDOM_DEVICE=ON RUN ./make_and_install_binary.sh @@ -45,7 +53,8 @@ RUN apt-get -y update && apt-get install -y --no-install-recommends \ libgoogle-glog0v5 \ libssl1.1 \ libre2-5 \ - zlib1g + zlib1g \ + librange-v3-dev COPY --from=dev /root/build/emp_game/bin/. /usr/local/bin/. RUN useradd -ms /bin/bash pcs diff --git a/docker/emp_games/common.cmake b/docker/emp_games/common.cmake index 9f5522675..651753a7b 100644 --- a/docker/emp_games/common.cmake +++ b/docker/emp_games/common.cmake @@ -18,7 +18,10 @@ file(GLOB emp_game_common_src "fbpcs/emp_games/common/**.c" "fbpcs/emp_games/common/**.cpp" "fbpcs/emp_games/common/**.h" - "fbpcs/emp_games/common/**.hpp") + "fbpcs/emp_games/common/**.hpp" + "fbpcs/emp_games/data_processing/global_parameters/**.cpp" + "fbpcs/emp_games/data_processing/global_parameters/**.h" + ) list(FILTER emp_game_common_src EXCLUDE REGEX ".*Test.*") add_library(empgamecommon STATIC ${emp_game_common_src}) @@ -30,7 +33,7 @@ target_link_libraries( ${AWSSDK_LINK_LIBRARIES} ${EMP-OT_LIBRARIES} google-cloud-cpp::storage - Folly::folly + ${FOLLY_LIBRARIES} re2) # pcf2 lift input processing diff --git a/extract-docker-binaries.sh b/extract-docker-binaries.sh index bbb7d6b6b..4ac1f400e 100755 --- a/extract-docker-binaries.sh +++ b/extract-docker-binaries.sh @@ -68,6 +68,7 @@ docker create -ti --name temp_container "${DOCKER_IMAGE_PATH}" docker cp temp_container:/usr/local/bin/lift_calculator "$SCRIPT_DIR/binaries_out/." docker cp temp_container:/usr/local/bin/pcf2_lift_calculator "$SCRIPT_DIR/binaries_out/." docker cp temp_container:/usr/local/bin/pcf2_lift_metadata_compaction "$SCRIPT_DIR/binaries_out/." +docker cp temp_container:/usr/local/bin/udp_encryptor "$SCRIPT_DIR/binaries_out/." docker cp temp_container:/usr/local/bin/decoupled_attribution_calculator "$SCRIPT_DIR/binaries_out/." docker cp temp_container:/usr/local/bin/decoupled_aggregation_calculator "$SCRIPT_DIR/binaries_out/." docker cp temp_container:/usr/local/bin/pcf2_attribution_calculator "$SCRIPT_DIR/binaries_out/." diff --git a/promote_scripts/promote_binaries.sh b/promote_scripts/promote_binaries.sh index 736894553..ecdb5d172 100755 --- a/promote_scripts/promote_binaries.sh +++ b/promote_scripts/promote_binaries.sh @@ -42,6 +42,7 @@ binary_names=( 'data_processing/private_id_dfca_id_combiner' 'validation/pc_pre_validation_cli' 'smart_agent/smart_agent_server' + 'udp_encryptor' ) s3_path="s3://one-docker-repository-prod" diff --git a/upload-binaries-to-s3-test.sh b/upload-binaries-to-s3-test.sh index 74d3f46de..f86c62e9c 100755 --- a/upload-binaries-to-s3-test.sh +++ b/upload-binaries-to-s3-test.sh @@ -46,6 +46,7 @@ data_processing_repo="s3://$one_docker_repo/data_processing" private_id_repo="s3://$one_docker_repo/pid" validation_repo="s3://$one_docker_repo/validation" smart_agent_repo="s3://$one_docker_repo/smart_agent" +udp_encryptor_package="s3://$one_docker_repo/data_processing/unified_data_process/UdpEncryptor/${TAG}/udp_encryptor" if [ "$PACKAGE" = "emp_games" ]; then cd binaries_out || exit @@ -59,6 +60,7 @@ aws s3 cp pcf2_aggregation_calculator "$pcf2_aggregation" aws s3 cp shard_aggregator "$shard_aggregator_package" aws s3 cp pcf2_shard_combiner "$pcf2_shard_combiner_package" aws s3 cp private_id_dfca_aggregator "$private_id_dfca_aggregator_package" +aws s3 cp udp_encryptor "$udp_encryptor_package" cd .. || exit fi diff --git a/upload_scripts/upload-binaries-using-onedocker.sh b/upload_scripts/upload-binaries-using-onedocker.sh index 3f0a1ae0e..587209a3e 100755 --- a/upload_scripts/upload-binaries-using-onedocker.sh +++ b/upload_scripts/upload-binaries-using-onedocker.sh @@ -63,6 +63,7 @@ pcf2_aggregation_path="private_attribution/pcf2_aggregation" shard_aggregator_path="private_attribution/shard-aggregator" pcf2_shard_combiner_path="private_attribution/pcf2_shard-combiner" private_id_dfca_aggregator_path="private_id_dfca/private_id_dfca_aggregator" +udp_encryptor_path="data_processing/unified_data_process/UdpEncryptor" if [ "$PACKAGE" = "emp_games" ]; then cd binaries_out || exit @@ -76,6 +77,7 @@ onedocker_upload "$pcf2_aggregation_path" pcf2_aggregation_calculator onedocker_upload "$shard_aggregator_path" shard_aggregator onedocker_upload "$pcf2_shard_combiner_path" pcf2_shard_combiner onedocker_upload "$private_id_dfca_aggregator_path" private_id_dfca_aggregator +onedocker_upload "$udp_encryptor_path" udp_encryptor cd .. || exit fi