From 057f50514b06411a3ede633d35d801993918deef Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:05:05 -0500 Subject: [PATCH 01/40] Synced with latest commit of libedgetpu libedgetpu has been updated to support TF 2.15.0. --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index 3164995..d900b8a 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit 3164995622300286ef2bb14d7fdc2792dae045b7 +Subproject commit d900b8a307528df979537ec98beb717b923ae444 From 767c4bc9bfabce8d308e352b1e283a4c63f85f72 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:06:32 -0500 Subject: [PATCH 02/40] Remove outdated deb package for python 2 in docker. --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3ca1dc0..c27160e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,6 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ sudo \ debhelper \ - python \ python3-all \ python3-numpy \ python3-setuptools \ From a4e64089635bd360735d11372175ce7e2ce7f937 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:07:15 -0500 Subject: [PATCH 03/40] Update version fo bazel to 6.1.0 in docker needed for TF2.15.0 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c27160e..8ae8f5f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -77,7 +77,7 @@ RUN git clone https://github.com/raspberrypi/tools.git && \ cd tools && \ git reset --hard 4a335520900ce55e251ac4f420f52bf0b2ab6b1f -ARG BAZEL_VERSION=4.0.0 +ARG BAZEL_VERSION=6.1.0 RUN wget -O /bazel https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh && \ bash /bazel && \ rm -f /bazel From a858b8d495bb040961fde66fd82ee5b40d05d533 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:09:00 -0500 Subject: [PATCH 04/40] Update WROKSPACE to allow compilatin of current version of TF 1. Reworked after deprecation fo Tensorflow/toolchains. 2. Added definition of TF 2.15.0 3. Updated google benchmark to fix missing header call. --- WORKSPACE | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 87ec4c9..b06ef57 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,6 +21,86 @@ local_repository( path = "libedgetpu", ) +# ================================================================== + +http_archive( + name = "org_tensorflow", + urls = [ + "https://github.com/tensorflow/tensorflow/archive/6887368d6d46223f460358323c4b76d61d1558a8.tar.gz", + ], + sha256 = "bb25fa4574e42ea4d452979e1d2ba3b86b39569d6b8106a846a238b880d73652", + strip_prefix = "tensorflow-" + "6887368d6d46223f460358323c4b76d61d1558a8", + ) + +# Add definition of tensorflow version 2.15.0 +http_archive( + name = "bazel_skylib", + sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + urls = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + ], +) + +http_archive( + name = "rules_python", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load( + "@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl", + "python_repository", +) + +python_repository(name = "python_version_repo") + +load("@python_version_repo//:py_version.bzl", "HERMETIC_PYTHON_VERSION") + +python_register_toolchains( + name = "python", + ignore_root_user_error = True, + python_version = HERMETIC_PYTHON_VERSION, +) + +load("@python//:defs.bzl", "interpreter") +load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse") + +NUMPY_ANNOTATIONS = { + "numpy": package_annotation( + additive_build_content = """\ +filegroup( + name = "includes", + srcs = glob(["site-packages/numpy/core/include/**/*.h"]), +) +cc_library( + name = "numpy_headers", + hdrs = [":includes"], + strip_include_prefix="site-packages/numpy/core/include/", +) +""", + ), +} + +pip_parse( + name = "pypi", + annotations = NUMPY_ANNOTATIONS, + python_interpreter_target = interpreter, + requirements = "@org_tensorflow//:requirements_lock_" + HERMETIC_PYTHON_VERSION.replace(".", "_") + ".txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() + +# ================================================================== + load("@libedgetpu//:workspace.bzl", "libedgetpu_dependencies") libedgetpu_dependencies() @@ -37,7 +117,7 @@ load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0") tf_workspace0() load("@coral_crosstool//:configure.bzl", "cc_crosstool") -cc_crosstool(name = "crosstool", cpp_version = "c++14") +cc_crosstool(name = "crosstool", cpp_version = "c++17") # External Dependencies http_archive( @@ -57,10 +137,10 @@ glog_library(with_gflags=0) http_archive( name = "com_github_google_benchmark", - sha256 = "6e40ccab16a91a7beff4b5b640b84846867e125ebce6ac0fe3a70c5bae39675f", - strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be", + sha256 = "8e7b955f04bc6984e4f14074d0d191474f76a6c8e849e04a9dced49bc975f2d4", + strip_prefix = "benchmark-344117638c8ff7e239044fd0fa7085839fc03021", urls = [ - "https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.tar.gz" + "https://github.com/google/benchmark/archive/344117638c8ff7e239044fd0fa7085839fc03021.tar.gz" ], ) From 28226deb33730835576352f1a161d8e274744ff7 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:15:54 -0500 Subject: [PATCH 05/40] Fix FTFS due to change in Eigen3 API Replace deprecated Eigen::all with Eigen::indexing::all --- coral/learn/backprop/layers.cc | 2 +- coral/learn/backprop/softmax_regression_model.cc | 2 +- coral/learn/backprop/test_utils.cc | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coral/learn/backprop/layers.cc b/coral/learn/backprop/layers.cc index 59cfd2f..599c92f 100644 --- a/coral/learn/backprop/layers.cc +++ b/coral/learn/backprop/layers.cc @@ -40,7 +40,7 @@ MatrixXf CrossEntropyGradient(const MatrixXf& c, const MatrixXf& p) { MatrixXf FullyConnected(const MatrixXf& mat_x, const MatrixXf& mat_w, const MatrixXf& mat_b) { MatrixXf mat_y = mat_x * mat_w; - mat_y.array().rowwise() += mat_b.array()(0, Eigen::all); + mat_y.array().rowwise() += mat_b.array()(0, Eigen::indexing::all); return mat_y; } diff --git a/coral/learn/backprop/softmax_regression_model.cc b/coral/learn/backprop/softmax_regression_model.cc index 5325945..039e630 100644 --- a/coral/learn/backprop/softmax_regression_model.cc +++ b/coral/learn/backprop/softmax_regression_model.cc @@ -129,7 +129,7 @@ void SoftmaxRegressionModel::Train(const TrainingData& data, const auto& batch_indices = GetBatchIndices(data.training_data, train_config.batch_size); MatrixXf train_batch, labels_batch; - train_batch = data.training_data(batch_indices, Eigen::all); + train_batch = data.training_data(batch_indices, Eigen::indexing::all); // Create one-hot label vectors labels_batch = MatrixXf::Zero(train_config.batch_size, num_classes_); diff --git a/coral/learn/backprop/test_utils.cc b/coral/learn/backprop/test_utils.cc index 85d61e3..31b4a9d 100644 --- a/coral/learn/backprop/test_utils.cc +++ b/coral/learn/backprop/test_utils.cc @@ -41,7 +41,7 @@ TrainingData ShuffleAndSplitData(const MatrixXf& data_matrix, std::mt19937(rd())); MatrixXf shuffled_data = MatrixXf::Zero(data_matrix.rows(), data_matrix.cols()); - shuffled_data = data_matrix(shuffled_indices, Eigen::all); + shuffled_data = data_matrix(shuffled_indices, Eigen::indexing::all); std::vector shuffled_labels(total_rows, -1); for (int i = 0; i < total_rows; ++i) { shuffled_labels[i] = labels_vector[shuffled_indices[i]]; @@ -50,9 +50,9 @@ TrainingData ShuffleAndSplitData(const MatrixXf& data_matrix, // Eigen::seq boundaries are inclusive on both sides. TrainingData fake_data; fake_data.training_data = - shuffled_data(Eigen::seq(0, num_train - 1), Eigen::all); + shuffled_data(Eigen::seq(0, num_train - 1), Eigen::indexing::all); fake_data.validation_data = - shuffled_data(Eigen::seq(num_train, Eigen::last), Eigen::all); + shuffled_data(Eigen::seq(num_train, Eigen::last), Eigen::indexing::all); fake_data.training_labels.assign(shuffled_labels.begin(), shuffled_labels.begin() + num_train); @@ -105,7 +105,7 @@ TrainingData GenerateMvnRandomData(const std::vector& class_sizes, MultiVariateNormalDistribution dist(means[i], cov_mats[i]); MatrixXf samples = dist.Sample(n); // Eigen::seq boundaries are inclusive on both sides. - data_matrix(Eigen::seq(start_index, start_index + n - 1), Eigen::all) = + data_matrix(Eigen::seq(start_index, start_index + n - 1), Eigen::indexing::all) = samples.transpose(); labels_vector.insert(labels_vector.end(), n, i); start_index += n; @@ -127,7 +127,7 @@ TrainingData GenerateUniformRandomData(const std::vector& class_sizes, int n = class_sizes[i]; MatrixXf samples = MatrixXf::Random(total_cols, n); // Eigen::seq boundaries are inclusive on both sides. - data_matrix(Eigen::seq(start_index, start_index + n - 1), Eigen::all) = + data_matrix(Eigen::seq(start_index, start_index + n - 1), Eigen::indexing::all) = samples.transpose(); labels_vector.insert(labels_vector.end(), n, i); start_index += n; From d88f35e752386204d8408e7777a9c9bc60d7e227 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:18:34 -0500 Subject: [PATCH 06/40] Replace signature_def_names with signature_keys This is for tflite::Interpreter The former call is deprecated, replaced by the latter --- coral/detection/adapter.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coral/detection/adapter.cc b/coral/detection/adapter.cc index 785443f..af29af5 100644 --- a/coral/detection/adapter.cc +++ b/coral/detection/adapter.cc @@ -78,11 +78,11 @@ std::vector GetDetectionResults(const tflite::Interpreter& interpreter, // If a model has signature, we use the signature output tensor names to parse // the results. Otherwise, we parse the results based on some assumption of // the output tensor order and size. - if (!interpreter.signature_def_names().empty()) { - CHECK_EQ(interpreter.signature_def_names().size(), 1); - VLOG(1) << "Signature name: " << *interpreter.signature_def_names()[0]; + if (!interpreter.signature_keys().empty()) { + CHECK_EQ(interpreter.signature_keys().size(), 1); + VLOG(1) << "Signature name: " << *interpreter.signature_keys()[0]; const auto& signature_output_map = interpreter.signature_outputs( - interpreter.signature_def_names()[0]->c_str()); + interpreter.signature_keys()[0]->c_str()); CHECK_EQ(signature_output_map.size(), 4); count = TensorData( *interpreter.tensor(signature_output_map.at("output_0"))); From 5797c5e2c87eb05b7936a97f926bead0a259c505 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:20:55 -0500 Subject: [PATCH 07/40] Fix ambiguous call leading to FTFS, by overriding TfLiteFloatArrayCopy --- coral/pipeline/internal/segment_runner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coral/pipeline/internal/segment_runner.cc b/coral/pipeline/internal/segment_runner.cc index b8c4791..3891592 100644 --- a/coral/pipeline/internal/segment_runner.cc +++ b/coral/pipeline/internal/segment_runner.cc @@ -82,7 +82,7 @@ absl::Status SegmentRunner::SetExternalTensorBuffer(const char* buffer, // its memory here. auto* quant_params_clone = reinterpret_cast( malloc(sizeof(TfLiteAffineQuantization))); - quant_params_clone->scale = TfLiteFloatArrayCopy(quant_params->scale); + quant_params_clone->scale = coral::internal::TfLiteFloatArrayCopy(quant_params->scale); CHECK(quant_params_clone->scale); quant_params_clone->zero_point = TfLiteIntArrayCopy(quant_params->zero_point); From 3a45b8431f064d2dcc9eab64becac28cd33b6672 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:26:32 -0500 Subject: [PATCH 08/40] Update Dockerfile.windows --- docker/Dockerfile.windows | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.windows b/docker/Dockerfile.windows index b25a5d2..e97ea34 100644 --- a/docker/Dockerfile.windows +++ b/docker/Dockerfile.windows @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/windows/servercore:1903 SHELL ["powershell", "-command"] -ARG BAZEL_VERSION=4.0.0 +ARG BAZEL_VERSION=6.1.0 # Install Bazel ADD https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.exe c:/windows/system32/bazel.exe @@ -21,7 +21,7 @@ RUN setx /M PATH $($Env:PATH + ';C:\Program Files\7-Zip') ARG MSYS_BASE_REPO=http://repo.msys2.org/distrib/x86_64 # Install msys2 -ARG MSYS_VERSION=20200602 +ARG MSYS_VERSION=20240113 ADD ${MSYS_BASE_REPO}/msys2-base-x86_64-${MSYS_VERSION}.tar.xz c:/windows/temp RUN 7z.exe x c:\windows\temp\msys2-base-x86_64-$env:MSYS_VERSION.tar.xz RUN 7z.exe x c:\msys2-base-x86_64-$env:MSYS_VERSION.tar -o"c:\\" @@ -36,16 +36,18 @@ RUN 7z.exe x c:\windows\temp\patch-$env:PATCH_VERSION-x86_64.pkg.tar.xz RUN 7z.exe x c:\patch-$env:PATCH_VERSION-x86_64.pkg.tar -o"c:\\msys64" # Install vim (for xxd) -ARG VIM_VERSION=8.2.0592-1 +ARG VIM_VERSION=VIM_VERSION=9.1.0000-1 ADD ${MSYS_PACKAGE_REPO}/vim-${VIM_VERSION}-x86_64.pkg.tar.xz c:/windows/temp -RUN 7z.exe x -y c:\windows\temp\vim-$env:VIM_VERSION-x86_64.pkg.tar.xz +RUN 7z.exe x -y c:\windows\temp\vim-$env:VIM_VERSION-x86_64.pkg.tar.zst RUN 7z.exe x -y c:\vim-$env:VIM_VERSION-x86_64.pkg.tar -o"c:\\msys64" RUN choco install -m -y python3 --version=3.5.4 RUN choco install -m -y python3 --version=3.6.8 RUN choco install -m -y python3 --version=3.7.9 RUN choco install -m -y python3 --version=3.8.10 -RUN choco install -m -y python3 --version=3.9.5 +RUN choco install -m -y python3 --version=3.9.13 +RUN choco install -m -y python3 --version=3.10.11 +RUN choco install -m -y python3 --version=3.11.8 RUN c:\python35\python.exe -m pip install --upgrade pip RUN c:\python35\python.exe -m pip install numpy six pillow wheel @@ -62,7 +64,13 @@ RUN c:\python38\python.exe -m pip install numpy six pillow wheel RUN c:\python39\python.exe -m pip install --upgrade pip RUN c:\python39\python.exe -m pip install numpy six pillow wheel +RUN c:\python310\python.exe -m pip install --upgrade pip +RUN c:\python310\python.exe -m pip install numpy six pillow wheel + +RUN c:\python311\python.exe -m pip install --upgrade pip +RUN c:\python311\python.exe -m pip install numpy six pillow wheel + # Install libusb release package -ARG LIBUSB_VERSION=1.0.24 +ARG LIBUSB_VERSION=1.0.27 ADD https://github.com/libusb/libusb/releases/download/v${LIBUSB_VERSION}/libusb-${LIBUSB_VERSION}.7z c:/windows/temp RUN 7z x -oc:\libusb c:\windows\temp\libusb-$env:LIBUSB_VERSION.7z From f36e4f538f9c9e429fe95c763fb734383eb52d0b Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:28:38 -0500 Subject: [PATCH 09/40] Updated README.md To include modern OS /Python versions --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d2db4bb..0db4336 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ You need to install the following software: 1. Bazel for macOS from https://github.com/bazelbuild/bazel/releases 1. MacPorts from https://www.macports.org/install.php 1. Ports of `python` interpreter and `numpy` library: `sudo port install - python35 python36 python37 py35-numpy py36-numpy py37-numpy` + python38 python39 python310 python311 py38-numpy py39-numpy py310-numpy python311-numpy` 1. Port of `libusb` library: `sudo port install libusb` Right after that all normal `make` commands should work as usual. You can run @@ -88,6 +88,6 @@ Docker allows to avoid complicated environment setup and build binaries for Linux on other operating systems without complicated setup, e.g., ``` -make DOCKER_IMAGE=debian:buster DOCKER_CPUS="k8 armv7a aarch64" DOCKER_TARGETS=tests docker-build -make DOCKER_IMAGE=ubuntu:18.04 DOCKER_CPUS="k8 armv7a aarch64" DOCKER_TARGETS=tests docker-build +make DOCKER_IMAGE=debian:bookworm DOCKER_CPUS="k8 armv7a aarch64" DOCKER_TARGETS=tests docker-build +make DOCKER_IMAGE=ubuntu:22.04 DOCKER_CPUS="k8 armv7a aarch64" DOCKER_TARGETS=tests docker-build ``` From accbdbc16396916d96c873fbbf3b154511c18611 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 17:47:29 -0500 Subject: [PATCH 10/40] Fix additional ambiguous call leading to FTFS, by overriding TfLiteFloatArrayCopy --- coral/tflite_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coral/tflite_utils.cc b/coral/tflite_utils.cc index 646ee35..2a8c1ba 100644 --- a/coral/tflite_utils.cc +++ b/coral/tflite_utils.cc @@ -46,7 +46,7 @@ TfLiteAffineQuantization* TfLiteAffineQuantizationCopy( auto* copy = static_cast( malloc(sizeof(TfLiteAffineQuantization))); CHECK(copy); - copy->scale = TfLiteFloatArrayCopy(src->scale); + copy->scale = coral::TfLiteFloatArrayCopy(src->scale); copy->zero_point = TfLiteIntArrayCopy(src->zero_point); copy->quantized_dimension = src->quantized_dimension; return copy; From 1d5222d0973d8f09c8b607cbcef5d0391e5df7e1 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 18:00:03 -0500 Subject: [PATCH 11/40] Fix additional FTFS due to change in Eigen3 API Replace deprecated Eigen::last with Eigen::placeholders::all --- coral/learn/backprop/test_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coral/learn/backprop/test_utils.cc b/coral/learn/backprop/test_utils.cc index 31b4a9d..e3b5a5c 100644 --- a/coral/learn/backprop/test_utils.cc +++ b/coral/learn/backprop/test_utils.cc @@ -52,7 +52,7 @@ TrainingData ShuffleAndSplitData(const MatrixXf& data_matrix, fake_data.training_data = shuffled_data(Eigen::seq(0, num_train - 1), Eigen::indexing::all); fake_data.validation_data = - shuffled_data(Eigen::seq(num_train, Eigen::last), Eigen::indexing::all); + shuffled_data(Eigen::seq(num_train, Eigen::placeholders::last), Eigen::indexing::all); fake_data.training_labels.assign(shuffled_labels.begin(), shuffled_labels.begin() + num_train); From 6f46ccf2d0310111dd23767506e1c5f4660e71f3 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 29 Feb 2024 22:17:59 -0500 Subject: [PATCH 12/40] Use std=c++17 not c++14 Several compilation errors are fixed --- .bazelrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 9b11d2c..640cac7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,8 +7,9 @@ build --enable_platform_specific_config build:linux --crosstool_top=@crosstool//:toolchains build:linux --compiler=gcc +build:linux --cxxopt=-std=c++17 -build:macos --cxxopt=-std=c++14 +build:macos --cxxopt=-std=c++17 build:windows --incompatible_restrict_string_escapes=false build:windows --cxxopt=/std:c++latest From 743e804c954e6387869de0604f88df05ee2f9fd2 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 10:55:42 -0500 Subject: [PATCH 13/40] Fix position of comment on TF version in WORKSPACE --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index b06ef57..b4c1f50 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,6 +23,7 @@ local_repository( # ================================================================== +# Add definition of tensorflow version 2.15.0 http_archive( name = "org_tensorflow", urls = [ @@ -32,7 +33,6 @@ http_archive( strip_prefix = "tensorflow-" + "6887368d6d46223f460358323c4b76d61d1558a8", ) -# Add definition of tensorflow version 2.15.0 http_archive( name = "bazel_skylib", sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", From aae943b5eca2c60a92e7fbb153dcc1e2e221ce7f Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 11:32:02 -0500 Subject: [PATCH 14/40] Updated rules-python to v0.31.0 and simplification of WORKSPACE With rules-python v.0.27.0 intruduced a better way to handle pypi requirements, removing the need of several calls within WORKSPACE. --- WORKSPACE | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index b4c1f50..f8525dd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -44,9 +44,9 @@ http_archive( http_archive( name = "rules_python", - sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", - strip_prefix = "rules_python-0.26.0", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", + sha256 = "c68bdc4fbec25de5b5493b8819cfc877c4ea299c0dcb15c244c5a00208cde311", + strip_prefix = "rules_python-0.31.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.31.0.tar.gz", ) load("@rules_python//python:repositories.bzl", "py_repositories") @@ -70,34 +70,7 @@ python_register_toolchains( ) load("@python//:defs.bzl", "interpreter") -load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse") - -NUMPY_ANNOTATIONS = { - "numpy": package_annotation( - additive_build_content = """\ -filegroup( - name = "includes", - srcs = glob(["site-packages/numpy/core/include/**/*.h"]), -) -cc_library( - name = "numpy_headers", - hdrs = [":includes"], - strip_include_prefix="site-packages/numpy/core/include/", -) -""", - ), -} - -pip_parse( - name = "pypi", - annotations = NUMPY_ANNOTATIONS, - python_interpreter_target = interpreter, - requirements = "@org_tensorflow//:requirements_lock_" + HERMETIC_PYTHON_VERSION.replace(".", "_") + ".txt", -) - -load("@pypi//:requirements.bzl", "install_deps") - -install_deps() +load("@rules_python//python:pip.bzl", "package_annotation") # ================================================================== From 5d59e6f133ab9ad7da6084437c6d019642bedb66 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 11:50:04 -0500 Subject: [PATCH 15/40] Temporarily use fork of libedgetpu with initial support for TF2.16. Stable libedgetpu only supports TF2.15.0. It will be reverted back to the official repo once that has been updated to TF2.16.0 --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9251969..9bc142a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "libedgetpu"] path = libedgetpu - url = https://github.com/google-coral/libedgetpu + url = https://github.com/feranick/libedgetpu [submodule "test_data"] path = test_data url = https://github.com/google-coral/test_data From 963848005dbef57b7e810b2d25a05320c309063a Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 11:52:37 -0500 Subject: [PATCH 16/40] Sync with latest feranick/libedgetpu --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index d900b8a..00f2584 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit d900b8a307528df979537ec98beb717b923ae444 +Subproject commit 00f258481f92bafb397fb9639e70e86efd1005ae From a676b35d0920cbf397885b2f4379e81abfd8848f Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 11:55:40 -0500 Subject: [PATCH 17/40] Add support for TF 2.16.0-rc0 --- WORKSPACE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index f8525dd..4243f51 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,14 +23,14 @@ local_repository( # ================================================================== -# Add definition of tensorflow version 2.15.0 +# Add definition of tensorflow version 2.16.0-rc0 http_archive( name = "org_tensorflow", urls = [ - "https://github.com/tensorflow/tensorflow/archive/6887368d6d46223f460358323c4b76d61d1558a8.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/4bdc149ac84738b06a592473595c1c9c2bd2a9a3.tar.gz", ], - sha256 = "bb25fa4574e42ea4d452979e1d2ba3b86b39569d6b8106a846a238b880d73652", - strip_prefix = "tensorflow-" + "6887368d6d46223f460358323c4b76d61d1558a8", + sha256 = "1bbdd3dd266de720e08b7093d2441a9bca183c32d7c8e4c84e05d17e770974bf", + strip_prefix = "tensorflow-" + "4bdc149ac84738b06a592473595c1c9c2bd2a9a3", ) http_archive( From 5d42cc8a991d8475641fabc1287ce097d7feaf03 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 12:28:18 -0500 Subject: [PATCH 18/40] Sync with latest libedgetpu commit --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index 00f2584..c685a7d 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit 00f258481f92bafb397fb9639e70e86efd1005ae +Subproject commit c685a7db1e072c1e1efe28ebb6f1aaa581ff158c From 87812af8395d30d982715aaea46404454c44a2ef Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 16:56:42 -0500 Subject: [PATCH 19/40] Revert commit aae943b5eca While the original commit simplified WORKSPACE, it is not alligned with the current version of TF. REverting to the previous version of rules_python v.0.26.0 which is the same shipped with TF 2.16.0. --- WORKSPACE | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4243f51..3409bb3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -44,9 +44,9 @@ http_archive( http_archive( name = "rules_python", - sha256 = "c68bdc4fbec25de5b5493b8819cfc877c4ea299c0dcb15c244c5a00208cde311", - strip_prefix = "rules_python-0.31.0", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.31.0.tar.gz", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ) load("@rules_python//python:repositories.bzl", "py_repositories") @@ -70,7 +70,34 @@ python_register_toolchains( ) load("@python//:defs.bzl", "interpreter") -load("@rules_python//python:pip.bzl", "package_annotation") +load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse") + +NUMPY_ANNOTATIONS = { + "numpy": package_annotation( + additive_build_content = """\ +filegroup( + name = "includes", + srcs = glob(["site-packages/numpy/core/include/**/*.h"]), +) +cc_library( + name = "numpy_headers", + hdrs = [":includes"], + strip_include_prefix="site-packages/numpy/core/include/", +) +""", + ), +} + +pip_parse( + name = "pypi", + annotations = NUMPY_ANNOTATIONS, + python_interpreter_target = interpreter, + requirements = "@org_tensorflow//:requirements_lock_" + HERMETIC_PYTHON_VERSION.replace(".", "_") + ".txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() # ================================================================== From 437af955f28ff12f827e69ce18ca02bc5a9b719a Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 1 Mar 2024 16:57:55 -0500 Subject: [PATCH 20/40] Sync with libedgetpu --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index c685a7d..e206c31 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit c685a7db1e072c1e1efe28ebb6f1aaa581ff158c +Subproject commit e206c3169e02c94f7c497adcf188a316e8740830 From 1c87cc142a6c744632737370695d9c071c95a6a3 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Sun, 3 Mar 2024 16:56:18 -0500 Subject: [PATCH 21/40] Use more modern version fo debian in docker.mk --- docker/docker.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker.mk b/docker/docker.mk index 0609036..8e524ee 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -3,7 +3,7 @@ DOCKER_MK_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) # Docker DOCKER_CPUS ?= k8 armv7a aarch64 DOCKER_TARGETS ?= -DOCKER_IMAGE ?= debian:stretch +DOCKER_IMAGE ?= debian:bookworm DOCKER_TAG_BASE ?= "bazel-cross" DOCKER_TAG := "$(DOCKER_TAG_BASE)-$(subst :,-,$(DOCKER_IMAGE))" DOCKER_SHELL_COMMAND ?= From f778cb8a4bbecd45c8cf907a4a191c0cc35431e9 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Sun, 3 Mar 2024 16:59:08 -0500 Subject: [PATCH 22/40] Update distributions in build script. Additional calls to gstreamer-1.-1 --- scripts/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 6ab604b..21ced2f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -45,11 +45,11 @@ for i in "$@"; do fi done -# Build for k8 (use Ubuntu 18.04 for compatibility with most platforms). -docker_build "k8" "ubuntu:18.04" +# Build for k8 (use Ubuntu 22.04 for compatibility with most platforms). +docker_build "k8" "ubuntu:22.04" # Build for armv7a. -docker_build "armv7a" "debian:stretch" +docker_build "armv7a" "debian:bookworm" # Build for aarch64. -docker_build "aarch64" "debian:stretch" +docker_build "aarch64" "debian:bookworm" From cd9d98f6f81b5ae76f80ab9223bc17e4a74451eb Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Mon, 4 Mar 2024 11:47:41 -0500 Subject: [PATCH 23/40] Sync with latest libedgetpu --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index e206c31..2cf811a 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit e206c3169e02c94f7c497adcf188a316e8740830 +Subproject commit 2cf811a0ec597117bfe126f5f8c58656ec56d3ca From e5331a733b37edd19d9bb8a394e97a00ce25d310 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Tue, 5 Mar 2024 14:52:02 -0500 Subject: [PATCH 24/40] Build against TF2.17.0-dev to resolve visibility issue in SCHEMA A visibility issue in TF SCHEMA was preventing compilation of libcoral. Ref: https://github.com/tensorflow/tensorflow/issues/63074 This is now fixed in TF 2.17.0-dev with commit 79ecb3f --- WORKSPACE | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3409bb3..6ff34a0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -24,13 +24,25 @@ local_repository( # ================================================================== # Add definition of tensorflow version 2.16.0-rc0 +#http_archive( +# name = "org_tensorflow", +# urls = [ +# "https://github.com/tensorflow/tensorflow/archive/4bdc149ac84738b06a592473595c1c9c2bd2a9a3.tar.gz", +# ], +# sha256 = "1bbdd3dd266de720e08b7093d2441a9bca183c32d7c8e4c84e05d17e770974bf", +# strip_prefix = "tensorflow-" + "4bdc149ac84738b06a592473595c1c9c2bd2a9a3", +# ) + +# Add definition of tensorflow version 2.17.0-dev with visibility commit for SCHEMA. +# Reference: https://github.com/tensorflow/tensorflow/issues/63074 +# This will be removed once TF 2.17.0 is stable. http_archive( name = "org_tensorflow", urls = [ - "https://github.com/tensorflow/tensorflow/archive/4bdc149ac84738b06a592473595c1c9c2bd2a9a3.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/79ecb3f8bb6bd73f0115fa9a97b630a6f745a426.tar.gz", ], - sha256 = "1bbdd3dd266de720e08b7093d2441a9bca183c32d7c8e4c84e05d17e770974bf", - strip_prefix = "tensorflow-" + "4bdc149ac84738b06a592473595c1c9c2bd2a9a3", + sha256 = "488dcc18e8d3896a54c9abdc657c43a580d70d72d69e6d6db07a4be95cdfbc3e", + strip_prefix = "tensorflow-" + "79ecb3f8bb6bd73f0115fa9a97b630a6f745a426", ) http_archive( From 139442bf1c0b6aa9ccaa4afacbccfed0987fe7a2 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Tue, 5 Mar 2024 16:42:49 -0500 Subject: [PATCH 25/40] Add arm-specific compiler flags to fix compilation due to undedefined variable types. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cefd685..48dcd1b 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,9 @@ BAZEL_BUILD_FLAGS := --compilation_mode=$(COMPILATION_MODE) \ --cpu=$(CPU) ifeq ($(CPU),aarch64) -BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off +BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --cxxopt=-mfp16-format=ieee else ifeq ($(CPU),armv7a) -BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off +BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --cxxopt=-mfp16-format=ieee endif # $(1): pattern, $(2) destination directory From 7c32c203d3d85e5a2bc485fdb3db5a13f72ac8ef Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Tue, 5 Mar 2024 17:13:27 -0500 Subject: [PATCH 26/40] Revert previous commit for aarch64 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 48dcd1b..41b8aa9 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ BAZEL_BUILD_FLAGS := --compilation_mode=$(COMPILATION_MODE) \ --cpu=$(CPU) ifeq ($(CPU),aarch64) -BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --cxxopt=-mfp16-format=ieee +BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off else ifeq ($(CPU),armv7a) BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --cxxopt=-mfp16-format=ieee endif From 98a9f2b4b723b1a42ec3522f9a18eeb436c0cdda Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Tue, 5 Mar 2024 18:24:35 -0500 Subject: [PATCH 27/40] Revert commit 139442b also for armhf Not needed for clean build with Docker. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 41b8aa9..cefd685 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ BAZEL_BUILD_FLAGS := --compilation_mode=$(COMPILATION_MODE) \ ifeq ($(CPU),aarch64) BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off else ifeq ($(CPU),armv7a) -BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --cxxopt=-mfp16-format=ieee +BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off endif # $(1): pattern, $(2) destination directory From d92c702cd218c640d5b2feceda7e9d159b025437 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Wed, 6 Mar 2024 00:05:10 -0500 Subject: [PATCH 28/40] Use bazel 6.5.0 for TF 2.17.0 --- docker/Dockerfile | 2 +- docker/Dockerfile.windows | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ae8f5f..6f45962 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -77,7 +77,7 @@ RUN git clone https://github.com/raspberrypi/tools.git && \ cd tools && \ git reset --hard 4a335520900ce55e251ac4f420f52bf0b2ab6b1f -ARG BAZEL_VERSION=6.1.0 +ARG BAZEL_VERSION=6.5.0 RUN wget -O /bazel https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh && \ bash /bazel && \ rm -f /bazel diff --git a/docker/Dockerfile.windows b/docker/Dockerfile.windows index e97ea34..8264685 100644 --- a/docker/Dockerfile.windows +++ b/docker/Dockerfile.windows @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/windows/servercore:1903 SHELL ["powershell", "-command"] -ARG BAZEL_VERSION=6.1.0 +ARG BAZEL_VERSION=6.5.0 # Install Bazel ADD https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.exe c:/windows/system32/bazel.exe From 08d1c665954d4e83179352678bd2a4d9faa70436 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Wed, 6 Mar 2024 00:38:18 -0500 Subject: [PATCH 29/40] Sync with libedgetpu --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index 2cf811a..d799c30 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit 2cf811a0ec597117bfe126f5f8c58656ec56d3ca +Subproject commit d799c30c90a9f39bfe0c6b447f963898665958c9 From e49f8f101a587de2d9eed47f93e01014f2240727 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Wed, 6 Mar 2024 11:04:11 -0500 Subject: [PATCH 30/40] Reinstate revised version of commit 139442bf1 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cefd685..7c13f59 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ BAZEL_BUILD_FLAGS := --compilation_mode=$(COMPILATION_MODE) \ ifeq ($(CPU),aarch64) BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off else ifeq ($(CPU),armv7a) -BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off +BAZEL_BUILD_FLAGS += --copt=-ffp-contract=off --copt=-mfp16-format=ieee endif # $(1): pattern, $(2) destination directory From bc7ee28e07de8d6b4d3a8eb83dab29ee62c3ba63 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 8 Mar 2024 11:01:49 -0500 Subject: [PATCH 31/40] Resync with libedgetpu --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index d799c30..f3e9264 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit d799c30c90a9f39bfe0c6b447f963898665958c9 +Subproject commit f3e9264865e85b7f2d66dc5eccd6581cec1b12f5 From 16f6bfa48ca51956f22d66bb1d5ecb4e9c774341 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 8 Mar 2024 11:03:51 -0500 Subject: [PATCH 32/40] Update stable TF version required to 2.16.1 This is disable in favor of the 2.17.0-dev with visibility patch. --- WORKSPACE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 6ff34a0..3af31db 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,14 +23,14 @@ local_repository( # ================================================================== -# Add definition of tensorflow version 2.16.0-rc0 +# Add definition of tensorflow version 2.16.1 stable. #http_archive( # name = "org_tensorflow", # urls = [ -# "https://github.com/tensorflow/tensorflow/archive/4bdc149ac84738b06a592473595c1c9c2bd2a9a3.tar.gz", +# "https://github.com/tensorflow/tensorflow/archive/5bc9d26649cca274750ad3625bd93422617eed4b.tar.gz", # ], -# sha256 = "1bbdd3dd266de720e08b7093d2441a9bca183c32d7c8e4c84e05d17e770974bf", -# strip_prefix = "tensorflow-" + "4bdc149ac84738b06a592473595c1c9c2bd2a9a3", +# sha256 = "fe592915c85d1a89c20f3dd89db0772ee22a0fbda78e39aa46a778d638a96abc", +# strip_prefix = "tensorflow-" + "5bc9d26649cca274750ad3625bd93422617eed4b", # ) # Add definition of tensorflow version 2.17.0-dev with visibility commit for SCHEMA. From 7b421d0d5aacac80bc540968cfb5747e6da2de98 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 21 Jun 2024 10:26:06 +0200 Subject: [PATCH 33/40] Sync with libedge for TF 2.17.0-rc0 --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index f3e9264..1494008 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit f3e9264865e85b7f2d66dc5eccd6581cec1b12f5 +Subproject commit 1494008c2c8013d2f739934ae12936ff9fb28a97 From 501030fda2ae0027b22e3b7267e29fc31f32793d Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 21 Jun 2024 14:02:33 +0200 Subject: [PATCH 34/40] Updated dockerfile.windows with latest dependencies --- docker/Dockerfile.windows | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.windows b/docker/Dockerfile.windows index 8264685..00669ce 100644 --- a/docker/Dockerfile.windows +++ b/docker/Dockerfile.windows @@ -21,7 +21,7 @@ RUN setx /M PATH $($Env:PATH + ';C:\Program Files\7-Zip') ARG MSYS_BASE_REPO=http://repo.msys2.org/distrib/x86_64 # Install msys2 -ARG MSYS_VERSION=20240113 +ARG MSYS_VERSION=20240507 ADD ${MSYS_BASE_REPO}/msys2-base-x86_64-${MSYS_VERSION}.tar.xz c:/windows/temp RUN 7z.exe x c:\windows\temp\msys2-base-x86_64-$env:MSYS_VERSION.tar.xz RUN 7z.exe x c:\msys2-base-x86_64-$env:MSYS_VERSION.tar -o"c:\\" @@ -36,7 +36,7 @@ RUN 7z.exe x c:\windows\temp\patch-$env:PATCH_VERSION-x86_64.pkg.tar.xz RUN 7z.exe x c:\patch-$env:PATCH_VERSION-x86_64.pkg.tar -o"c:\\msys64" # Install vim (for xxd) -ARG VIM_VERSION=VIM_VERSION=9.1.0000-1 +ARG VIM_VERSION=VIM_VERSION=9.1.0445-1 ADD ${MSYS_PACKAGE_REPO}/vim-${VIM_VERSION}-x86_64.pkg.tar.xz c:/windows/temp RUN 7z.exe x -y c:\windows\temp\vim-$env:VIM_VERSION-x86_64.pkg.tar.zst RUN 7z.exe x -y c:\vim-$env:VIM_VERSION-x86_64.pkg.tar -o"c:\\msys64" @@ -46,8 +46,8 @@ RUN choco install -m -y python3 --version=3.6.8 RUN choco install -m -y python3 --version=3.7.9 RUN choco install -m -y python3 --version=3.8.10 RUN choco install -m -y python3 --version=3.9.13 -RUN choco install -m -y python3 --version=3.10.11 -RUN choco install -m -y python3 --version=3.11.8 +RUN choco install -m -y python3 --version=3.10.14 +RUN choco install -m -y python3 --version=3.11.9 RUN c:\python35\python.exe -m pip install --upgrade pip RUN c:\python35\python.exe -m pip install numpy six pillow wheel From 3bd6aa0d61daa7df57ab429b86f92acb036e224a Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Sun, 7 Jul 2024 12:27:43 +0200 Subject: [PATCH 35/40] Added support for temsorflow 2.17.0-rc1 --- libedgetpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libedgetpu b/libedgetpu index 1494008..d609eaa 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit 1494008c2c8013d2f739934ae12936ff9fb28a97 +Subproject commit d609eaa614b06856d55185464cedbe4b6e384045 From 576612c3656d121b523b44f8c59fb03d707a2a9c Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 12 Jul 2024 15:52:39 +0200 Subject: [PATCH 36/40] Add support for TF 2.17.0 stable --- WORKSPACE | 20 ++++---------------- libedgetpu | 2 +- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3af31db..e729d7d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,28 +23,16 @@ local_repository( # ================================================================== -# Add definition of tensorflow version 2.16.1 stable. +# Add definition of tensorflow version 2.17.0 stable. #http_archive( # name = "org_tensorflow", # urls = [ -# "https://github.com/tensorflow/tensorflow/archive/5bc9d26649cca274750ad3625bd93422617eed4b.tar.gz", +# "https://github.com/tensorflow/tensorflow/archive/ad6d8cc177d0c868982e39e0823d0efbfb95f04c.tar.gz", # ], -# sha256 = "fe592915c85d1a89c20f3dd89db0772ee22a0fbda78e39aa46a778d638a96abc", -# strip_prefix = "tensorflow-" + "5bc9d26649cca274750ad3625bd93422617eed4b", +# sha256 = "75b8dc9b33afff6f2e2d2e2dacc26dd0973bdcee94eec2af290828c1bc574bdc, +# strip_prefix = "tensorflow-" + "ad6d8cc177d0c868982e39e0823d0efbfb95f04c", # ) -# Add definition of tensorflow version 2.17.0-dev with visibility commit for SCHEMA. -# Reference: https://github.com/tensorflow/tensorflow/issues/63074 -# This will be removed once TF 2.17.0 is stable. -http_archive( - name = "org_tensorflow", - urls = [ - "https://github.com/tensorflow/tensorflow/archive/79ecb3f8bb6bd73f0115fa9a97b630a6f745a426.tar.gz", - ], - sha256 = "488dcc18e8d3896a54c9abdc657c43a580d70d72d69e6d6db07a4be95cdfbc3e", - strip_prefix = "tensorflow-" + "79ecb3f8bb6bd73f0115fa9a97b630a6f745a426", - ) - http_archive( name = "bazel_skylib", sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", diff --git a/libedgetpu b/libedgetpu index d609eaa..28287cf 160000 --- a/libedgetpu +++ b/libedgetpu @@ -1 +1 @@ -Subproject commit d609eaa614b06856d55185464cedbe4b6e384045 +Subproject commit 28287cf2fd7bbcbebf52855036e9be571584a266 From 4e096ece407529f71f273893c8fa45b0ca51ba6b Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Thu, 8 Aug 2024 18:01:54 +0200 Subject: [PATCH 37/40] Added support for Ubuntu:24.04 24.04 introduces a new way to handle repositories, using sources rather than lists. --- docker/Dockerfile | 18 ++++++-- docker/Dockerfile.orig | 83 +++++++++++++++++++++++++++++++++++ docker/update_sources.orig.sh | 26 +++++++++++ docker/update_sources.sh | 36 +++++++++++++++ 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 docker/Dockerfile.orig create mode 100755 docker/update_sources.orig.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f45962..62247b1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -51,10 +51,20 @@ RUN if grep 'Bionic Beaver' /etc/os-release > /dev/null; then \ # On older Ubuntu these packages can't be installed in a multi-arch fashion. # Instead we download the debs and extract them for build time linking. -RUN mkdir /debs && chmod a=rwx /debs && cd /debs && apt-get update && apt-get download \ - libglib2.0-0 \ - libglib2.0-0:armhf \ - libglib2.0-0:arm64 \ + +RUN if grep 'Noble Numbat' /etc/os-release > /dev/null; then \ + mkdir /debs && chmod a=rwx /debs && cd /debs && apt-get update && apt-get download \ + libglib2.0-0t64 \ + libglib2.0-0t64:armhf \ + libglib2.0-0t64:arm64; \ + else \ + mkdir /debs && chmod a=rwx /debs && cd /debs && apt-get update && apt-get download \ + libglib2.0-0 \ + libglib2.0-0:armhf \ + libglib2.0-0:arm64; \ + fi + +RUN cd /debs && apt-get update && apt-get download --ignore-missing \ libglib2.0-dev \ libglib2.0-dev:armhf \ libglib2.0-dev:arm64 \ diff --git a/docker/Dockerfile.orig b/docker/Dockerfile.orig new file mode 100644 index 0000000..6f45962 --- /dev/null +++ b/docker/Dockerfile.orig @@ -0,0 +1,83 @@ +ARG IMAGE +FROM ${IMAGE} + +COPY update_sources.sh / +RUN /update_sources.sh + +RUN dpkg --add-architecture armhf +RUN dpkg --add-architecture arm64 +RUN echo 'APT::Immediate-Configure false;' >> /etc/apt/apt.conf + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + libc6-dev:arm64 \ + libc6-dev:armhf \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + sudo \ + debhelper \ + python3-all \ + python3-numpy \ + python3-setuptools \ + python3-six \ + python3-wheel \ + libpython3-dev \ + libpython3-dev:armhf \ + libpython3-dev:arm64 \ + build-essential \ + crossbuild-essential-armhf \ + crossbuild-essential-arm64 \ + libusb-1.0-0-dev \ + libusb-1.0-0-dev:arm64 \ + libusb-1.0-0-dev:armhf \ + zlib1g-dev \ + zlib1g-dev:armhf \ + zlib1g-dev:arm64 \ + pkg-config \ + p7zip-full \ + zip \ + unzip \ + curl \ + wget \ + git \ + vim \ + mc \ + software-properties-common + +# Bionic Beaver == Ubuntu 18.04 +RUN if grep 'Bionic Beaver' /etc/os-release > /dev/null; then \ + add-apt-repository ppa:ubuntu-toolchain-r/test \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-9 g++-9; \ + fi + +# On older Ubuntu these packages can't be installed in a multi-arch fashion. +# Instead we download the debs and extract them for build time linking. +RUN mkdir /debs && chmod a=rwx /debs && cd /debs && apt-get update && apt-get download \ + libglib2.0-0 \ + libglib2.0-0:armhf \ + libglib2.0-0:arm64 \ + libglib2.0-dev \ + libglib2.0-dev:armhf \ + libglib2.0-dev:arm64 \ + libgstreamer1.0-0 \ + libgstreamer1.0-0:armhf \ + libgstreamer1.0-0:arm64 \ + libgstreamer1.0-dev \ + libgstreamer1.0-dev:armhf \ + libgstreamer1.0-dev:arm64 \ + libgstreamer-plugins-base1.0-0 \ + libgstreamer-plugins-base1.0-0:armhf \ + libgstreamer-plugins-base1.0-0:arm64 \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer-plugins-base1.0-dev:armhf \ + libgstreamer-plugins-base1.0-dev:arm64 + +RUN for d in /debs/*.deb; do dpkg -x $d /usr/system_libs; done + +RUN git clone https://github.com/raspberrypi/tools.git && \ + cd tools && \ + git reset --hard 4a335520900ce55e251ac4f420f52bf0b2ab6b1f + +ARG BAZEL_VERSION=6.5.0 +RUN wget -O /bazel https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh && \ + bash /bazel && \ + rm -f /bazel diff --git a/docker/update_sources.orig.sh b/docker/update_sources.orig.sh new file mode 100755 index 0000000..dd74f6d --- /dev/null +++ b/docker/update_sources.orig.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +. /etc/os-release + +[[ "${NAME}" == "Ubuntu" ]] || exit 0 + +sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list + +cat <> /etc/apt/sources.list +deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe +deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe +deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe +EOT diff --git a/docker/update_sources.sh b/docker/update_sources.sh index dd74f6d..51bdeb1 100755 --- a/docker/update_sources.sh +++ b/docker/update_sources.sh @@ -19,8 +19,44 @@ sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list +if [ ${UBUNTU_CODENAME} == "noble" ]; then + +echo "NOBLE" + +rm /etc/apt/sources.list.d/ubuntu.sources +cat <> /etc/apt/sources.list.d/ubuntu.sources + +Types: deb +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: ${UBUNTU_CODENAME} ${UBUNTU_CODENAME}-updates ${UBUNTU_CODENAME}-backports +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Architectures: amd64 + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +Types: deb +URIs: http://security.ubuntu.com/ubuntu/ +Suites: ${UBUNTU_CODENAME}-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Architectures: amd64 + +Types: deb +URIs: http://ports.ubuntu.com/ubuntu-ports +Suites: ${UBUNTU_CODENAME} ${UBUNTU_CODENAME}-updates ${UBUNTU_CODENAME}-security +Components: main universe +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Architectures: arm64 armhf + +EOT + +else + cat <> /etc/apt/sources.list deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe EOT + +fi From 139a39c8204d0cff44b514b9fae5ada503d6292d Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Fri, 9 Aug 2024 14:37:49 +0200 Subject: [PATCH 38/40] Change su user to root to allow compilation in Ubuntu 24.04 --- docker/docker.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker.mk b/docker/docker.mk index 8e524ee..68ba280 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -24,7 +24,7 @@ chmod a+w /; \ groupadd --gid $(shell id -g) $(shell id -g -n); \ useradd -m -e '' -s /bin/bash --gid $(shell id -g) --uid $(shell id -u) $(shell id -u -n); \ echo '$(shell id -u -n) ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers; \ -su $(shell id -u -n) $(if $(1),-c '$(1)',) +su $(if $(1),-c '$(1)',) endef docker-image: From b74ede21cbad39e4d6a907a10a0cffc049a9fc3f Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Sat, 10 Aug 2024 13:26:22 +0200 Subject: [PATCH 39/40] Pass correct TF_PYTHON_VERSION during build. --- docker/docker.mk | 2 +- docker/docker.mk.orig | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docker/docker.mk.orig diff --git a/docker/docker.mk b/docker/docker.mk index 68ba280..11cc8f3 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -16,7 +16,7 @@ endif WORKSPACE := /workspace MAKE_COMMAND := \ for cpu in $(DOCKER_CPUS); do \ - make CPU=\$${cpu} COMPILATION_MODE=$(COMPILATION_MODE) -C $(WORKSPACE)/$(DOCKER_WORKSPACE_CD) $(DOCKER_TARGETS) || exit 1; \ + TF_PYTHON_VERSION=$(DOCKER_TF_PYTHON_VERSION) make CPU=\$${cpu} COMPILATION_MODE=$(COMPILATION_MODE) -C $(WORKSPACE)/$(DOCKER_WORKSPACE_CD) $(DOCKER_TARGETS) || exit 1; \ done define run_command diff --git a/docker/docker.mk.orig b/docker/docker.mk.orig new file mode 100644 index 0000000..68ba280 --- /dev/null +++ b/docker/docker.mk.orig @@ -0,0 +1,42 @@ +DOCKER_MK_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) + +# Docker +DOCKER_CPUS ?= k8 armv7a aarch64 +DOCKER_TARGETS ?= +DOCKER_IMAGE ?= debian:bookworm +DOCKER_TAG_BASE ?= "bazel-cross" +DOCKER_TAG := "$(DOCKER_TAG_BASE)-$(subst :,-,$(DOCKER_IMAGE))" +DOCKER_SHELL_COMMAND ?= +DOCKER_WORKSPACE_CD ?= + +ifndef DOCKER_WORKSPACE +$(error DOCKER_WORKSPACE is not defined) +endif + +WORKSPACE := /workspace +MAKE_COMMAND := \ +for cpu in $(DOCKER_CPUS); do \ + make CPU=\$${cpu} COMPILATION_MODE=$(COMPILATION_MODE) -C $(WORKSPACE)/$(DOCKER_WORKSPACE_CD) $(DOCKER_TARGETS) || exit 1; \ +done + +define run_command +chmod a+w /; \ +groupadd --gid $(shell id -g) $(shell id -g -n); \ +useradd -m -e '' -s /bin/bash --gid $(shell id -g) --uid $(shell id -u) $(shell id -u -n); \ +echo '$(shell id -u -n) ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers; \ +su $(if $(1),-c '$(1)',) +endef + +docker-image: + docker build $(DOCKER_IMAGE_OPTIONS) -t $(DOCKER_TAG) \ + --build-arg IMAGE=$(DOCKER_IMAGE) $(DOCKER_MK_DIR) + +docker-shell: docker-image + docker run --rm -i --tty -v $(DOCKER_WORKSPACE):$(WORKSPACE) \ + --workdir $(WORKSPACE)/$(DOCKER_WORKSPACE_CD) \ + $(DOCKER_TAG) /bin/bash -c "$(call run_command,$(DOCKER_SHELL_COMMAND))" + +docker-build: docker-image + docker run --rm -i $(shell tty -s && echo --tty) -v $(DOCKER_WORKSPACE):$(WORKSPACE) \ + $(DOCKER_TAG) /bin/bash -c "$(call run_command,$(MAKE_COMMAND))" + From d7054de85ee7568e96b11f9a9b66bf8c02dd66c0 Mon Sep 17 00:00:00 2001 From: Nicola Ferralis Date: Sun, 11 Aug 2024 12:06:18 +0200 Subject: [PATCH 40/40] Updated third party libraries in Docker.windows --- docker/Dockerfile.windows | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.windows b/docker/Dockerfile.windows index 00669ce..ec46152 100644 --- a/docker/Dockerfile.windows +++ b/docker/Dockerfile.windows @@ -21,7 +21,7 @@ RUN setx /M PATH $($Env:PATH + ';C:\Program Files\7-Zip') ARG MSYS_BASE_REPO=http://repo.msys2.org/distrib/x86_64 # Install msys2 -ARG MSYS_VERSION=20240507 +ARG MSYS_VERSION=20240727 ADD ${MSYS_BASE_REPO}/msys2-base-x86_64-${MSYS_VERSION}.tar.xz c:/windows/temp RUN 7z.exe x c:\windows\temp\msys2-base-x86_64-$env:MSYS_VERSION.tar.xz RUN 7z.exe x c:\msys2-base-x86_64-$env:MSYS_VERSION.tar -o"c:\\" @@ -30,13 +30,13 @@ RUN setx /M PATH $($Env:PATH + ';C:\msys64\usr\bin') ARG MSYS_PACKAGE_REPO=https://repo.msys2.org/msys/x86_64 # Install patch -ARG PATCH_VERSION=2.7.6-1 +ARG PATCH_VERSION=2.7.6-2 ADD ${MSYS_PACKAGE_REPO}/patch-${PATCH_VERSION}-x86_64.pkg.tar.xz c:/windows/temp RUN 7z.exe x c:\windows\temp\patch-$env:PATCH_VERSION-x86_64.pkg.tar.xz RUN 7z.exe x c:\patch-$env:PATCH_VERSION-x86_64.pkg.tar -o"c:\\msys64" # Install vim (for xxd) -ARG VIM_VERSION=VIM_VERSION=9.1.0445-1 +ARG VIM_VERSION=VIM_VERSION=9.1.0643-1 ADD ${MSYS_PACKAGE_REPO}/vim-${VIM_VERSION}-x86_64.pkg.tar.xz c:/windows/temp RUN 7z.exe x -y c:\windows\temp\vim-$env:VIM_VERSION-x86_64.pkg.tar.zst RUN 7z.exe x -y c:\vim-$env:VIM_VERSION-x86_64.pkg.tar -o"c:\\msys64"