Make redis-rs part of this repo (valkey-io#2456)

* Make redis-rs part of this repo
* Improved PYTHON DEVELOPERS.md file
* Added Makefile for unified build method for all the languages

.github/workflows/python.yml

@@ -216,9 +216,9 @@ jobs:
 
             - name: Install dependencies
               if: always()
-              uses: threeal/pipx-install-action@latest
-              with:
-                  packages: flake8 isort black
+              working-directory: ./python
+              run: |
+                  sudo apt install -y python3-pip python3 flake8 isort black
 
             - name: Lint python with isort
               if: always()

.github/workflows/semgrep.yml

@@ -33,4 +33,4 @@ jobs:
       # Fetch project source with GitHub Actions Checkout.
       - uses: actions/checkout@v3
       # Run the "semgrep ci" command on the command line of the docker image.
-      - run: semgrep ci --config auto --no-suppress-errors
+      - run: semgrep ci --config auto --no-suppress-errors --exclude-rule generic.secrets.security.detected-private-key.detected-private-key

.gitignore

@@ -43,6 +43,8 @@ logger-rs.linux-x64-gnu.node
 utils/clusters/
 utils/tls_crts/
 utils/TestUtils.js
+.build/
+.project
 
 # OSS Review Toolkit (ORT) files
 **/ort*/**

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "submodules/redis-rs"]
-	path = submodules/redis-rs
-	url = https://github.com/amazon-contributing/redis-rs

Makefile

@@ -0,0 +1,118 @@
+.PHONY: all java java-test python python-test node node-test check-redis-server go go-test
+
+BLUE=\033[34m
+YELLOW=\033[33m
+GREEN=\033[32m
+RESET=\033[0m
+ROOT_DIR=$(shell pwd)
+PYENV_DIR=$(shell pwd)/python/.env
+PY_PATH=$(shell find python/.env -name "site-packages"|xargs readlink -f)
+PY_GLIDE_PATH=$(shell pwd)/python/python/
+
+all: java java-test python python-test node node-test go go-test python-lint java-lint
+
+##
+## Java targets
+##
+java:
+	@echo "$(GREEN)Building for Java (release)$(RESET)"
+	@cd java && ./gradlew :client:buildAllRelease
+
+java-lint:
+	@echo "$(GREEN)Running spotlessCheck$(RESET)"
+	@cd java && ./gradlew :spotlessCheck
+	@echo "$(GREEN)Running spotlessApply$(RESET)"
+	@cd java && ./gradlew :spotlessApply
+
+java-test: check-redis-server
+	@echo "$(GREEN)Running integration tests$(RESET)"
+	@cd java && ./gradlew :integTest:test
+
+##
+## Python targets
+##
+python: .build/python_deps
+	@echo "$(GREEN)Building for Python (release)$(RESET)"
+	@cd python && VIRTUAL_ENV=$(PYENV_DIR) .env/bin/maturin develop --release --strip
+
+python-lint: .build/python_deps
+	@echo "$(GREEN)Building Linters for python$(RESET)"
+	cd python && 																		\
+		export VIRTUAL_ENV=$(PYENV_DIR); 												\
+		export PYTHONPATH=$(PY_PATH):$(PY_GLIDE_PATH); 									\
+		export PATH=$(PYENV_DIR)/bin:$(PATH);		 									\
+		isort . --profile black --skip-glob python/glide/protobuf --skip-glob .env && 	\
+		black . --exclude python/glide/protobuf --exclude .env && 						\
+		flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics      		\
+			--exclude=python/glide/protobuf,.env/* --extend-ignore=E230				&&	\
+		flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127  		\
+			--statistics --exclude=python/glide/protobuf,.env/*             			\
+			--extend-ignore=E230
+
+python-test: .build/python_deps check-redis-server
+	cd python && PYTHONPATH=$(PY_PATH):$(PY_GLIDE_PATH) .env/bin/pytest --asyncio-mode=auto
+
+.build/python_deps:
+	@echo "$(GREEN)Generating protobuf files...$(RESET)"
+	@protoc -Iprotobuf=$(ROOT_DIR)/glide-core/src/protobuf/ \
+		--python_out=$(ROOT_DIR)/python/python/glide $(ROOT_DIR)/glide-core/src/protobuf/*.proto
+	@echo "$(GREEN)Building environment...$(RESET)"
+	@cd python && python3 -m venv .env
+	@echo "$(GREEN)Installing requirements...$(RESET)"
+	@cd python && .env/bin/pip install -r requirements.txt
+	@cd python && .env/bin/pip install -r dev_requirements.txt
+	@mkdir -p .build/ && touch .build/python_deps
+
+##
+## NodeJS targets
+##
+node: .build/node_deps
+	@echo "$(GREEN)Building for NodeJS (release)...$(RESET)"
+	@cd node && npm run build:release
+
+.build/node_deps:
+	@echo "$(GREEN)Installing NodeJS dependencies...$(RESET)"
+	@cd node && npm i
+	@cd node/rust-client && npm i
+	@mkdir -p .build/ && touch .build/node_deps
+
+node-test: .build/node_deps check-redis-server
+	@echo "$(GREEN)Running tests for NodeJS$(RESET)"
+	@cd node && npm run build
+	cd node && npm test
+
+node-lint: .build/node_deps
+	@echo "$(GREEN)Running linters for NodeJS$(RESET)"
+	@cd node && npx run lint:fix
+
+##
+## Go targets
+##
+
+
+go: .build/go_deps
+	$(MAKE) -C go build
+
+go-test: .build/go_deps
+	$(MAKE) -C go test
+
+go-lint: .build/go_deps
+	$(MAKE) -C go lint
+
+.build/go_deps:
+	@echo "$(GREEN)Installing GO dependencies...$(RESET)"
+	$(MAKE) -C go install-build-tools install-dev-tools
+	@mkdir -p .build/ && touch .build/go_deps
+
+##
+## Common targets
+##
+check-redis-server:
+	which redis-server
+
+clean:
+	rm -fr .build/
+
+help:
+	@echo "$(GREEN)Listing Makefile targets:$(RESET)"
+	@echo $(shell grep '^[^#[:space:]].*:' Makefile|cut -d":" -f1|grep -v PHONY|grep -v "^.build"|sort)

benchmarks/rust/Cargo.toml

@@ -11,7 +11,7 @@ authors = ["Valkey GLIDE Maintainers"]
 tokio = { version = "1", features = ["macros", "time", "rt-multi-thread"] }
 glide-core = { path = "../../glide-core" }
 logger_core = {path = "../../logger_core"}
-redis = { path = "../../submodules/redis-rs/redis", features = ["aio"] }
+redis = { path = "../../glide-core/redis-rs/redis", features = ["aio"] }
 futures = "0.3.28"
 rand = "0.8.5"
 itoa = "1.0.6"

csharp/lib/Cargo.toml

@@ -12,7 +12,7 @@ name = "glide_rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-redis = { path = "../../submodules/redis-rs/redis", features = ["aio", "tokio-comp","tokio-native-tls-comp"] }
+redis = { path = "../../glide-core/redis-rs/redis", features = ["aio", "tokio-comp","tokio-native-tls-comp"] }
 glide-core = { path = "../../glide-core" }
 tokio = { version = "^1", features = ["rt", "macros", "rt-multi-thread", "time"] }
 logger_core = {path = "../../logger_core"}

glide-core/Cargo.toml

@@ -10,7 +10,7 @@ authors = ["Valkey GLIDE Maintainers"]
 [dependencies]
 bytes = "1"
 futures = "^0.3"
-redis = { path = "../submodules/redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp", "connection-manager","cluster", "cluster-async"] }
+redis = { path = "./redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp", "connection-manager","cluster", "cluster-async"] }
 tokio = { version = "1", features = ["macros", "time"] }
 logger_core = {path = "../logger_core"}
 dispose = "0.5.0"
@@ -42,7 +42,7 @@ serial_test = "3"
 criterion = { version = "^0.5", features = ["html_reports", "async_tokio"] }
 which = "5"
 ctor = "0.2.2"
-redis = { path = "../submodules/redis-rs/redis", features = ["tls-rustls-insecure"] }
+redis = { path = "./redis-rs/redis", features = ["tls-rustls-insecure"] }
 iai-callgrind = "0.9"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 glide-core = { path = ".", features = ["socket-layer"] } # always enable this feature in tests.

glide-core/redis-rs/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+members = ["redis", "redis-test"]
+resolver = "2"

glide-core/redis-rs/LICENSE

@@ -0,0 +1,33 @@
+Copyright (c) 2022 by redis-rs contributors
+
+Redis cluster code in parts copyright (c) 2018 by Atsushi Koge.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * The names of the contributors may not be used to endorse or
+      promote products derived from this software without specific
+      prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

glide-core/redis-rs/Makefile

@@ -0,0 +1,96 @@
+build:
+	@cargo build
+
+test:
+	@echo "===================================================================="
+	@echo "Build all features with lock file"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" cargo build --locked --all-features
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type TCP without features"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --no-default-features -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type TCP with all features and RESP2"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type TCP with all features and RESP3"
+	@echo "===================================================================="
+	@REDISRS_SERVER_TYPE=tcp PROTOCOL=RESP3 cargo test -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type TCP with all features and Rustls support"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp+tls RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type TCP with all features and native-TLS support"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp+tls RUST_BACKTRACE=1 cargo test --locked -p redis --features=json,tokio-native-tls-comp,connection-manager,cluster-async -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type UNIX"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=unix RUST_BACKTRACE=1 cargo test --locked -p redis --test parser --test test_basic --test test_types --all-features -- --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing Connection Type UNIX SOCKETS"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=unix RUST_BACKTRACE=1 cargo test --locked -p redis --all-features -- --test-threads=1 --skip test_cluster --skip test_async_cluster --skip test_module --skip test_cluster_scan
+
+	@echo "===================================================================="
+	@echo "Testing async-std with Rustls"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --features=async-std-rustls-comp,cluster-async -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing async-std with native-TLS"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked -p redis --features=async-std-native-tls-comp,cluster-async -- --nocapture --test-threads=1 --skip test_module
+
+	@echo "===================================================================="
+	@echo "Testing redis-test"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" RUST_BACKTRACE=1 cargo test --locked -p redis-test 
+
+
+test-module:
+	@echo "===================================================================="
+	@echo "Testing RESP2 with module support enabled (currently only RedisJSON)"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 cargo test --locked --all-features test_module -- --test-threads=1
+
+	@echo "===================================================================="
+	@echo "Testing RESP3 with module support enabled (currently only RedisJSON)"
+	@echo "===================================================================="
+	@RUSTFLAGS="-D warnings" REDISRS_SERVER_TYPE=tcp RUST_BACKTRACE=1 RESP3=true cargo test --all-features test_module -- --test-threads=1
+
+test-single: test
+
+bench:
+	cargo bench --all-features
+
+docs:
+	@RUSTFLAGS="-D warnings" RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps
+
+upload-docs: docs
+	@./upload-docs.sh
+
+style-check:
+	@rustup component add rustfmt 2> /dev/null
+	cargo fmt --all -- --check
+
+lint:
+	@rustup component add clippy 2> /dev/null
+	cargo clippy --all-features --all --tests --examples -- -D clippy::all -D warnings
+
+fuzz:
+	cd afl/parser/ && \
+	cargo afl build --bin fuzz-target && \
+	cargo afl fuzz -i in -o out target/debug/fuzz-target
+
+.PHONY: build test bench docs upload-docs style-check lint fuzz