diff --git a/.vscode/settings.json b/.vscode/settings.json index ef488543df..72bcb0d6f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,7 +14,7 @@ "node/rust-client/Cargo.toml", "logger_core/Cargo.toml", "csharp/lib/Cargo.toml", - "submodules/redis-rs/Cargo.toml", + "glide-core/redis-rs/Cargo.toml", "benchmarks/rust/Cargo.toml", "java/Cargo.toml" ], diff --git a/glide-core/Cargo.toml b/glide-core/Cargo.toml index 51de808bd2..28cad8e646 100644 --- a/glide-core/Cargo.toml +++ b/glide-core/Cargo.toml @@ -10,27 +10,42 @@ authors = ["Valkey GLIDE Maintainers"] [dependencies] bytes = "1" futures = "^0.3" -redis = { path = "./redis-rs/redis", features = ["aio", "tokio-comp", "tokio-rustls-comp", "connection-manager","cluster", "cluster-async"] } +redis = { path = "./redis-rs/redis", features = [ + "aio", + "tokio-comp", + "tokio-rustls-comp", + "connection-manager", + "cluster", + "cluster-async", +] } tokio = { version = "1", features = ["macros", "time"] } -logger_core = {path = "../logger_core"} +logger_core = { path = "../logger_core" } dispose = "0.5.0" -tokio-util = {version = "^0.7", features = ["rt"], optional = true} +tokio-util = { version = "^0.7", features = ["rt"], optional = true } num_cpus = { version = "^1.15", optional = true } tokio-retry = "0.3.0" -protobuf = { version= "3", features = ["bytes", "with-bytes"], optional = true } +protobuf = { version = "3", features = [ + "bytes", + "with-bytes", +], optional = true } integer-encoding = { version = "4.0.0", optional = true } thiserror = "1" rand = { version = "0.8.5" } futures-intrusive = "0.5.0" directories = { version = "4.0", optional = true } once_cell = "1.18.0" -arcstr = "1.1.5" sha1_smol = "1.0.0" nanoid = "0.4.0" async-trait = { version = "0.1.24" } [features] -socket-layer = ["directories", "integer-encoding", "num_cpus", "protobuf", "tokio-util"] +socket-layer = [ + "directories", + "integer-encoding", + "num_cpus", + "protobuf", + "tokio-util", +] standalone_heartbeat = [] [dev-dependencies] @@ -45,7 +60,9 @@ ctor = "0.2.2" redis = { path = "./redis-rs/redis", features = ["tls-rustls-insecure"] } iai-callgrind = "0.9" tokio = { version = "1", features = ["rt-multi-thread"] } -glide-core = { path = ".", features = ["socket-layer"] } # always enable this feature in tests. +glide-core = { path = ".", features = [ + "socket-layer", +] } # always enable this feature in tests. [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(standalone_heartbeat)'] } diff --git a/glide-core/redis-rs/redis/Cargo.toml b/glide-core/redis-rs/redis/Cargo.toml index 46f6fe9231..0179669eaf 100644 --- a/glide-core/redis-rs/redis/Cargo.toml +++ b/glide-core/redis-rs/redis/Cargo.toml @@ -69,19 +69,19 @@ dashmap = { version = "6.0", optional = true } async-trait = { version = "0.1.24", optional = true } # Only needed for tokio support -backoff-tokio = { package = "backoff", version = "0.4.0", optional = true, features = ["tokio"] } +backoff-tokio = { package = "backoff", version = "0.4.0", optional = true, features = [ + "tokio", +] } # Only needed for native tls native-tls = { version = "0.2", optional = true } tokio-native-tls = { version = "0.3", optional = true } -async-native-tls = { version = "0.4", optional = true } # Only needed for rustls rustls = { version = "0.22", optional = true } webpki-roots = { version = "0.26", optional = true } rustls-native-certs = { version = "0.7", optional = true } tokio-rustls = { version = "0.25", optional = true } -futures-rustls = { version = "0.25", optional = true } rustls-pemfile = { version = "2", optional = true } rustls-pki-types = { version = "1", optional = true } @@ -98,7 +98,6 @@ num-bigint = "0.4.4" ahash = { version = "0.8.11", optional = true } tracing = "0.1" -arcstr = "1.1.5" # Optional uuid support uuid = { version = "1.6.1", optional = true } @@ -114,16 +113,34 @@ default = [ "tokio-rustls-comp", "connection-manager", "cluster", - "cluster-async" + "cluster-async", ] acl = [] -aio = ["bytes", "pin-project-lite", "futures-util", "futures-util/alloc", "futures-util/sink", "tokio/io-util", "tokio-util", "tokio-util/codec", "combine/tokio", "async-trait", "fast-math", "dispose"] +aio = [ + "bytes", + "pin-project-lite", + "futures-util", + "futures-util/alloc", + "futures-util/sink", + "tokio/io-util", + "tokio-util", + "tokio-util/codec", + "combine/tokio", + "async-trait", + "fast-math", + "dispose", +] geospatial = [] json = ["serde", "serde/derive", "serde_json"] cluster = ["crc16", "rand", "derivative"] script = ["sha1_smol"] tls-native-tls = ["native-tls"] -tls-rustls = ["rustls", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types"] +tls-rustls = [ + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", +] tls-rustls-insecure = ["tls-rustls"] tls-rustls-webpki-roots = ["tls-rustls", "webpki-roots"] tokio-comp = ["aio", "tokio/net", "backoff-tokio"] @@ -154,7 +171,12 @@ futures-time = "3" criterion = "0.4" partial-io = { version = "0.5", features = ["tokio", "quickcheck1"] } quickcheck = "1.0.3" -tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread", "time"] } +tokio = { version = "1", features = [ + "rt", + "macros", + "rt-multi-thread", + "time", +] } tempfile = "=3.6.0" once_cell = "1" anyhow = "1" @@ -225,3 +247,6 @@ required-features = ["connection-manager"] [[example]] name = "streams" required-features = ["streams"] + +[package.metadata.cargo-machete] +ignored = ["strum"] diff --git a/glide-core/redis-rs/redis/src/aio/connection.rs b/glide-core/redis-rs/redis/src/aio/connection.rs index 5adef7869f..d07af48f28 100644 --- a/glide-core/redis-rs/redis/src/aio/connection.rs +++ b/glide-core/redis-rs/redis/src/aio/connection.rs @@ -51,27 +51,6 @@ fn test() { assert_sync::(); } -impl Connection { - pub(crate) fn map(self, f: impl FnOnce(C) -> D) -> Connection { - let Self { - con, - buf, - decoder, - db, - pubsub, - protocol, - } = self; - Connection { - con: f(con), - buf, - decoder, - db, - pubsub, - protocol, - } - } -} - impl Connection where C: Unpin + AsyncRead + AsyncWrite + Send, @@ -190,17 +169,6 @@ where } } -pub(crate) async fn connect( - connection_info: &ConnectionInfo, - socket_addr: Option, -) -> RedisResult> -where - C: Unpin + RedisRuntime + AsyncRead + AsyncWrite + Send, -{ - let (con, _ip) = connect_simple::(connection_info, socket_addr).await?; - Connection::new(&connection_info.redis, con).await -} - impl ConnectionLike for Connection where C: Unpin + AsyncRead + AsyncWrite + Send, diff --git a/glide-core/redis-rs/redis/src/client.rs b/glide-core/redis-rs/redis/src/client.rs index fd8c4c08b4..f6a7b4ef91 100644 --- a/glide-core/redis-rs/redis/src/client.rs +++ b/glide-core/redis-rs/redis/src/client.rs @@ -113,22 +113,6 @@ impl Client { crate::aio::Connection::new(&self.connection_info.redis, con).await } - /// Returns an async connection from the client. - #[cfg(feature = "tokio-comp")] - #[cfg_attr(docsrs, doc(cfg(feature = "tokio-comp")))] - #[deprecated( - note = "aio::Connection is deprecated. Use client::get_multiplexed_tokio_connection instead." - )] - #[allow(deprecated)] - pub async fn get_tokio_connection(&self) -> RedisResult { - use crate::aio::RedisRuntime; - Ok( - crate::aio::connect::(&self.connection_info, None) - .await? - .map(RedisRuntime::boxed), - ) - } - /// Returns an async connection from the client. #[cfg(feature = "tokio-comp")] #[cfg_attr(docsrs, doc(cfg(feature = "tokio-comp")))] @@ -245,53 +229,6 @@ impl Client { .await } - /// Returns an async multiplexed connection from the client and a future which must be polled - /// to drive any requests submitted to it (see `get_multiplexed_tokio_connection`). - /// - /// A multiplexed connection can be cloned, allowing requests to be be sent concurrently - /// on the same underlying connection (tcp/unix socket). - /// The multiplexer will return a timeout error on any request that takes longer then `response_timeout`. - #[cfg(feature = "tokio-comp")] - #[cfg_attr(docsrs, doc(cfg(feature = "tokio-comp")))] - pub async fn create_multiplexed_tokio_connection_with_response_timeout( - &self, - response_timeout: std::time::Duration, - glide_connection_options: GlideConnectionOptions, - ) -> RedisResult<( - crate::aio::MultiplexedConnection, - impl std::future::Future, - )> { - self.create_multiplexed_async_connection_inner::( - response_timeout, - None, - glide_connection_options, - ) - .await - .map(|(conn, driver, _ip)| (conn, driver)) - } - - /// Returns an async multiplexed connection from the client and a future which must be polled - /// to drive any requests submitted to it (see `get_multiplexed_tokio_connection`). - /// - /// A multiplexed connection can be cloned, allowing requests to be be sent concurrently - /// on the same underlying connection (tcp/unix socket). - #[cfg(feature = "tokio-comp")] - #[cfg_attr(docsrs, doc(cfg(feature = "tokio-comp")))] - pub async fn create_multiplexed_tokio_connection( - &self, - glide_connection_options: GlideConnectionOptions, - ) -> RedisResult<( - crate::aio::MultiplexedConnection, - impl std::future::Future, - )> { - self.create_multiplexed_tokio_connection_with_response_timeout( - std::time::Duration::MAX, - glide_connection_options, - ) - .await - .map(|conn_res| (conn_res.0, conn_res.1)) - } - /// Returns an async [`ConnectionManager`][connection-manager] from the client. /// /// The connection manager wraps a @@ -375,45 +312,6 @@ impl Client { .await } - /// Returns an async [`ConnectionManager`][connection-manager] from the client. - /// - /// The connection manager wraps a - /// [`MultiplexedConnection`][multiplexed-connection]. If a command to that - /// connection fails with a connection error, then a new connection is - /// established in the background and the error is returned to the caller. - /// - /// This means that on connection loss at least one command will fail, but - /// the connection will be re-established automatically if possible. Please - /// refer to the [`ConnectionManager`][connection-manager] docs for - /// detailed reconnecting behavior. - /// - /// A connection manager can be cloned, allowing requests to be be sent concurrently - /// on the same underlying connection (tcp/unix socket). - /// - /// [connection-manager]: aio/struct.ConnectionManager.html - /// [multiplexed-connection]: aio/struct.MultiplexedConnection.html - #[cfg(feature = "connection-manager")] - #[cfg_attr(docsrs, doc(cfg(feature = "connection-manager")))] - #[deprecated(note = "use get_connection_manager_with_backoff_and_timeouts instead")] - pub async fn get_tokio_connection_manager_with_backoff_and_timeouts( - &self, - exponent_base: u64, - factor: u64, - number_of_retries: usize, - response_timeout: std::time::Duration, - connection_timeout: std::time::Duration, - ) -> RedisResult { - crate::aio::ConnectionManager::new_with_backoff_and_timeouts( - self.clone(), - exponent_base, - factor, - number_of_retries, - response_timeout, - connection_timeout, - ) - .await - } - /// Returns an async [`ConnectionManager`][connection-manager] from the client. /// /// The connection manager wraps a diff --git a/glide-core/redis-rs/redis/src/cluster_async/connections_container.rs b/glide-core/redis-rs/redis/src/cluster_async/connections_container.rs index 2bfbb8b934..72d3c69efa 100644 --- a/glide-core/redis-rs/redis/src/cluster_async/connections_container.rs +++ b/glide-core/redis-rs/redis/src/cluster_async/connections_container.rs @@ -359,7 +359,7 @@ mod tests { } fn create_container_with_strategy( - stragey: ReadFromReplicaStrategy, + strategy: ReadFromReplicaStrategy, use_management_connections: bool, ) -> ConnectionsContainer { let slot_map = SlotMap::new( @@ -409,7 +409,7 @@ mod tests { ConnectionsContainer { slot_map, connection_map, - read_from_replica_strategy: stragey, + read_from_replica_strategy: strategy, topology_hash: 0, } } diff --git a/glide-core/redis-rs/redis/src/cluster_async/connections_logic.rs b/glide-core/redis-rs/redis/src/cluster_async/connections_logic.rs index 7de2493000..7a2306ca19 100644 --- a/glide-core/redis-rs/redis/src/cluster_async/connections_logic.rs +++ b/glide-core/redis-rs/redis/src/cluster_async/connections_logic.rs @@ -5,7 +5,7 @@ use super::{ Connect, }; use crate::{ - aio::{ConnectionLike, DisconnectNotifier, Runtime}, + aio::{ConnectionLike, DisconnectNotifier}, client::GlideConnectionOptions, cluster::get_connection_info, cluster_client::ClusterParams, @@ -462,9 +462,7 @@ async fn check_connection(conn: &mut C, timeout: std::time::Duration) -> Redi where C: ConnectionLike + Send + 'static, { - Runtime::locate() - .timeout(timeout, crate::cmd("PING").query_async::<_, String>(conn)) - .await??; + tokio::time::timeout(timeout, crate::cmd("PING").query_async::<_, String>(conn)).await??; Ok(()) } diff --git a/glide-core/redis-rs/redis/src/cluster_async/mod.rs b/glide-core/redis-rs/redis/src/cluster_async/mod.rs index c8628c16bb..3b5733f128 100644 --- a/glide-core/redis-rs/redis/src/cluster_async/mod.rs +++ b/glide-core/redis-rs/redis/src/cluster_async/mod.rs @@ -1207,7 +1207,7 @@ where } } - // Validate all existing user connections and try to reconnect if nessesary. + // Validate all existing user connections and try to reconnect if necessary. // In addition, as a safety measure, drop nodes that do not have any assigned slots. // This function serves as a cheap alternative to slot_refresh() and thus can be used much more frequently. // The function does not discover the topology from the cluster and assumes the cached topology is valid. @@ -1282,7 +1282,7 @@ where let connections_container = inner.conn_lock.read().await; let cluster_params = &inner.cluster_params; let subscriptions_by_address = &inner.subscriptions_by_address; - let glide_connection_optons = &inner.glide_connection_options; + let glide_connection_options = &inner.glide_connection_options; stream::iter(addresses.into_iter()) .fold( @@ -1304,7 +1304,7 @@ where node_option, &cluster_params, conn_type, - glide_connection_optons.clone(), + glide_connection_options.clone(), ) .await; match node { @@ -1517,7 +1517,7 @@ where Self::check_topology_and_refresh_if_diff(inner.clone(), &RefreshPolicy::Throttable) .await; if !topology_changed { - // This serves as a safety measure for validating pubsub subsctiptions state in case it has drifted + // This serves as a safety measure for validating pubsub subscriptions state in case it has drifted // while topology stayed the same. // For example, a failed attempt to refresh a connection which is triggered from refresh_pubsub_subscriptions(), // might leave a node unconnected indefinitely in case topology is stable and no request are attempted to this node. diff --git a/glide-core/redis-rs/redis/src/cluster_client.rs b/glide-core/redis-rs/redis/src/cluster_client.rs index 5815bede1e..185b8547ad 100644 --- a/glide-core/redis-rs/redis/src/cluster_client.rs +++ b/glide-core/redis-rs/redis/src/cluster_client.rs @@ -327,13 +327,13 @@ impl ClusterClientBuilder { self } - /// Sets maximal wait time in millisceonds between retries for the new ClusterClient. + /// Sets maximal wait time in milliseconds between retries for the new ClusterClient. pub fn max_retry_wait(mut self, max_wait: u64) -> ClusterClientBuilder { self.builder_params.retries_configuration.max_wait_time = max_wait; self } - /// Sets minimal wait time in millisceonds between retries for the new ClusterClient. + /// Sets minimal wait time in milliseconds between retries for the new ClusterClient. pub fn min_retry_wait(mut self, min_wait: u64) -> ClusterClientBuilder { self.builder_params.retries_configuration.min_wait_time = min_wait; self @@ -400,9 +400,9 @@ impl ClusterClientBuilder { } /// Enables periodic connections checks for this client. - /// If enabled, the conenctions to the cluster nodes will be validated periodicatly, per configured interval. + /// If enabled, the connections to the cluster nodes will be validated periodically, per configured interval. /// In addition, for tokio runtime, passive disconnections could be detected instantly, - /// triggering reestablishemnt, w/o waiting for the next periodic check. + /// triggering reestablishment, w/o waiting for the next periodic check. #[cfg(feature = "cluster-async")] pub fn periodic_connections_checks(mut self, interval: Duration) -> ClusterClientBuilder { self.builder_params.connections_validation_interval = Some(interval); diff --git a/glide-core/redis-rs/redis/src/types.rs b/glide-core/redis-rs/redis/src/types.rs index a024f16a7d..5f050028b1 100644 --- a/glide-core/redis-rs/redis/src/types.rs +++ b/glide-core/redis-rs/redis/src/types.rs @@ -178,6 +178,12 @@ pub(crate) enum ServerError { }, } +impl From for RedisError { + fn from(_: tokio::time::error::Elapsed) -> Self { + RedisError::from((ErrorKind::IoError, "Operation timed out")) + } +} + impl From for RedisError { fn from(value: ServerError) -> Self { // TODO - Consider changing RedisError to explicitly represent whether an error came from the server or not. Today it is only implied. diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index a3391c3282..7185ce1359 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -65,11 +65,7 @@ Before starting this step, make sure you've installed all software requirments. git clone https://github.com/valkey-io/valkey-glide.git cd valkey-glide ``` -2. Initialize git submodule: - ```bash - git submodule update --init --recursive - ``` -3. Install all node dependencies: +2. Install all node dependencies: ```bash cd node @@ -79,7 +75,7 @@ Before starting this step, make sure you've installed all software requirments. cd .. ``` -4. Build the Node wrapper (Choose a build option from the following and run it from the `node` folder): +3. Build the Node wrapper (Choose a build option from the following and run it from the `node` folder): 1. Build in release mode, stripped from all debug symbols (optimized and minimized binary size): @@ -101,14 +97,14 @@ Before starting this step, make sure you've installed all software requirments. Once building completed, you'll find the compiled JavaScript code in the`./build-ts` folder. -5. Run tests: +4. Run tests: 1. Ensure that you have installed server and valkey-cli on your host. You can download Valkey at the following link: [Valkey Download page](https://valkey.io/download/). 2. Execute the following command from the node folder: ```bash npm run build # make sure we have a debug build compiled first npm test ``` -6. Integrating the built GLIDE package into your project: +5. Integrating the built GLIDE package into your project: Add the package to your project using the folder path with the command `npm install /node`. - For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. diff --git a/node/tests/SharedTests.ts b/node/tests/SharedTests.ts index eea277241a..7b1af14ed1 100644 --- a/node/tests/SharedTests.ts +++ b/node/tests/SharedTests.ts @@ -9269,7 +9269,6 @@ export function runBaseTests(config: { { sortOrder: SortOrder.DESC, storeDist: true }, ), ).toEqual(3); - // TODO deep close to https://github.com/maasencioh/jest-matcher-deep-close-to expect( await client.zrangeWithScores( key2, @@ -9277,11 +9276,20 @@ export function runBaseTests(config: { { reverse: true }, ), ).toEqual( - convertElementsAndScores({ - edge2: 236529.17986494553, - Palermo: 166274.15156960033, - Catania: 0.0, - }), + expect.arrayContaining([ + { + element: "edge2", + score: expect.closeTo(236529.17986494553, 0.0001), + }, + { + element: "Palermo", + score: expect.closeTo(166274.15156960033, 0.0001), + }, + { + element: "Catania", + score: expect.closeTo(0.0, 0.0001), + }, + ]), ); // test search by box, unit: feet, from member, with limited count 2, with hash