From 8c83e0db9e5fe8d50a2914792846092d36e6e8f5 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 15:21:35 +0900 Subject: [PATCH] fix: Broken python binding CI --- README.md | 10 +++------- binding/python/examples/cli/setup.py | 8 ++------ binding/python/examples/main.py | 4 ++++ binding/python/examples/utils.py | 10 ++++++++++ binding/python/tests/data_replication.py | 8 +++++++- binding/python/tests/utils.py | 10 ++++++++++ 6 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 binding/python/examples/utils.py diff --git a/README.md b/README.md index f28cd757..a12809cb 100644 --- a/README.md +++ b/README.md @@ -199,13 +199,9 @@ Last index: 3 ## Bootstrapping from WAL -You can bootstrap cluster from WAL (Write Ahead Logs), and WAL's snapshot. - -This feature is useful in cases where a failure occurs in more than the number of nodes in the quorum, requiring a restart of the cluster, or when there is a need to reboot the cluster after making a batch change to the cluster members. - -Use the `restore_wal_from` and `restore_wal_snapshot_from` options in `RaftConfig`. - -See [this example](https://github.com/lablup/raftify/blob/main/examples/memstore/static-members/src/main.rs) for more details. +If there are previous logs remaining in the log directory, the raft node will automatically apply them after the node is bootstrapped. +If you intend to bootstrap the cluster from the scratch, please remove the previous log directory. +To ignore the previous logs and bootstrap the cluster from a snapshot, use the `Config.bootstrap_from_snapshot` option. ## Support for other languages diff --git a/binding/python/examples/cli/setup.py b/binding/python/examples/cli/setup.py index 638eeeeb..1a6d236e 100644 --- a/binding/python/examples/cli/setup.py +++ b/binding/python/examples/cli/setup.py @@ -23,7 +23,7 @@ def get_requirements(env: str = None): version=VERSION, description="Experimental High level Raft framework", long_description=README, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", author="Lablup Inc.", maintainer="jopemachine", maintainer_email="jopemachine@naver.com", @@ -35,9 +35,5 @@ def get_requirements(env: str = None): install_requires=default_requires, zip_safe=False, include_package_data=True, - entry_points={ - 'console_scripts': [ - 'raftify_cli = raftify_cli.cli:main' - ] - } + entry_points={"console_scripts": ["raftify_cli = raftify_cli.cli:main"]}, ) diff --git a/binding/python/examples/main.py b/binding/python/examples/main.py index c0456eba..e571ee92 100644 --- a/binding/python/examples/main.py +++ b/binding/python/examples/main.py @@ -20,6 +20,7 @@ from .deserializer import register_custom_deserializer from .web_server_api import routes, WebServer from .state_machine import HashStore +from .utils import ensure_directory_exist, get_storage_path def load_peers() -> Peers: @@ -113,6 +114,9 @@ async def main(): logger = Logger(setup_logger()) store = HashStore() + storage_path = get_storage_path(cfg.log_dir, node_id) + ensure_directory_exist(storage_path) + tasks = [] raft = Raft.bootstrap(node_id, raft_addr, store, cfg, logger) tasks.append(raft.run()) diff --git a/binding/python/examples/utils.py b/binding/python/examples/utils.py new file mode 100644 index 00000000..543ebc0d --- /dev/null +++ b/binding/python/examples/utils.py @@ -0,0 +1,10 @@ +import os + + +def get_storage_path(log_dir: str, node_id: int) -> str: + return f"{log_dir}/node-{node_id}" + + +def ensure_directory_exist(storage_path: str): + if not os.path.exists(storage_path): + os.makedirs(storage_path) diff --git a/binding/python/tests/data_replication.py b/binding/python/tests/data_replication.py index 60630055..9c456594 100644 --- a/binding/python/tests/data_replication.py +++ b/binding/python/tests/data_replication.py @@ -1,6 +1,10 @@ import asyncio import pytest -from utils import load_peers, wait_for_until_cluster_size_increase +from utils import ( + ensure_directory_exist, + load_peers, + wait_for_until_cluster_size_increase, +) from constant import THREE_NODE_EXAMPLE from harness.raft_server import RAFTS, run_rafts from harness.state_machine import SetCommand @@ -8,6 +12,8 @@ @pytest.mark.asyncio async def test_data_replication(): + ensure_directory_exist("./logs") + peers = load_peers(THREE_NODE_EXAMPLE) asyncio.create_task(run_rafts(peers)) await asyncio.sleep(2) diff --git a/binding/python/tests/utils.py b/binding/python/tests/utils.py index a0501ccb..0e08b92b 100644 --- a/binding/python/tests/utils.py +++ b/binding/python/tests/utils.py @@ -1,4 +1,5 @@ from asyncio import sleep +import os import tomli from pathlib import Path from raftify import InitialRole, Peer, Peers, Raft @@ -45,3 +46,12 @@ async def wait_for_until_cluster_size_decrease(raft: Raft, target: int): # Wait for the conf_change reflected to the cluster await sleep(0.1) + + +def get_storage_path(log_dir: str, node_id: int) -> str: + return f"{log_dir}/node-{node_id}" + + +def ensure_directory_exist(storage_path: str): + if not os.path.exists(storage_path): + os.makedirs(storage_path)