From 4c6e9dd9dd1fb4c6ae6fe5a7de397fc850a39a12 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 27 Jun 2022 16:16:35 -0400 Subject: [PATCH 1/4] Add test for subprocesses in executor jobs The functions executed by the executor may try to spawn subprocesses. For instance, if the function needs to do wandb.init(), it will try to spawn a subprocess. We need to support this functionality. --- tests/unittests/executor/test_executor.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/unittests/executor/test_executor.py b/tests/unittests/executor/test_executor.py index 6ee2f9711..59bd92298 100644 --- a/tests/unittests/executor/test_executor.py +++ b/tests/unittests/executor/test_executor.py @@ -1,6 +1,6 @@ import sys import time -from multiprocessing import TimeoutError +from multiprocessing import TimeoutError, Process import pytest @@ -200,6 +200,13 @@ def nested_jobs(executor): return sum(all_results) +def job_with_subprocess(): + p = Process(target=time.sleep, args=(1,)) + p.start() + p.join() + return 1 + + @pytest.mark.parametrize("backend", [SingleExecutor]) def test_executor_is_serializable(backend): with backend(5) as executor: @@ -282,3 +289,14 @@ def test_executors_del_does_not_raise(backend): del executor.client del executor + + +@pytest.mark.parametrize("backend", backends) +def test_executor_support_subprocess(backend): + """A subprocess can be spawn inside the process""" + + with backend(5) as executor: + futures = [executor.submit(job_with_subprocess) for _ in range(10)] + all_results = executor.wait(futures) + + assert sum(all_results) == 10 From 465209d09e1b0274c1fb5d952d738f82659e1c63 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Mon, 27 Jun 2022 16:20:00 -0400 Subject: [PATCH 2/4] isort --- tests/unittests/executor/test_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/executor/test_executor.py b/tests/unittests/executor/test_executor.py index 59bd92298..df8319042 100644 --- a/tests/unittests/executor/test_executor.py +++ b/tests/unittests/executor/test_executor.py @@ -1,6 +1,6 @@ import sys import time -from multiprocessing import TimeoutError, Process +from multiprocessing import Process, TimeoutError import pytest From b4bbac3baca64082f088b89c05dcc49becf669da Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 12 Jul 2022 12:15:30 -0400 Subject: [PATCH 3/4] Test change for code --- tests/unittests/executor/test_executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/executor/test_executor.py b/tests/unittests/executor/test_executor.py index df8319042..63288e809 100644 --- a/tests/unittests/executor/test_executor.py +++ b/tests/unittests/executor/test_executor.py @@ -201,6 +201,7 @@ def nested_jobs(executor): def job_with_subprocess(): + """Testing PR review code change""" p = Process(target=time.sleep, args=(1,)) p.start() p.join() From 1e49eb3c0d76b3adccac53dd4b7b7e5a849bb310 Mon Sep 17 00:00:00 2001 From: Setepenre Date: Tue, 12 Jul 2022 12:20:15 -0400 Subject: [PATCH 4/4] Remove Fake docstring --- tests/unittests/executor/test_executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/executor/test_executor.py b/tests/unittests/executor/test_executor.py index 63288e809..df8319042 100644 --- a/tests/unittests/executor/test_executor.py +++ b/tests/unittests/executor/test_executor.py @@ -201,7 +201,6 @@ def nested_jobs(executor): def job_with_subprocess(): - """Testing PR review code change""" p = Process(target=time.sleep, args=(1,)) p.start() p.join()