-
Notifications
You must be signed in to change notification settings - Fork 5
/
09-runner-all-files.py
32 lines (25 loc) · 958 Bytes
/
09-runner-all-files.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import subprocess
import time
from glob import glob
import random
test_files = sorted(glob('./data/test_set/log_input_*.csv.gz'))
test_files = [x[-28:] for x in test_files]
commands_to_run = ['/home/sai/.conda/envs/myenv/bin/python 76-05-submission-one-test-file.py {}'.format(x)
for x in test_files]
random.shuffle(commands_to_run)
max_procs = 16
processes = []
while (len(processes) > 0) or (len(commands_to_run) > 0):
if (len(processes) < max_procs) & (len(commands_to_run) > 0):
this_cmd = commands_to_run.pop()
print(this_cmd)
this_proc = subprocess.Popen(this_cmd, shell=True)
processes.append(this_proc)
for i in range(len(processes)):
this_proc = processes[i]
if this_proc.poll() is not None:
return_code = this_proc.poll()
print('Process exited with code: ' + str(return_code))
del processes[i]
break
time.sleep(1)