Bump up python formatter yapf version (#13916)

This commit updates python formatter yapf to 0.40.2 and python code.
Update documents and dockerfiles

ONE-DCO-1.0-Signed-off-by: Hyeongseok Oh <hseok82.oh@samsung.com>

.github/workflows/check-format.yml

@@ -30,15 +30,15 @@ jobs:
           python-version: '3.x'
 
       # C format: clang-format-16
-      # Python format: yapf==0.22.0
+      # Python format: yapf==0.40.2
       - name: Install packages
         run: |
           sudo apt-get install -y gnupg2 software-properties-common
           wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
           sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main"
           sudo apt-get update && sudo apt-get install -qqy clang-format-16
           python -m pip install --upgrade pip
-          pip install yapf==0.22.0
+          pip install yapf==0.40.2
 
       - name: Check
         run: ./nnas format

.style.yapf

@@ -1,3 +1,4 @@
 [style]
 BASED_ON_STYLE = pep8
 COLUMN_LIMIT = 90
+BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = False

compiler/bcq-tools/generate_bcq_metadata.py

@@ -37,24 +37,21 @@ def _get_parser():
         description=("Command line tool to generate metadata of BCQ nodes"))
 
     # Input and output path.
-    parser.add_argument(
-        "-i",
-        "--input_path",
-        type=str,
-        help="Full filepath of the input file.",
-        required=True)
-    parser.add_argument(
-        "-o",
-        "--output_path",
-        type=str,
-        help="Full filepath of the output file.",
-        required=True)
-    parser.add_argument(
-        "-O",
-        "--output_arrays",
-        type=str,
-        help="Original model output arrays",
-        required=True)
+    parser.add_argument("-i",
+                        "--input_path",
+                        type=str,
+                        help="Full filepath of the input file.",
+                        required=True)
+    parser.add_argument("-o",
+                        "--output_path",
+                        type=str,
+                        help="Full filepath of the output file.",
+                        required=True)
+    parser.add_argument("-O",
+                        "--output_arrays",
+                        type=str,
+                        help="Original model output arrays",
+                        required=True)
 
     return parser
 

compiler/circle-part-value-py-test/conftest.py

@@ -10,8 +10,9 @@ def extract_test_args(s):
 def pytest_addoption(parser):
     parser.addoption("--test_list", action="store", help="Path to test list")
     parser.addoption("--bin_dir", action="store", help="Directory including artifacts")
-    parser.addoption(
-        "--circle_part_driver", action="store", help="Path to circle part driver")
+    parser.addoption("--circle_part_driver",
+                     action="store",
+                     help="Path to circle part driver")
 
 
 def pytest_generate_tests(metafunc):

compiler/circle-part-value-py-test/test_circle_part_value.py

@@ -57,24 +57,20 @@ def part_eval(test_name, bin_dir, circle_part_driver):
     for i in range(num_inputs):
         input_details = interpreter.get_input_details()[i]
         if input_details["dtype"] == np.float32:
-            input_data = np.array(
-                np.random.random_sample(input_details["shape"]), input_details["dtype"])
+            input_data = np.array(np.random.random_sample(input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.uint8:
-            input_data = np.array(
-                np.random.randint(0, 256, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 256, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int16:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int32:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int64:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.bool_:
             input_data = np.array(
                 np.random.choice(a=[True, False], size=input_details["shape"]),

compiler/circle-part-value-test/part_eval_one.py

@@ -72,18 +72,17 @@
     input_details_dtype = input_details["dtype"]
     input_details_shape = input_details["shape"]
     if input_details_dtype == np.float32:
-        input_data = np.array(
-            np.random.random_sample(input_details_shape), input_details_dtype)
+        input_data = np.array(np.random.random_sample(input_details_shape),
+                              input_details_dtype)
     elif input_details_dtype == np.int16:
-        input_data = np.array(
-            np.random.randint(0, 100, size=input_details_shape), input_details_dtype)
+        input_data = np.array(np.random.randint(0, 100, size=input_details_shape),
+                              input_details_dtype)
     elif input_details_dtype == np.uint8:
-        input_data = np.array(
-            np.random.randint(0, 256, size=input_details_shape), input_details_dtype)
+        input_data = np.array(np.random.randint(0, 256, size=input_details_shape),
+                              input_details_dtype)
     elif input_details_dtype == np.bool_:
-        input_data = np.array(
-            np.random.choice(a=[True, False], size=input_details_shape),
-            input_details_dtype)
+        input_data = np.array(np.random.choice(a=[True, False], size=input_details_shape),
+                              input_details_dtype)
     else:
         raise SystemExit("Unsupported input dtype")
 
@@ -124,8 +123,8 @@
                 raise SystemExit("Execution result of " + tflite_model +
                                  " does not match with " + circle_model)
         elif output_dtype == np.float32:
-            if np.allclose(
-                    luci_output_data, intp_output_data, rtol=1.e-5, atol=1.e-5) == False:
+            if np.allclose(luci_output_data, intp_output_data, rtol=1.e-5,
+                           atol=1.e-5) == False:
                 raise SystemExit("Execution result of " + tflite_model +
                                  " does not match with " + circle_model)
         elif output_dtype == np.int64:

compiler/fm-equalize/fm-equalize

@@ -33,14 +33,16 @@ from pathlib import Path
 def _get_parser():
     parser = argparse.ArgumentParser(
         description='Command line tool to equalize feature map (FM) value distribution')
-    parser.add_argument(
-        "-i", "--input", type=str, help="Path to the input circle model.", required=True)
-    parser.add_argument(
-        "-o",
-        "--output",
-        type=str,
-        help="Path to the output circle model.",
-        required=True)
+    parser.add_argument("-i",
+                        "--input",
+                        type=str,
+                        help="Path to the input circle model.",
+                        required=True)
+    parser.add_argument("-o",
+                        "--output",
+                        type=str,
+                        help="Path to the output circle model.",
+                        required=True)
     parser.add_argument(
         "-f",
         "--fme_patterns",
@@ -133,11 +135,10 @@ def main():
             Path(output_model).with_suffix('.fme_patterns.json').name)
 
         # Step 1. Run fme-detect to find equalization patterns
-        _run_fme_detect(
-            str(input_model),
-            str(fme_patterns),
-            verbose=verbose,
-            allow_dup_op=allow_dup_op)
+        _run_fme_detect(str(input_model),
+                        str(fme_patterns),
+                        verbose=verbose,
+                        allow_dup_op=allow_dup_op)
 
         # Copy fme_patterns to the given path
         if args.fme_patterns != None:
@@ -152,8 +153,10 @@ def main():
             os.system(f'cp {fme_patterns} {args.fme_patterns}')
 
         # Step 3. Run fme-apply
-        _run_fme_apply(
-            str(input_model), str(fme_patterns), str(output_model), verbose=verbose)
+        _run_fme_apply(str(input_model),
+                       str(fme_patterns),
+                       str(output_model),
+                       verbose=verbose)
 
 
 if __name__ == '__main__':

compiler/luci-pass-value-py-test/conftest.py

@@ -9,12 +9,15 @@ def extract_test_args(s):
 
 def pytest_addoption(parser):
     parser.addoption("--test_list", action="store", help="Path to test list")
-    parser.addoption(
-        "--tflite_dir", action="store", help="Directory including tflite file")
-    parser.addoption(
-        "--circle_dir", action="store", help="Directory including circle file")
-    parser.addoption(
-        "--luci_eval_driver", action="store", help="Path to luci eval driver")
+    parser.addoption("--tflite_dir",
+                     action="store",
+                     help="Directory including tflite file")
+    parser.addoption("--circle_dir",
+                     action="store",
+                     help="Directory including circle file")
+    parser.addoption("--luci_eval_driver",
+                     action="store",
+                     help="Path to luci eval driver")
 
 
 def pytest_generate_tests(metafunc):

compiler/luci-pass-value-py-test/test_luci_eval.py

@@ -38,24 +38,20 @@ def luci_eval_verify(test_name,
     for i in range(num_inputs):
         input_details = interpreter.get_input_details()[i]
         if input_details["dtype"] == np.float32:
-            input_data = np.array(
-                np.random.random_sample(input_details["shape"]), input_details["dtype"])
+            input_data = np.array(np.random.random_sample(input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.uint8:
-            input_data = np.array(
-                np.random.randint(0, 256, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 256, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int16:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int32:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.int64:
-            input_data = np.array(
-                np.random.randint(0, 100, size=input_details["shape"]),
-                input_details["dtype"])
+            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                                  input_details["dtype"])
         elif input_details["dtype"] == np.bool_:
             input_data = np.array(
                 np.random.choice(a=[True, False], size=input_details["shape"]),
@@ -70,12 +66,11 @@ def luci_eval_verify(test_name,
     interpreter.invoke()
 
     # Execute luci interpreter.
-    subprocess.run(
-        [
-            eval_driver, circle_model,
-            str(num_inputs), circle_model + ".input", circle_model + ".output"
-        ],
-        check=True)
+    subprocess.run([
+        eval_driver, circle_model,
+        str(num_inputs), circle_model + ".input", circle_model + ".output"
+    ],
+                   check=True)
 
     # Compare the results.
     inpt_output_details = interpreter.get_output_details()
@@ -92,23 +87,33 @@ def luci_eval_verify(test_name,
         intp_output_data = interpreter.get_tensor(output_tensor)
         err_msg = "Execution result of " + tflite_model + " does not match with " + circle_model
         if output_details["dtype"] == np.uint8:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=rtolint, atol=atolint), err_msg
+            assert np.allclose(luci_output_data,
+                               intp_output_data,
+                               rtol=rtolint,
+                               atol=atolint), err_msg
         elif output_details["dtype"] == np.float32:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=rtolf32, atol=atolf32), err_msg
+            assert np.allclose(luci_output_data,
+                               intp_output_data,
+                               rtol=rtolf32,
+                               atol=atolf32), err_msg
         elif output_details["dtype"] == np.int64:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=rtolint, atol=atolint), err_msg
+            assert np.allclose(luci_output_data,
+                               intp_output_data,
+                               rtol=rtolint,
+                               atol=atolint), err_msg
         elif output_details["dtype"] == np.int32:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=rtolint, atol=atolint), err_msg
+            assert np.allclose(luci_output_data,
+                               intp_output_data,
+                               rtol=rtolint,
+                               atol=atolint), err_msg
         elif output_details["dtype"] == np.int16:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=rtolint, atol=atolint), err_msg
+            assert np.allclose(luci_output_data,
+                               intp_output_data,
+                               rtol=rtolint,
+                               atol=atolint), err_msg
         elif output_details["dtype"] == np.bool_:
-            assert np.allclose(
-                luci_output_data, intp_output_data, rtol=0, atol=0), err_msg
+            assert np.allclose(luci_output_data, intp_output_data, rtol=0,
+                               atol=0), err_msg
         else:
             assert False, "Unsupported data type: " + output_details["dtype"]
 

compiler/luci-pass-value-test/eval_result_verifier.py

@@ -39,16 +39,14 @@
 for i in range(num_inputs):
     input_details = interpreter.get_input_details()[i]
     if input_details["dtype"] == np.float32:
-        input_data = np.array(
-            np.random.random_sample(input_details["shape"]), input_details["dtype"])
+        input_data = np.array(np.random.random_sample(input_details["shape"]),
+                              input_details["dtype"])
     elif input_details["dtype"] == np.uint8:
-        input_data = np.array(
-            np.random.randint(0, 256, size=input_details["shape"]),
-            input_details["dtype"])
+        input_data = np.array(np.random.randint(0, 256, size=input_details["shape"]),
+                              input_details["dtype"])
     elif input_details["dtype"] == np.int16:
-        input_data = np.array(
-            np.random.randint(0, 100, size=input_details["shape"]),
-            input_details["dtype"])
+        input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
+                              input_details["dtype"])
     elif input_details["dtype"] == np.bool_:
         input_data = np.array(
             np.random.choice(a=[True, False], size=input_details["shape"]),
@@ -63,12 +61,11 @@
 interpreter.invoke()
 
 # Execute luci interpreter.
-subprocess.run(
-    [
-        driver, circle_model,
-        str(num_inputs), circle_model + ".input", circle_model + ".output"
-    ],
-    check=True)
+subprocess.run([
+    driver, circle_model,
+    str(num_inputs), circle_model + ".input", circle_model + ".output"
+],
+               check=True)
 
 # Compare the results.
 inpt_output_details = interpreter.get_output_details()
@@ -89,8 +86,8 @@
                 raise SystemExit("Execution result of " + tflite_model +
                                  " does not match with " + circle_model)
         elif output_details["dtype"] == np.float32:
-            if np.allclose(
-                    luci_output_data, intp_output_data, rtol=1.e-5, atol=1.e-5) == False:
+            if np.allclose(luci_output_data, intp_output_data, rtol=1.e-5,
+                           atol=1.e-5) == False:
                 raise SystemExit("Execution result of " + tflite_model +
                                  " does not match with " + circle_model)
         elif output_details["dtype"] == np.int64:

compiler/luci-ref-value-py-test/conftest.py

@@ -15,8 +15,9 @@ def pytest_addoption(parser):
     parser.addoption("--tflrecipe", action="store", help="Path to tfl recipies")
     parser.addoption("--circlerecipe", action="store", help="Path to circle recipies")
     parser.addoption("--binary", action="store", help="Path to test binary")
-    parser.addoption(
-        "--luci_eval_driver", action="store", help="Path to luci eval driver")
+    parser.addoption("--luci_eval_driver",
+                     action="store",
+                     help="Path to luci eval driver")
 
 
 def copy_if_changed(src_filepath, dst_filepath):