[Auto_Scan]Summary statistics of op/pass (#8079)

lite/tests/unittest_py/auto_scan_base.py

@@ -16,12 +16,14 @@
 import unittest
 import abc
 import os
+import sys
 import enum
 import time
 import logging
 import shutil
 import paddle
 import paddle.fluid as fluid
+import global_var_model as gl
 from paddle.fluid.initializer import NumpyArrayInitializer
 from paddle.fluid.core import PassVersionChecker
 import paddle.fluid.core as core
@@ -248,6 +250,7 @@ def run_test(self, quant=False, prog_configs=None):
                 # judge validity of program
                 if not self.is_program_valid(prog_config, paddlelite_config):
                     self.num_invalid_programs_list[predictor_idx] += 1
+                    gl.set_not_supported_ops(self.get_target(), sys.argv[0])
                     continue
                 self.num_ran_programs_list[predictor_idx] += 1
 
@@ -283,6 +286,7 @@ def run_test(self, quant=False, prog_configs=None):
                             self.ignore_log("[ACCURACY_ERROR] " + ignore_info[
                                 2] + ' ' + ' vs ' + self.paddlelite_config_str(
                                     pred_config))
+                            gl.set_out_diff_ops(self.get_target(), sys.argv[0])
                         elif ignore_info[
                                 1] == IgnoreReasonsBase.PADDLELITE_NOT_SUPPORT:
                             paddle_lite_not_support_flag = True
@@ -295,6 +299,7 @@ def run_test(self, quant=False, prog_configs=None):
                             raise NotImplementedError
                         break
                 if paddle_lite_not_support_flag:
+                    gl.set_not_supported_ops(self.get_target(), sys.argv[0])
                     continue
 
                 if os.path.exists(self.cache_dir):
@@ -329,6 +334,7 @@ def run_test(self, quant=False, prog_configs=None):
                 self.success_log('PredictorConfig: ' +
                                  self.paddlelite_config_str(pred_config))
         self.assertTrue(status)
+        gl.set_success_ops(self.get_target(), sys.argv[0])
 
     def inference_config_str(self, config) -> bool:
         dic = {}
@@ -428,6 +434,7 @@ def run_test(prog_config):
             return self.run_test(quant=quant, prog_configs=[prog_config])
 
         # if current unittest is not active on the input targ    paddlelite_not_support_flag = Trueet, we will exit directly.
+        gl.set_all_test_ops(self.get_target(), sys.argv[0])
         if not self.is_actived():
             logging.info("Error: This test is not actived on " +
                          self.get_target())

lite/tests/unittest_py/global_var_model.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+from pathlib import Path
+import os
+
+statics_data = {
+    "targets": set(),
+    "all_test_ops": {
+        "Host": set(),
+        "X86": set(),
+        "ARM": set(),
+        "OpenCL": set(),
+        "Metal": set()
+    },
+    "success_ops": {
+        "Host": set(),
+        "X86": set(),
+        "ARM": set(),
+        "OpenCL": set(),
+        "Metal": set()
+    },
+    "out_diff_ops": {
+        "Host": set(),
+        "X86": set(),
+        "ARM": set(),
+        "OpenCL": set(),
+        "Metal": set()
+    },
+    "not_supported_ops": {
+        "Host": set(),
+        "X86": set(),
+        "ARM": set(),
+        "OpenCL": set(),
+        "Metal": set()
+    },
+}
+static_file = Path("./statics_data")
+static_file_path_str = "./statics_data"
+
+
+# coding=utf-8
+def set_value(kind, target, op):
+    if not static_file.exists():
+        global statics_data
+    else:
+        with open(static_file_path_str, "rb") as f:
+            statics_data = pickle.load(f)
+
+    statics_data["targets"].add(target)
+    statics_data[kind][target].add(op)
+
+    with open(static_file_path_str, "wb") as f:
+        pickle.dump(statics_data, f)
+
+
+def set_all_test_ops(target, op):
+    set_value("all_test_ops", target, op)
+
+
+def set_success_ops(target, op):
+    set_value("success_ops", target, op)
+
+
+def set_out_diff_ops(target, op):
+    set_value("out_diff_ops", target, op)
+
+
+def set_not_supported_ops(target, op):
+    set_value("not_supported_ops", target, op)
+
+
+def display():
+    print("----------------------Unit Test Summary---------------------")
+    with open("./statics_data", "rb") as f:
+        statics_data = pickle.load(f)
+        targets = statics_data["targets"]
+
+    for target in targets:
+        all_test_ops = statics_data["all_test_ops"][target]
+        not_supported_ops = statics_data["not_supported_ops"][target]
+        out_diff_ops = statics_data["out_diff_ops"][target]
+        success_ops = statics_data["success_ops"][
+            target] - not_supported_ops - out_diff_ops
+
+        print("Target =", target)
+        print("Number of test ops =", len(all_test_ops))
+        print("Number of success ops =", len(success_ops))
+        print("Number of not supported ops =", len(not_supported_ops))
+        print("Number of output diff ops =", len(out_diff_ops))
+        print("\nDetails:")
+        print("Success ops:")
+        print(list(success_ops))
+        print("\nNot supported ops:")
+        print(list(not_supported_ops))
+        print("\nOutput diff ops:")
+        print(list(out_diff_ops))
+        print("\n")
+
+
+if __name__ == "__main__":
+    display()

lite/tests/unittest_py/op/auto_scan.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+unset GREP_OPTIONS
+ARCH=""
+TEST_FILE=""
+COLLECT_TEST_INFO=false
+function get_inputs {
+    # Parse command line.
+    for i in "$@"; do
+        case $i in
+            --target=*)
+                ARCH="${i#*=}"
+                shift
+                ;;
+            test_info)
+                COLLECT_TEST_INFO=true
+                shift
+                ;;
+            *)
+                TEST_FILE="$i"
+                ;;
+        esac
+    done
+}
+
+function run_test {
+  if [[ $ARCH = "" ]] || [[ $1 = "" ]]; then
+    echo "Error input: ./auto_scan test_assign_op.py --target=Host"
+    exit 1
+  fi
+
+  if [[ $ARCH = "ARM" ]] || [[ $ARCH = "OpenCL" ]] || [[ $ARCH = "Metal" ]]; then
+    cd ../rpc_service
+    sh start_rpc_server.sh
+    cd ../op
+    python3.8 $1 --target=$ARCH
+  else
+    python3.7 $1 --target=$ARCH
+  fi
+}
+
+get_inputs $@
+
+if [ $COLLECT_TEST_INFO = true ]; then
+  tests=$(ls | grep test)
+  for test in $tests; do
+      run_test $test
+  done
+else
+  run_test $TEST_FILE
+  rm -f ../.test_num ../.test_names
+fi
+
+if [[ $ARCH = "ARM" ]] || [[ $ARCH = "OpenCL" ]] || [[ $ARCH = "Metal" ]]; then
+  python3.9 ../global_var_model.py
+else
+  python3.7 ../global_var_model.py
+fi

lite/tests/unittest_py/pass/auto_scan.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+set -e
+unset GREP_OPTIONS
+ARCH=""
+TEST_FILE=""
+COLLECT_TEST_INFO=false
+function get_inputs {
+    # Parse command line.
+    for i in "$@"; do
+        case $i in
+            --target=*)
+                ARCH="${i#*=}"
+                shift
+                ;;
+            test_info)
+                COLLECT_TEST_INFO=true
+                shift
+                ;;
+            *)
+                # unknown option
+                TEST_FILE="$i"
+                ;;
+        esac
+    done
+}
+
+function run_test {
+  if [[ $ARCH = "" ]] || [[ $1 = "" ]]; then
+    echo "Error input: ./auto_scan test_assign_op.py --target=Host"
+    exit 1
+  fi
+
+  if [[ $ARCH = "ARM" ]] || [[ $ARCH = "OpenCL" ]] || [[ $ARCH = "Metal" ]]; then
+    cd ../rpc_service
+    sh start_rpc_server.sh
+    cd ../op
+    python3.8 $1 --target=$ARCH
+  else
+    python3.7 $1 --target=$ARCH
+  fi
+}
+
+get_inputs $@
+
+if [ $COLLECT_TEST_INFO = true ]; then
+  tests=$(ls | grep test)
+  for test in $tests; do
+      run_test $test
+  done
+else
+  run_test $TEST_FILE
+  rm -f ../.test_num ../.test_names
+fi
+
+if [[ $ARCH = "ARM" ]] || [[ $ARCH = "OpenCL" ]] || [[ $ARCH = "Metal" ]]; then
+  python3.9 ../global_var_model.py
+else
+  python3.7 ../global_var_model.py
+fi