[NNAdapter] Support Dynamic dims

lite/backends/nnadapter/nnadapter/src/driver/cambricon_mlu/converter/reshape.cc

@@ -15,7 +15,9 @@
 #include "operation/reshape.h"
 #include "driver/cambricon_mlu/converter.h"
 #include "utility/debug.h"
+#include "utility/hints.h"
 #include "utility/logging.h"
+#include "utility/modeling.h"
 
 namespace nnadapter {
 namespace cambricon_mlu {
@@ -28,6 +30,33 @@ int ConvertReshape(Converter* converter, core::Operation* operation) {
   if (!input_tensor) {
     input_tensor = converter->ConvertOperand(input_operand);
   }
+  if (IsTemporaryShapeOperand(shape_operand)) {
+    auto& temporary_shape = *(GetTemporaryShape(shape_operand));
+    auto shape_count = temporary_shape.count;
+    auto shape_data = temporary_shape.data;
+    for (uint32_t i = 0; i < shape_count; i++) {
+      if (shape_data[i] == 0) {
+        if (input_operand->type.dimensions.data[i] == NNADAPTER_UNKNOWN) {
+          shape_data[i] = -1;
+        } else {
+          shape_data[i] = input_operand->type.dimensions.data[i];
+        }
+      }
+    }
+  } else if (IsConstantOperand(shape_operand)) {
+    auto shape_count = shape_operand->length / sizeof(int32_t);
+    auto shape_data = reinterpret_cast<int32_t*>(shape_operand->buffer);
+    for (uint32_t i = 0; i < shape_count; i++) {
+      if (shape_data[i] == 0) {
+        if (input_operand->type.dimensions.data[i] == NNADAPTER_UNKNOWN) {
+          shape_data[i] = -1;
+        } else {
+          shape_data[i] = input_operand->type.dimensions.data[i];
+        }
+      }
+    }
+  }
+
   auto shape_tensor = converter->ConvertOperand(shape_operand);
   auto reshape_node =
       converter->network()->AddIReshapeNode(input_tensor, shape_tensor);

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/converter/fully_connected.cc

@@ -23,10 +23,21 @@ namespace huawei_ascend_npu {
 
 int ConvertFullyConnected(Converter* converter, core::Operation* operation) {
   FULLY_CONNECTED_OPERATION_EXTRACT_INPUTS_OUTPUTS
-  auto batch_size =
-      ProductionOfDimensions(input_operand->type.dimensions.data,
-                             input_operand->type.dimensions.count) /
-      input_size;
+  int64_t input_production = 1;
+  for (uint32_t i = 0; i < input_operand->type.dimensions.count; i++) {
+    auto dimension = input_operand->type.dimensions.data[i];
+    if (dimension < 0) {
+      input_production = -1;
+      break;
+    }
+    input_production *= dimension;
+  }
+  int64_t batch_size;
+  if (input_production < 0) {
+    batch_size = -1;
+  } else {
+    batch_size = input_production / input_size;
+  }
   NNADAPTER_VLOG(5) << "batch_size: " << batch_size;
 
   // Convert to GE operators

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/converter/reshape.cc

@@ -32,39 +32,39 @@ int ConvertReshape(Converter* converter, core::Operation* operation) {
   }
   std::shared_ptr<Operator> shape_operator = nullptr;
   if (IsTemporaryShapeOperand(shape_operand)) {
-    if (IsOperandWithDynamicShape(shape_operand)) {
-      shape_operator = converter->GetMappedOperator(shape_operand);
-      if (!shape_operator) {
-        shape_operator = converter->ConvertOperand(shape_operand);
-      }
-    } else {
-      auto& temporary_shape = *(GetTemporaryShape(shape_operand));
-      auto shape_count = temporary_shape.count;
-      auto shape_data = temporary_shape.data;
-      for (uint32_t i = 0; i < shape_count; i++) {
-        if (shape_data[i] == 0) {
+    auto& temporary_shape = *(GetTemporaryShape(shape_operand));
+    auto shape_count = temporary_shape.count;
+    auto shape_data = temporary_shape.data;
+    for (uint32_t i = 0; i < shape_count; i++) {
+      if (shape_data[i] == 0) {
+        if (input_operand->type.dimensions.data[i] == NNADAPTER_UNKNOWN) {
+          shape_data[i] = -1;
+        } else {
           shape_data[i] = input_operand->type.dimensions.data[i];
         }
       }
-      shape_operator = converter->AddInt32ConstantOperator(
-          std::vector<int32_t>(shape_data, shape_data + shape_count));
     }
+    shape_operator = converter->AddInt32ConstantOperator(
+        std::vector<int32_t>(shape_data, shape_data + shape_count));
   } else if (IsConstantOperand(shape_operand)) {
     auto shape_count = shape_operand->length / sizeof(int32_t);
     auto shape_data = reinterpret_cast<int32_t*>(shape_operand->buffer);
     for (uint32_t i = 0; i < shape_count; i++) {
-      if (shape_data[i] == 0 &&
-          input_operand->type.dimensions.data[i] != NNADAPTER_UNKNOWN) {
-        shape_data[i] = input_operand->type.dimensions.data[i];
+      if (shape_data[i] == 0) {
+        if (input_operand->type.dimensions.data[i] == NNADAPTER_UNKNOWN) {
+          shape_data[i] = -1;
+        } else {
+          shape_data[i] = input_operand->type.dimensions.data[i];
+        }
       }
     }
     shape_operator = converter->AddInt32ConstantOperator(
         std::vector<int32_t>(shape_data, shape_data + shape_count));
   } else {
-    NNADAPTER_LOG(FATAL) << "Unsupported shape lifetime: "
-                         << OperandLifetimeCodeToString(
-                                shape_operand->type.lifetime);
-    return NNADAPTER_INVALID_PARAMETER;
+    shape_operator = converter->GetMappedOperator(shape_operand);
+    if (!shape_operator) {
+      shape_operator = converter->ConvertOperand(shape_operand);
+    }
   }
   auto reshape_op = converter->AddOperator<ge::op::Reshape>(output_operand);
   SET_INPUT(reshape_op, x, input_operator);

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/engine.cc

@@ -70,6 +70,78 @@ Context::Context(void* device, const char* properties) : device_(device) {
         GetStringFromEnv(HUAWEI_ASCEND_NPU_PROFILING_FILE_PATH);
   }
   NNADAPTER_LOG(INFO) << "profiling path: " << profiling_file_path_;
+  // HUAWEI_ASCEND_NPU_DUMP_MODEL_FILE_PATH
+  if (key_values.count(HUAWEI_ASCEND_NPU_DUMP_MODEL_FILE_PATH)) {
+    ascend_config_params_.dump_model_path =
+        key_values[HUAWEI_ASCEND_NPU_DUMP_MODEL_FILE_PATH];
+  } else {
+    ascend_config_params_.dump_model_path =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_DUMP_MODEL_FILE_PATH);
+  }
+  NNADAPTER_LOG(INFO) << "dump model path: "
+                      << ascend_config_params_.dump_model_path;
+  // HUAWEI_ASCEND_NPU_PRECISION_MODE
+  if (key_values.count(HUAWEI_ASCEND_NPU_PRECISION_MODE)) {
+    ascend_config_params_.precision_mode =
+        key_values[HUAWEI_ASCEND_NPU_PRECISION_MODE];
+  } else {
+    ascend_config_params_.precision_mode =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_PRECISION_MODE);
+  }
+  NNADAPTER_LOG(INFO) << "precision mode: "
+                      << ascend_config_params_.precision_mode;
+  if (ascend_config_params_.precision_mode == "allow_mix_precision") {
+    // HUAWEI_ASCEND_NPU_MODIFY_MIXLIST_FILE_PATH
+    if (key_values.count(HUAWEI_ASCEND_NPU_MODIFY_MIXLIST_FILE_PATH)) {
+      ascend_config_params_.modify_mixlist_path =
+          key_values[HUAWEI_ASCEND_NPU_MODIFY_MIXLIST_FILE_PATH];
+    } else {
+      ascend_config_params_.modify_mixlist_path =
+          GetStringFromEnv(HUAWEI_ASCEND_NPU_MODIFY_MIXLIST_FILE_PATH);
+    }
+    NNADAPTER_LOG(INFO) << "modify mixlist path: "
+                        << ascend_config_params_.modify_mixlist_path;
+  }
+  // HUAWEI_ASCEND_NPU_OP_SELECT_IMPL_MODE
+  if (key_values.count(HUAWEI_ASCEND_NPU_OP_SELECT_IMPL_MODE)) {
+    ascend_config_params_.op_select_impl_mode =
+        key_values[HUAWEI_ASCEND_NPU_OP_SELECT_IMPL_MODE];
+  } else {
+    ascend_config_params_.op_select_impl_mode =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_OP_SELECT_IMPL_MODE);
+  }
+  NNADAPTER_LOG(INFO) << "op select impl mode: "
+                      << ascend_config_params_.op_select_impl_mode;
+  // HUAWEI_ASCEND_NPU_OPTYPELIST_FOR_IMPLMODE
+  if (key_values.count(HUAWEI_ASCEND_NPU_OPTYPELIST_FOR_IMPLMODE)) {
+    ascend_config_params_.op_type_list_for_impl_mode =
+        key_values[HUAWEI_ASCEND_NPU_OPTYPELIST_FOR_IMPLMODE];
+  } else {
+    ascend_config_params_.op_type_list_for_impl_mode =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_OPTYPELIST_FOR_IMPLMODE);
+  }
+  NNADAPTER_LOG(INFO) << "op type list for impl mode: "
+                      << ascend_config_params_.op_type_list_for_impl_mode;
+  // HUAWEI_ASCEND_NPU_ENABLE_COMPRESS_WEIGHT
+  if (key_values.count(HUAWEI_ASCEND_NPU_ENABLE_COMPRESS_WEIGHT)) {
+    ascend_config_params_.enable_compress_weight =
+        key_values[HUAWEI_ASCEND_NPU_ENABLE_COMPRESS_WEIGHT];
+  } else {
+    ascend_config_params_.enable_compress_weight =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_ENABLE_COMPRESS_WEIGHT);
+  }
+  NNADAPTER_LOG(INFO) << "enable compressw weight: "
+                      << ascend_config_params_.enable_compress_weight;
+  // HUAWEI_ASCEND_NPU_AUTO_TUNE_MODE
+  if (key_values.count(HUAWEI_ASCEND_NPU_AUTO_TUNE_MODE)) {
+    ascend_config_params_.auto_tune_mode =
+        key_values[HUAWEI_ASCEND_NPU_AUTO_TUNE_MODE];
+  } else {
+    ascend_config_params_.auto_tune_mode =
+        GetStringFromEnv(HUAWEI_ASCEND_NPU_AUTO_TUNE_MODE);
+  }
+  NNADAPTER_LOG(INFO) << "auto tune mode: "
+                      << ascend_config_params_.auto_tune_mode;
 }
 
 Context::~Context() {}
@@ -163,7 +235,8 @@ int Program::Build(core::Model* model, core::Cache* cache) {
                               model_buffer,
                               dynamic_shape_info,
                               optional_shape_str,
-                              dynamic_shape_mode_)) {
+                              dynamic_shape_mode_,
+                              context_->ascend_config_params())) {
       NNADAPTER_LOG(FATAL)
           << "Failed to build a CANN OM model and serialize it into a buffer!";
       return NNADAPTER_DEVICE_INTERNAL_ERROR;

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/engine.h

@@ -40,13 +40,15 @@ class Context {
     return selected_device_ids_.empty() ? 0 : selected_device_ids_[0];
   }
   std::string profiling_file_path() { return profiling_file_path_; }
+  AscendConfigParams* ascend_config_params() { return &ascend_config_params_; }
   ~Context();
 
  private:
   void* device_{nullptr};
   void* context_{nullptr};
   std::vector<int> selected_device_ids_;
   std::string profiling_file_path_ = "";
+  AscendConfigParams ascend_config_params_;
 };
 
 class Program {

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/model_client.cc

@@ -89,19 +89,7 @@ bool AclModelClient::LoadModel(const void* data, size_t size) {
     NNADAPTER_LOG(WARNING) << "ACL model had been already loaded.";
     return true;
   }
-  ACL_CALL(aclmdlQuerySizeFromMem(
-      data, size, &model_memory_size_, &model_weight_size_));
-  ACL_CALL(aclrtMalloc(
-      &model_memory_ptr_, model_memory_size_, ACL_MEM_MALLOC_HUGE_FIRST));
-  ACL_CALL(aclrtMalloc(
-      &model_weight_ptr_, model_weight_size_, ACL_MEM_MALLOC_HUGE_FIRST));
-  ACL_CALL(aclmdlLoadFromMemWithMem(data,
-                                    size,
-                                    &model_id_,
-                                    model_memory_ptr_,
-                                    model_memory_size_,
-                                    model_weight_ptr_,
-                                    model_weight_size_));
+  ACL_CALL(aclmdlLoadFromMem(data, size, &model_id_));
   auto model_desc = aclmdlCreateDesc();
   if (!model_desc) {
     NNADAPTER_LOG(ERROR) << "Failed to create ACL model description!";
@@ -126,16 +114,6 @@ void AclModelClient::UnloadModel() {
     DestroyDataset(&output_dataset_);
   }
   ACL_CALL(aclmdlUnload(model_id_));
-  if (model_memory_ptr_) {
-    ACL_CALL(aclrtFree(model_memory_ptr_));
-    model_memory_ptr_ = nullptr;
-    model_memory_size_ = 0;
-  }
-  if (model_weight_ptr_) {
-    ACL_CALL(aclrtFree(model_weight_ptr_));
-    model_weight_ptr_ = nullptr;
-    model_weight_size_ = 0;
-  }
   ACL_CALL(aclmdlDestroyDesc(model_desc_));
   model_desc_ = nullptr;
   NNADAPTER_VLOG(5) << "Unload a ACL model success(model_id=" << model_id_
@@ -300,6 +278,8 @@ bool AclModelClient::Process(uint32_t input_count,
   }
   NNADAPTER_CHECK_EQ(output_count, aclmdlGetDatasetNumBuffers(output_dataset_));
   // Copy the input data from host to device
+  bool is_dynamic_dims = false;
+  std::vector<int64_t> dynamic_dims;
   for (uint32_t i = 0; i < input_count; i++) {
     auto arg = FindArgumentByIndex(input_arguments, i, input_count);
     NNADAPTER_CHECK(arg) << "Input argument " << i << " does not exist!";
@@ -327,6 +307,7 @@ bool AclModelClient::Process(uint32_t input_count,
       }
       NNADAPTER_VLOG(3) << "The " << j << "th dimension of the " << i
                         << "th input is " << dimension;
+      dynamic_dims.push_back(dimension);
     }
     // Set true dynamic shapes
     if (is_dynamic_shape) {
@@ -343,7 +324,7 @@ bool AclModelClient::Process(uint32_t input_count,
                                  type.dimensions.data[3]);
           break;
         case DYNAMIC_SHAPE_MODE_N_DIMS:
-          aclmdlSetInputDynamicDims(model_id_, input_dataset_, i, &dimensions);
+          is_dynamic_dims = true;
           break;
         default:
           NNADAPTER_LOG(FATAL) << "Unsupported dynamic shape mode: "
@@ -362,6 +343,17 @@ bool AclModelClient::Process(uint32_t input_count,
     ACL_CALL(aclrtMemcpy(
         device_ptr, length, host_ptr, length, ACL_MEMCPY_HOST_TO_DEVICE));
   }
+  // Set dynamic dims
+  if (is_dynamic_dims) {
+    size_t index;
+    aclmdlGetInputIndexByName(model_desc_, ACL_DYNAMIC_TENSOR_NAME, &index);
+    aclmdlIODims dimensions;
+    dimensions.dimCount = dynamic_dims.size();
+    for (size_t i = 0; i < dimensions.dimCount; i++) {
+      dimensions.dims[i] = dynamic_dims[i];
+    }
+    aclmdlSetInputDynamicDims(model_id_, input_dataset_, index, &dimensions);
+  }
   // Model execution
   auto start_time = GetCurrentUS();
   ProfilingStart();

lite/backends/nnadapter/nnadapter/src/driver/huawei_ascend_npu/model_client.h

@@ -65,10 +65,6 @@ class AclModelClient {
   int device_id_{0};
   aclrtContext context_{nullptr};
   uint32_t model_id_{0};
-  size_t model_memory_size_;
-  size_t model_weight_size_;
-  void* model_memory_ptr_;
-  void* model_weight_ptr_;
   aclmdlDesc* model_desc_{nullptr};
   aclmdlDataset* input_dataset_{nullptr};
   aclmdlDataset* output_dataset_{nullptr};