Es/lpt/lpt to ngraph fixes2 with master

inference-engine/src/cldnn_engine/cldnn_config.cpp

@@ -181,6 +181,13 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap)
             } else {
                 THROW_IE_EXCEPTION << NOT_FOUND_str << "Unsupported property value by plugin: " << val;
             }
+        } else if (key.compare(PluginConfigInternalParams::KEY_LP_TRANSFORMS_VERSION) == 0) {
+            if (val == PluginConfigInternalParams::LP_TRANSFORMS_CNNNETWORK)
+                lptVersion = LptVersion::cnnNetwork;
+            else if (val == PluginConfigInternalParams::LP_TRANSFORMS_NGRAPH)
+                lptVersion = LptVersion::nGraph;
+            else
+                THROW_IE_EXCEPTION << "Wrong value for property key " << PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE;
         } else if (key.compare(CLDNNConfigParams::KEY_CLDNN_NV12_TWO_INPUTS) == 0) {
             if (val.compare(PluginConfigParams::YES) == 0) {
                 nv12_two_inputs = true;

inference-engine/src/cldnn_engine/cldnn_config.h

@@ -19,6 +19,11 @@
 namespace CLDNNPlugin {
 
 struct Config {
+    enum LptVersion {
+        cnnNetwork,
+        nGraph
+    };
+
     Config() : throughput_streams(1),
                useProfiling(false),
                dumpCustomKernels(false),
@@ -49,6 +54,7 @@ struct Config {
     bool memory_pool_on;
     bool enableDynamicBatch;
     bool enableInt8;
+    LptVersion lptVersion = LptVersion::nGraph;
     bool nv12_two_inputs;
     bool enable_fp16_for_quantized_models;
     cldnn::priority_mode_types queuePriority;

inference-engine/src/cldnn_engine/cldnn_engine.cpp

@@ -33,7 +33,9 @@
 #include <transformations/opset_conversions/convert_opset2_to_opset1.hpp>
 #include <transformations/opset_conversions/convert_opset3_to_opset2.hpp>
 #include <transformations/init_node_info.hpp>
+#include <transformations/convert_precision.hpp>
 #include <transformations/rt_info/fused_names_attribute.hpp>
+
 #include <legacy/convert_function_to_cnn_network.hpp>
 #include <legacy/ie_util_internal.hpp>
 #include <legacy/graph_transformer.h>
@@ -42,6 +44,9 @@
 #include "cldnn_executable_network.h"
 #include "cldnn_custom_layer.h"
 
+#include <transformations/low_precision/transformer.hpp>
+#include <transformations/low_precision/mat_mul.hpp>
+
 #ifdef __linux__
 #include <dlfcn.h>
 #endif
@@ -72,8 +77,10 @@ cldnn::device_info clDNNEngine::GetDeviceInfo(const std::map<std::string, std::s
     return device_info;
 }
 
-InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneAndTransformNetwork(const InferenceEngine::ICNNNetwork& network) const {
+InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneAndTransformNetwork(const InferenceEngine::ICNNNetwork& network, CLDNNPlugin::Config config) const {
     std::shared_ptr<ICNNNetwork> clonedNetwork = cloneNetwork(network);
+    bool baselineIsFP16 = false;
+
     if (clonedNetwork->getFunction()) {
         const auto transformations_callback = [](const std::shared_ptr<const ::ngraph::Node> &node) -> bool {
             // Reshape->Permute->Reshape pattern in theory can change output rank, so this check is added to be sure
@@ -112,6 +119,12 @@ InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneAndTransformNetwork(const In
                 return can_use_reduce;
             }
 
+            if (auto add_op = std::dynamic_pointer_cast<const ngraph::opset1::Add>(node)) {
+                return ngraph::is_type<ngraph::opset1::Convolution>(add_op->get_input_node_shared_ptr(0)) ||
+                       ngraph::is_type<ngraph::opset1::GroupConvolution>(add_op->get_input_node_shared_ptr(0)) ||
+                       ngraph::is_type<ngraph::opset1::MatMul>(add_op->get_input_node_shared_ptr(0));
+            }
+
             return std::dynamic_pointer_cast<const ::ngraph::opset2::Gelu>(node) ||
                    std::dynamic_pointer_cast<const ::ngraph::opset3::ShuffleChannels>(node) ||
                    std::dynamic_pointer_cast<const ::ngraph::opset2::BatchToSpace>(node) ||
@@ -126,24 +139,75 @@ InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneAndTransformNetwork(const In
         // Disable shape inference (WA for generic operations)
         ::ngraph::op::GenericIE::DisableReshape noReshape(nGraphFunc);
 
-        // Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
-        ngraph::pass::Manager manager;
-        manager.register_pass<ngraph::pass::InitNodeInfo>();
-        // WA: ConvertPriorBox must be executed before the 1st ConstantFolding pass
-        manager.register_pass<ngraph::pass::ConvertPriorBox>();
-        manager.register_pass<ngraph::pass::CommonOptimizations>();
-        manager.register_pass<ngraph::pass::ConvertOpSet3ToOpSet2>();
-        manager.register_pass<ngraph::pass::ConvertOpSet2ToOpSet1>();
-        manager.register_pass<ngraph::pass::ConvertOpSet1ToLegacy>();
-
-        manager.set_callback(transformations_callback);
-        manager.run_passes(nGraphFunc);
-
-        ngraph::pass::Manager ti_manager;
-        // Unroll will be called after all conversions
-        // temporarily switch back to plugin unroller from NGraph unroller until TI output names are corrected
-        // ti_manager.register_pass<ngraph::pass::UnrollTensorIterator>();
-        ti_manager.run_passes(nGraphFunc);
+        bool enableInt8;
+
+        {
+            // Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
+            ngraph::pass::Manager manager;
+            manager.register_pass<ngraph::pass::InitNodeInfo>();
+            // WA: ConvertPriorBox must be executed before the 1st ConstantFolding pass
+            manager.register_pass<ngraph::pass::ConvertPriorBox>();
+            manager.register_pass<ngraph::pass::CommonOptimizations>();
+            manager.register_pass<ngraph::pass::ConvertOpSet3ToOpSet2>();
+            manager.register_pass<ngraph::pass::ConvertOpSet2ToOpSet1>();
+
+
+            manager.set_callback(transformations_callback);
+            manager.run_passes(nGraphFunc);
+
+            const auto fp16_callback = [&baselineIsFP16](const std::shared_ptr<const ::ngraph::Node> &node) -> bool {
+                if (!baselineIsFP16 && node->get_output_element_type(0) == ngraph::element::f16) {
+                    baselineIsFP16 = true;
+                }
+
+                return true;
+            };
+
+            ngraph::pass::Manager conversion_manager;
+
+            enableInt8 = config.enableInt8 &&
+                (config.lptVersion == Config::LptVersion::nGraph) &&
+                ngraph::pass::low_precision::LowPrecisionTransformer::isFunctionQuantized(nGraphFunc);
+            if (enableInt8) {
+                // [WA part1] Convert quantized FP16 model to FP32 to avoid possible overflow and mixed precision errors
+                conversion_manager.register_pass<ngraph::pass::ConvertPrecision>(ngraph::element::f16, ngraph::element::f32);
+            }
+
+            conversion_manager.set_callback(fp16_callback);
+            conversion_manager.run_passes(nGraphFunc);
+
+            ngraph::pass::Manager ti_manager;
+            // Unroll will be called after all conversions
+            // temporarily switch back to plugin unroller from NGraph unroller until TI output names are corrected
+            // ti_manager.register_pass<ngraph::pass::UnrollTensorIterator>();
+            ti_manager.run_passes(nGraphFunc);
+        }
+
+        using namespace ngraph::pass::low_precision;
+        if (enableInt8) {
+            auto params = LayerTransformation::Params(
+                true,  // updatePrecisions
+                LayerTransformation::QuantizedTensorAlignment::UpdateLevel,  // quantizedTensorAlignmentOnActivations
+                LayerTransformation::QuantizedTensorAlignment::None,  // quantizedTensorAlignmentOnWeights
+                true);  // supportAsymmetricQuantization
+            LowPrecisionTransformer transformer(LowPrecisionTransformer::getAllTransformations(params)
+                .add<MatMulTransformation, ngraph::opset1::MatMul>(LayerTransformation::Params(params).setSupportAsymmetricQuantization(false)));
+
+            transformer.transform(nGraphFunc);
+        }
+
+        {
+            ngraph::pass::Manager manager = ngraph::pass::Manager();
+            manager.register_pass<ngraph::pass::ConvertOpSet1ToLegacy>();
+            manager.set_callback(transformations_callback);
+            manager.run_passes(nGraphFunc);
+
+            ngraph::pass::Manager ti_manager;
+            // Unroll will be called after all conversions
+            // temporarily switch back to plugin unroller from NGraph unroller until TI output names are corrected
+            // ti_manager.register_pass<ngraph::pass::UnrollTensorIterator>();
+            ti_manager.run_passes(nGraphFunc);
+        }
 
         clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, *clonedNetwork);
     }
@@ -155,6 +219,14 @@ InferenceEngine::ICNNNetwork::Ptr clDNNEngine::CloneAndTransformNetwork(const In
         transformator.fullTrim();
     }
 
+    if (baselineIsFP16) {
+        InputsDataMap inputsMap;
+        clonedNetwork->getInputsInfo(inputsMap);
+
+        auto input0 = getInputTo(inputsMap.begin()->second->getInputData());
+        input0.begin()->second->params["FP16"];
+    }
+
     return clonedNetwork;
 }
 
@@ -257,7 +329,7 @@ ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEn
 
     context = m_defaultContext;
 
-    return std::make_shared<CLDNNExecNetwork>(*CloneAndTransformNetwork(network), context, conf);
+    return std::make_shared<CLDNNExecNetwork>(*CloneAndTransformNetwork(network, conf), context, conf);
 }
 
 ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network,
@@ -281,7 +353,7 @@ ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEn
         conf.max_dynamic_batch = static_cast<int>(network.getBatchSize());
     }
 
-    return std::make_shared<CLDNNExecNetwork>(*CloneAndTransformNetwork(network), casted, conf);
+    return std::make_shared<CLDNNExecNetwork>(*CloneAndTransformNetwork(network, conf), casted, conf);
 }
 
 RemoteContext::Ptr clDNNEngine::CreateContext(const ParamMap& params) {
@@ -324,7 +396,7 @@ QueryNetworkResult clDNNEngine::QueryNetwork(const ICNNNetwork& network,
         for (auto&& node : function->get_ops()) {
             originalOps.emplace(node->get_friendly_name());
         }
-        auto clonedNetwork = CloneAndTransformNetwork(network);
+        auto clonedNetwork = CloneAndTransformNetwork(network, _impl->m_config);
         std::unordered_set<std::string> supported;
         std::unordered_set<std::string> unsupported;
 

inference-engine/src/cldnn_engine/cldnn_engine.h

@@ -27,7 +27,8 @@ class clDNNEngine : public InferenceEngine::InferencePluginInternal,
     CLDNNRemoteCLContext::Ptr m_defaultContext;
 
     cldnn::device_info GetDeviceInfo(const std::map<std::string, std::string> &config) const;
-    InferenceEngine::ICNNNetwork::Ptr CloneAndTransformNetwork(const InferenceEngine::ICNNNetwork& network) const;
+    InferenceEngine::ICNNNetwork::Ptr CloneAndTransformNetwork(const InferenceEngine::ICNNNetwork& network,
+                                                               CLDNNPlugin::Config config) const;
 public:
     clDNNEngine();
 

inference-engine/src/cldnn_engine/cldnn_program.cpp

@@ -397,7 +397,39 @@ Program::Program(InferenceEngine::ICNNNetwork& network, std::shared_ptr<const cl
     , p_currentOutputs({}) {
     InitFormat(network);
 
+    InputsDataMap inputsMap;
+    network.getInputsInfo(inputsMap);
+
+    auto input0 = getInputTo(inputsMap.begin()->second->getInputData());
+
+    bool baselineIsFP16 = false;
+    if (input0.begin()->second->params.count("FP16") != 0) {
+        baselineIsFP16 = true;
+    }
+
+    bool fqFound = false;
+    bool allFQareSupported = true;
     if (config.enableInt8) {
+        auto it = details::CNNNetworkIterator(&network);
+        auto end = details::CNNNetworkIterator();
+        while (it != end) {
+            auto& layer = *it;
+            if (layer->precision == Precision::FP16) {
+                baselineIsFP16 = true;
+            }
+
+            if (CaselessEq<std::string>()(layer->type, "FakeQuantize")) {
+                fqFound = true;
+                auto levels = layer->GetParamAsUInt("levels");
+                if (levels != 255 && levels != 256) {
+                    allFQareSupported = false;
+                }
+            }
+            it++;
+        }
+    }
+
+    if (config.enableInt8 && (config.lptVersion == Config::LptVersion::cnnNetwork)) {
         auto params = LayerTransformation::Params(true,  // updatePrecisions
                                                   true,  // quantizeOutputs
                                                   true,  // weightsToConst
@@ -413,38 +445,17 @@ Program::Program(InferenceEngine::ICNNNetwork& network, std::shared_ptr<const cl
                 .add<FullyConnectedTransformation>(LayerTransformation::Params(params).setSupportAsymmetricQuantization(false), "FullyConnected")
                 .add<GemmTransformation>(LayerTransformation::Params(params).setSupportAsymmetricQuantization(false), "GEMM");
 
-        bool fqFound = false;
-        bool allFQareSupported = true;
-        bool baselineIsFP16 = false;
-        {
-            auto it = details::CNNNetworkIterator(&network);
-            auto end = details::CNNNetworkIterator();
-            while (it != end) {
-                auto& layer = *it;
-                if (layer->precision == Precision::FP16) {
-                    baselineIsFP16 = true;
-                }
-
-                if (CaselessEq<std::string>()(layer->type, "FakeQuantize")) {
-                    fqFound = true;
-                    auto levels = layer->GetParamAsUInt("levels");
-                    if (levels != 255 && levels != 256) {
-                        allFQareSupported = false;
-                    }
-                }
-                it++;
-            }
-        }
-
         // [WA part1] Convert quantized FP16 model to FP32 to avoid possible overflow and mixed precision errors
         if (fqFound && allFQareSupported) {
             NetPass::ConvertPrecision(network, Precision::FP16, Precision::FP32);
         }
 
         LowPrecisionTransformer transformer(transforms);
         transformer.transform(network);
+    }
 
-        // [WA part2] Try to find non-quantized layers and convert them back to FP16
+    // [WA part2] Try to find non-quantized layers and convert them back to FP16
+    if (config.enableInt8) {
         if (fqFound && baselineIsFP16 && config.enable_fp16_for_quantized_models) {
             auto layersSorted = BFSSort(network);
 

inference-engine/src/legacy_api/include/legacy/ngraph_ops/eltwise.hpp

@@ -22,13 +22,17 @@ class INFERENCE_ENGINE_API_CLASS(Eltwise) : public Op {
 
     Eltwise(const Output<Node>& data1,
             const Output<Node>& data2,
-            const ELTWISE_TYPE eltwise_type);
+            const ELTWISE_TYPE eltwise_type,
+            const element::Type output_type = element::undefined);
 
     void validate_and_infer_types() override;
 
     std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
 
     ELTWISE_TYPE eltwise_type;
+
+private:
+    element::Type m_output_type;
 };
 
 }  // namespace op

inference-engine/src/legacy_api/include/legacy/ngraph_ops/fully_connected.hpp

@@ -29,17 +29,21 @@ class INFERENCE_ENGINE_API_CLASS(FullyConnected) : public Op {
     FullyConnected(const Output<Node> & A,
                    const Output<Node> & B,
                    const Output<Node> & C,
-                   const Shape & output_shape);
+                   const Shape & output_shape,
+                   const element::Type output_type = element::undefined);
 
     void validate_and_infer_types() override;
 
     std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
 
-    size_t get_out_size() { return m_output_size; }
+    size_t get_out_size() const { return m_output_size; }
+
+    element::Type get_output_type() const { return m_output_type; }
 
 private:
     size_t m_output_size = 0;
     Shape m_output_shape = {};
+    element::Type m_output_type;
 };
 
 }  // namespace op

inference-engine/src/legacy_api/include/legacy/ngraph_ops/normalize_ie.hpp

@@ -25,7 +25,8 @@ class INFERENCE_ENGINE_API_CLASS(NormalizeIE) : public Op {
                 const Output<Node>& weights,
                 float eps,
                 bool across_spatial,
-                bool channel_shared);
+                bool channel_shared,
+                const ngraph::element::Type output_type);
 
     float get_eps() const { return m_eps; }
     bool get_channel_shared() const  { return m_channel_shared;}
@@ -39,6 +40,7 @@ class INFERENCE_ENGINE_API_CLASS(NormalizeIE) : public Op {
     float m_eps;
     bool m_across_spatial;
     bool m_channel_shared;
+    ngraph::element::Type m_output_type;
 };
 
 }  // namespace op

inference-engine/src/legacy_api/include/legacy/ngraph_ops/power.hpp

@@ -19,13 +19,16 @@ class INFERENCE_ENGINE_API_CLASS(PowerIE) : public Op {
     const NodeTypeInfo& get_type_info() const override { return type_info; }
 
     PowerIE(const Output<Node>& data_batch,
-            const float power, const float scale, const float shift);
+            const float power, const float scale, const float shift, const element::Type output_type = element::undefined);
 
     void validate_and_infer_types() override;
 
     std::shared_ptr<Node> clone_with_new_inputs(const OutputVector& new_args) const override;
 
     float scale, power, shift;
+
+private:
+    element::Type m_output_type;
 };
 
 }  // namespace op