tensorflow - Errors with reading pb file in TensorRT and readNetFromTensorflow in C++

Question

I have Python code along with TensorRT with Docker container 20.03, which has CUDA 10.2 and TensorRT 7.0.0

from __future__ import print_function

import warnings

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from imutils.paths import list_images
from keras import backend as K
from keras.callbacks import CSVLogger
from keras.layers import *
from keras.models import Model
from keras.optimizers import Adam
from scipy.io import loadmat
from scipy.misc import imread
from skimage.io import imsave
from skimage.transform import resize

from scipy.io import loadmat
import cv2
import tensorflow as tf

Ny=1024
target_size = (Ny, Ny)

warnings.filterwarnings('ignore')

LR= 1e-4
E, BS = 2,4 


def get_unet(img_rows, img_cols):
    inputs = Input((img_rows, img_cols, 1))
    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
    conv1 = BatchNormalization()(conv1)
    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
    conv1 = BatchNormalization()(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
    conv2 = BatchNormalization()(conv2)
    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
    conv2 = BatchNormalization()(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
    conv3 = BatchNormalization()(conv3)
    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)
    conv3 = BatchNormalization()(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)
    conv4 = BatchNormalization()(conv4)
    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)
    conv4 = BatchNormalization()(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)

    conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)
    conv5 = BatchNormalization()(conv5)
    conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(conv5)
    conv5 = BatchNormalization()(conv5)
    
    conv5 = Dropout(0.5)(conv5)

    up6 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv4], axis=3)
    conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(up6)
    conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv6)

    up7 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv3], axis=3)
    conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(up7)
    conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv7)

    up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2], axis=3)
    conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(up8)
    conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv8)

    up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1], axis=3)
    conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(up9)
    conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv9)


    conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)
    '''
    def get_unet(img_rows, img_cols):
        inputs = Input((img_rows, img_cols, 1))
        conv9=inputs

        conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)
    '''    
    
    model = Model(inputs=[inputs], outputs=[conv10])
    
    model.compile(optimizer=Adam(lr=LR), 
                      loss='binary_crossentropy',
                      metrics=['binary_crossentropy'])


    return model


train_images=np.zeros([1, 1024,1024,1], dtype=float)
annot_train=np.zeros([1, 1024,1024,1], dtype=float)
test_images=np.zeros([1, 1024,1024,1], dtype=float)
annot_test=np.zeros([1, 1024,1024,1], dtype=float)

img = cv2.imread('owlResized.bmp', 0)/255.0 
label = cv2.imread('owlResized.bmp', 0)/255.0
train_images[0,:,:,0], annot_train[0,:,:,0] =img, label
test_images[0,:,:,0], annot_test[0,:,:,0] =img, label

print("finished reading")
C = np.concatenate([annot_test, annot_train])
I = np.concatenate([test_images, train_images])

unet = get_unet(Ny, Ny)
filepath="W-E_{epoch:02d}-L_{loss:.6f}"+outName+".h5"
history = unet.fit(I, C, verbose=2, epochs=E, batch_size=BS, validation_split=0.1)

#save pb
unet.save('owlSimple', overwrite=True, include_optimizer=False, save_format='tf')

Then did

python3 -m tf2onnx.convert --opset 12 --saved-model ./owlSimple --output owlSimple12.onnx

I also tried opset of 9,10,11. They all seemed to convert successfully

When I tried to run opsets 11 and 12 in TensorRT, I would get as error:

ERROR: ModelImporter.cpp:92 In function parseGraph:
[8] Assertion failed: convertOnnxWeights(initializer, &weights, ctx)
ERROR: could not parse the model.

setBindingDimensions
Segmentation fault (core dumped)

With opset 10, I got:

While parsing node number 45 [Resize]:
ERROR: builtin_op_importers.cpp:2412 In function importResize:
[8] Assertion failed: scales.is_weights() && "Resize scales must be an initializer!"
ERROR: could not parse the model.

and with Opset9 I got

While parsing node number 45 [Upsample]:
ERROR: builtin_op_importers.cpp:3240 In function importUpsample:
[8] Assertion failed: scales_input.is_weights()
ERROR: could not parse the model.

I got those errors even when I did

wget https://raw.githubusercontent.com/rmccorm4/tensorrt-utils/master/OSS/build_OSS.sh
source build_OSS.sh

If I further simplified the Unet by replacing the Unet with

inputs = Input((img_rows, img_cols, 1))
conv9=inputs
conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)

as seen in the commented out code above, then I get these errors when running the various opsets in TensorRT:

 input_2:0: dynamic input is missing dimensions in profile 0.
 Network validation failed.
 Segmentation fault (core dumped)

The TensorRT C++ I used is (It is based on code from https://www.learnopencv.com/how-to-run-inference-using-tensorrt-c-api/):

#include <iostream>
#include <fstream>
#include <NvInfer.h>
#include <NvInferRuntime.h>
#include <memory>
#include <NvOnnxParser.h>
#include <vector>
#include <cuda_runtime_api.h>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/core/cuda.hpp>
#include <opencv2/cudawarping.hpp>
#include <opencv2/core.hpp>
#include <opencv2/cudaarithm.hpp>
#include <algorithm>
#include <numeric>
#include <opencv2/dnn/dnn.hpp>

#include <opencv2/opencv.hpp>
#include <opencv2/cudaimgproc.hpp>

using namespace cv;


// utilities ----------------------------------------------------------------------------------------------------------
// class to log errors, warnings, and other information during the build and inference phases
class Logger : public nvinfer1::ILogger
{
public:
    void log(Severity severity, const char* msg) override {
        // remove this 'if' if you need more logged info
        if ((severity == Severity::kERROR) || (severity == Severity::kINTERNAL_ERROR)) {
            std::cout << msg << "
";
        }
    }
} gLogger;

// destroy TensorRT objects if something goes wrong
struct TRTDestroy
{
    template <class T>
    void operator()(T* obj) const
    {
        if (obj)
        {
            obj->destroy();
        }
    }
};

template <class T>
using TRTUniquePtr = std::unique_ptr<T, TRTDestroy>;

// calculate size of tensor
size_t getSizeByDim(const nvinfer1::Dims& dims)
{
    size_t size = 1;
    for (size_t i = 0; i < dims.nbDims; ++i)
    {
        size *= dims.d[i];
    }
    return size;
}

// get classes names
std::vector<std::string> getClassNames(const std::string& imagenet_classes)
{
    std::ifstream classes_file(imagenet_classes);
    std::vector<std::string> classes;
    if (!classes_file.good())
    {
        std::cerr << "ERROR: can't read file with classes names.
";
        return classes;
    }
    std::string class_name;
    while (std::getline(classes_file, class_name))
    {
        classes.push_back(class_name);
    }
    return classes;
}


// preprocessing stage ------------------------------------------------------------------------------------------------
void preprocessImage(const std::string& image_path, float* gpu_input, const nvinfer1::Dims& dims)
{
    // read input image
    cv::Mat frame = cv::imread(image_path);
    if (frame.empty())
    {
        std::cerr << "Input image " << image_path << " load failed
";
        return;
    }
    cv::cuda::GpuMat gpu_frame;
    gpu_frame.upload(frame);

    int channels=1;
    auto input_width = dims.d[1];
    auto input_height = dims.d[0];
    
    auto input_size = cv::Size(input_width, input_height);
    cv::cuda::GpuMat resized=gpu_frame;

    //normalize
    cv::cuda::GpuMat flt_image;
    resized.convertTo(flt_image, CV_32FC1, 1.f / 255.f);

    std::vector<cv::cuda::GpuMat> chw;
    for (size_t i = 0; i < channels; ++i)
    {
        chw.emplace_back(cv::cuda::GpuMat(input_size, CV_32FC1, gpu_input + i * input_width * input_height));
    }
    cv::cuda::split(flt_image, chw);
}


// post-processing stage ----------------------------------------------------------------------------------------------
void postprocessResults(float *gpu_output, const nvinfer1::Dims &dims, int batch_size)
{
    std::vector<float> cpu_output(getSizeByDim(dims) * batch_size);
    cudaMemcpy(cpu_output.data(), gpu_output, cpu_output.size() * sizeof(float), cudaMemcpyDeviceToHost);

    auto rows = dims.d[1];
    auto cols = dims.d[2];
    cv::Mat Finalmat = cv::Mat(rows, cols, CV_32FC1); // initialize matrix of uchar of 1-channel where you will store vec data
    

    //copy vector to mat
    memcpy(Finalmat.data, cpu_output.data(), cpu_output.size()*sizeof(float));
    Finalmat.convertTo(Finalmat, CV_8UC3, 255.0);
    cv::imwrite("trt_output.bmp", Finalmat);
}


// initialize TensorRT Context and Engine and parse ONNX model --------------------------------------------------------------------
void parseOnnxModel(const std::string& model_path, TRTUniquePtr<nvinfer1::ICudaEngine>& engine,
                    TRTUniquePtr<nvinfer1::IExecutionContext>& context)
{
    nvinfer1::IBuilder *builder = nvinfer1::createInferBuilder(gLogger);
    const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
    nvinfer1::INetworkDefinition *network = builder->createNetworkV2(explicitBatch);
    
    TRTUniquePtr<nvonnxparser::IParser> parser{nvonnxparser::createParser(*network, gLogger)};
    
    // parse ONNX
    if (!parser->parseFromFile(model_path.c_str(), static_cast<int>(nvinfer1::ILogger::Severity::kINFO)))
    {
        std::cerr << "ERROR: could not parse the model.
";
        return;
    }

    //create Config to configure engine parameters such as max memory or set FP16 mode
    TRTUniquePtr<nvinfer1::IBuilderConfig> config{builder->createBuilderConfig()};
    
    nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
    profile->setDimensions("in

Answer 1

等待大神答复