TDA4VM: EdgeAI Apps can't run sample OD networks by compiled using edgeai-benchmark

# Copyright (c) 2018-2021, Texas Instruments
# All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import argparse
import cv2
from jai_benchmark import *


def get_imagedet_dataset_loaders(settings, download=False):
    # this example uses the datasets.COCODetection data loader
    dataset_calib_cfg = dict(
        path=f'{settings.datasets_path}/coco',
        split='val2017',
        shuffle=True,
        num_frames=min(settings.calibration_frames,5000))

    # dataset parameters for actual inference
    dataset_val_cfg = dict(
        path=f'{settings.datasets_path}/coco',
        split='val2017',
        shuffle=False, # can be set to True as well, if needed
        num_frames=min(settings.num_frames,5000))

    # you are free to use any other data loaded provided in datasets folder or write your own instead of this
    calib_dataset = datasets.COCODetection(**dataset_calib_cfg, download=download)
    val_dataset = datasets.COCODetection(**dataset_val_cfg, download=download)
    return calib_dataset, val_dataset


def create_configs(settings, work_dir):
    '''
    configs for each model pipeline
    - calibration_dataset: dataset to be used for import - should support __len__ and __getitem__.
    - input_dataset: dataset to be used for inference - should support __len__ and __getitem__
      Output of __getitem__ should be understood by the preprocess stage.
      For example, if the dataset returns image filenames, the first entry in the preprocess can be an image read class.
    - preprocess is just a list of transforms wrapped in utils.TransformsCompose.
      It depends on what the dataset class outputs and what the model expects.
      We have some default transforms defined in settings.
    - postprocess is also a list of transforms wrapped in utils.TransformsCompose
      It depends on what the model outputs and what the metric evaluation expects.
    - metric - evaluation metric (eg. accuracy). If metric is not defined in the pipeline,
      evaluate() function of the dataset will be called.

    parameters for calibration_dataset and input_dataset
    - path: folder containing images
    - split: provide a .txt file containing two entries in each line
      first entry in each line is image file name (starting from path above),
      for classification, second entry is class id (just set to 0 if you don't know what it is)
        example:
          image10.jpg 0
          tomato/image2.jpg 9
      for segmentation, second entry is the label image.
      for detection, second entry is not used right now in this script.
    '''

    # get dataset loaders
    imagedet_calib_dataset, imagedet_val_dataset = get_imagedet_dataset_loaders(settings)

    # in these examples, the session types cfgs are hardcoded for simplicity
    # however, in the configs in the root of this repository, they depend on session_type_dict
    runtime_options_onnxrt = settings.get_runtime_options(constants.SESSION_NAME_ONNXRT, is_qat=False)

    preproc_transforms = preprocess.PreProcessTransforms(settings)
    postproc_transforms = postprocess.PostProcessTransforms(settings)

    common_session_cfg = sessions.get_common_session_cfg(settings, work_dir=work_dir)
    onnx_session_cfg = sessions.get_onnx_session_cfg(settings, work_dir=work_dir)
    onnx_bgr_session_cfg = sessions.get_onnx_bgr_session_cfg(settings, work_dir=work_dir)
    onnx_quant_session_cfg = sessions.get_onnx_quant_session_cfg(settings, work_dir=work_dir)
    onnx_bgr_quant_session_cfg = sessions.get_onnx_bgr_quant_session_cfg(settings, work_dir=work_dir)
    jai_session_cfg = sessions.get_jai_session_cfg(settings, work_dir=work_dir)
    jai_quant_session_cfg = sessions.get_jai_quant_session_cfg(settings, work_dir=work_dir)
    mxnet_session_cfg = sessions.get_mxnet_session_cfg(settings, work_dir=work_dir)
    tflite_session_cfg = sessions.get_tflite_session_cfg(settings, work_dir=work_dir)
    tflite_quant_session_cfg = sessions.get_tflite_quant_session_cfg(settings, work_dir=work_dir)

    pipeline_configs = {
        'imagedet-4': dict(
            task_type='detection',
            calibration_dataset=imagedet_calib_dataset,
            input_dataset=imagedet_val_dataset,
            preprocess=preproc_transforms.get_transform_onnx(),
            session=sessions.ONNXRTSession(**onnx_session_cfg,
                runtime_options=runtime_options_onnxrt,
                model_path='/home/sefau18/edgeai-benchmark/work_dirs/best.onnx'),
            postprocess=postproc_transforms.get_transform_segmentation_onnx(),
            #metric=dict(label_offset_pred=datasets.coco_det_label_offset_90to90()),
            model_info=dict(metric_reference={'accuracy_ap[.5:.95]%':23.0})
        ),
    }
    return pipeline_configs


if __name__ == '__main__':
    # the cwd must be the root of the respository
    if os.path.split(os.getcwd())[-1] == 'scripts':
        os.chdir('../')
    #

    parser = argparse.ArgumentParser()
    parser.add_argument('settings_file', type=str)
    cmds = parser.parse_args()
    settings = config_settings.ConfigSettings(cmds.settings_file, model_shortlist=None)

    work_dir = os.path.join(settings.modelartifacts_path, f'{settings.tensor_bits}bits')
    print(f'work_dir = {work_dir}')

    # now run the actual pipeline
    pipeline_configs = create_configs(settings, work_dir)

    # run the accuracy pipeline
    tools.run_accuracy(settings, work_dir, pipeline_configs)

sefau18@ubuntu:~/edgeai-benchmark$ ./run_custom_pc.sh 
Entering: ./work_dirs/modelartifacts/8bits/od-5040_tvmdlr_coco_gluoncv-mxnet_ssd_512_mobilenet1.0_coco-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5040_tvmdlr_coco_gluoncv-mxnet_ssd_512_mobilenet1.0_coco-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/od-5020_tvmdlr_coco_gluoncv-mxnet_yolo3_mobilenet1.0_coco-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5020_tvmdlr_coco_gluoncv-mxnet_yolo3_mobilenet1.0_coco-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/ss-5720_tvmdlr_cocoseg21_edgeai-tv_fpn_aspp_regnetx800mf_edgeailite_512x512_20210405_onnx.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/ss-5720_tvmdlr_cocoseg21_edgeai-tv_fpn_aspp_regnetx800mf_edgeailite_512x512_20210405_onnx.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3480_tvmdlr_imagenet1k_gluoncv-mxnet_hrnet_w18_small_v2_c-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3480_tvmdlr_imagenet1k_gluoncv-mxnet_hrnet_w18_small_v2_c-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/ss-5720_tvmdlr_cocoseg21_edgeai-tv_fpn_aspp_regnetx800mf_edgeailite_512x512_20210405_onnx.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/ss-5720_tvmdlr_cocoseg21_edgeai-tv_fpn_aspp_regnetx800mf_edgeailite_512x512_20210405_onnx.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3430_tvmdlr_imagenet1k_gluoncv-mxnet_xception-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3430_tvmdlr_imagenet1k_gluoncv-mxnet_xception-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/ss-5818_tvmdlr_ti-robokit_edgeai-tv_deeplabv3plus_mobilenetv2_tv_edgeailite_robokit-zed1hd_768x432_qat-p2_onnx.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/ss-5818_tvmdlr_ti-robokit_edgeai-tv_deeplabv3plus_mobilenetv2_tv_edgeailite_robokit-zed1hd_768x432_qat-p2_onnx.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/od-5030_tvmdlr_coco_gluoncv-mxnet_ssd_512_resnet50_v1_coco-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5030_tvmdlr_coco_gluoncv-mxnet_ssd_512_resnet50_v1_coco-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3420_tvmdlr_imagenet1k_gluoncv-mxnet_resnet50_v1d-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3420_tvmdlr_imagenet1k_gluoncv-mxnet_resnet50_v1d-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3420_tvmdlr_imagenet1k_gluoncv-mxnet_resnet50_v1d-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3420_tvmdlr_imagenet1k_gluoncv-mxnet_resnet50_v1d-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3410_tvmdlr_imagenet1k_gluoncv-mxnet_mobilenetv2_1.0-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3410_tvmdlr_imagenet1k_gluoncv-mxnet_mobilenetv2_1.0-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/od-5040_tvmdlr_coco_gluoncv-mxnet_ssd_512_mobilenet1.0_coco-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5040_tvmdlr_coco_gluoncv-mxnet_ssd_512_mobilenet1.0_coco-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3410_tvmdlr_imagenet1k_gluoncv-mxnet_mobilenetv2_1.0-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3410_tvmdlr_imagenet1k_gluoncv-mxnet_mobilenetv2_1.0-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/od-5030_tvmdlr_coco_gluoncv-mxnet_ssd_512_resnet50_v1_coco-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5030_tvmdlr_coco_gluoncv-mxnet_ssd_512_resnet50_v1_coco-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/ss-5818_tvmdlr_ti-robokit_edgeai-tv_deeplabv3plus_mobilenetv2_tv_edgeailite_robokit-zed1hd_768x432_qat-p2_onnx.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/ss-5818_tvmdlr_ti-robokit_edgeai-tv_deeplabv3plus_mobilenetv2_tv_edgeailite_robokit-zed1hd_768x432_qat-p2_onnx.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3430_tvmdlr_imagenet1k_gluoncv-mxnet_xception-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3430_tvmdlr_imagenet1k_gluoncv-mxnet_xception-symbol_json.tar.gz.link.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/cl-3480_tvmdlr_imagenet1k_gluoncv-mxnet_hrnet_w18_small_v2_c-symbol_json.tar.gz.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/cl-3480_tvmdlr_imagenet1k_gluoncv-mxnet_hrnet_w18_small_v2_c-symbol_json.tar.gz.link/artifacts: Not a directory
Entering: ./work_dirs/modelartifacts/8bits/od-5020_tvmdlr_coco_gluoncv-mxnet_yolo3_mobilenet1.0_coco-symbol_json.tar.gz.link.link
run_set_target_device.sh: line 59: cd: ./work_dirs/modelartifacts/8bits/od-5020_tvmdlr_coco_gluoncv-mxnet_yolo3_mobilenet1.0_coco-symbol_json.tar.gz.link.link/artifacts: Not a directory
TIDL_TOOLS_PATH=/home/sefau18/edgeai-benchmark/tidl_tools
LD_LIBRARY_PATH=/home/sefau18/edgeai-benchmark/tidl_tools
PYTHONPATH=:
===================================================================
work_dir = ./work_dirs/modelartifacts/8bits
loading annotations into memory...
Done (t=0.27s)
creating index...
index created!
loading annotations into memory...
Done (t=0.35s)
creating index...
index created!
configs to run: ['imagedet-4_onnxrt_edgeai-benchmark_work_dirs_best_onnx']
number of configs: 1
TASKS                                                       |          |     0% 0/1| [< ]
INFO:20220610-071646: starting process on parallel_device - 0   0%|          || 0/1 [00:00<?, ?it/
INFO:20220610-071649: model_path - /home/sefau18/edgeai-benchmark/work_dirs/best.onnx
INFO:20220610-071649: model_file - /home/sefau18/edgeai-benchmark/work_dirs/modelartifacts/8bits/imagedet-4_onnxrt_edgeai-benchmark_work_dirs_best_onnx/model/best.onnx

INFO:20220610-071649: running - imagedet-4_onnxrt_edgeai-benchmark_work_dirs_best_onnx
INFO:20220610-071649: pipeline_config - {'task_type': 'detection', 'calibration_dataset': <jai_benchmark.datasets.coco_det.COCODetection object at 0x7f8bf24e36d8>, 'input_dataset': <jai_benchmark.datasets.coco_det.COCODetection object at 0x7f8bfa8ddef0>, 'preprocess': <jai_benchmark.preprocess.PreProcessTransforms object at 0x7f8bfa8dde48>, 'session': <jai_benchmark.sessions.onnxrt_session.ONNXRTSession object at 0x7f8bec81bf28>, 'postprocess': <jai_benchmark.postprocess.PostProcessTransforms object at 0x7f8bec81bf98>, 'model_info': {'metric_reference': {'accuracy_ap[.5:.95]%': 23.0}}}
INFO:20220610-071649: import  - imagedet-4_onnxrt_edgeai-benchmark_work_dirs_best_onnx
Preliminary subgraphs created = 1 
Final number of subgraphs created are : 1, - Offloaded Nodes - 1, Total Nodes - 1 

Preliminary subgraphs created = 1 
Final number of subgraphs created are : 1, - Offloaded Nodes - 1, Total Nodes - 1 

[ONNXRuntimeError] : 1 : FAIL : Failed to add kernel for TIDL_0 com.microsoft TIDLExecutionProvider: Conflicting with a registered kernel with op versions.
Traceback (most recent call last):
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/pipelines/pipeline_runner.py", line 136, in _run_pipeline
    accuracy_result = accuracy_pipeline(description)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/pipelines/accuracy_pipeline.py", line 104, in __call__
    param_result = self._run(description=description)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/pipelines/accuracy_pipeline.py", line 130, in _run
    self._import_model(description)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/pipelines/accuracy_pipeline.py", line 182, in _import_model
    self._run_with_log(session.import_model, calib_data)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/pipelines/accuracy_pipeline.py", line 282, in _run_with_log
    return func(*args, **kwargs)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/sessions/onnxrt_session.py", line 52, in import_model
    self.interpreter = self._create_interpreter(is_import=True)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/sessions/onnxrt_session.py", line 131, in _create_interpreter
    provider_options=[runtime_options, {}], sess_options=sess_options)
  File "/home/sefau18/.local/lib/python3.6/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 283, in __init__
    self._create_inference_session(providers, provider_options)
  File "/home/sefau18/.local/lib/python3.6/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 315, in _create_inference_session
    sess.initialize_session(providers, provider_options)
onnxruntime.capi.onnxruntime_pybind11_state.Fail: [ONNXRuntimeError] : 1 : FAIL : Failed to add kernel for TIDL_0 com.microsoft TIDLExecutionProvider: Conflicting with a registered kernel with op versions.
TASKS                                                       | 100%|██████████|| 1/1 [00:04<00:00, 


-------------------------------------------------------------------
===================================================================
settings: {'include_files': None, 'pipeline_type': 'accuracy', 'num_frames': 10000, 'calibration_frames': 50, 'calibration_iterations': 50, 'configs_path': './configs', 'models_path': '../edgeai-modelzoo/models', 'modelartifacts_path': './work_dirs/modelartifacts', 'datasets_path': './dependencies/datasets', 'target_device': 'pc', 'parallel_devices': [0, 1, 2, 3], 'tensor_bits': 8, 'runtime_options': None, 'run_import': True, 'run_inference': True, 'run_missing': True, 'detection_thr': 0.3, 'detection_max': 1000, 'save_output': False, 'model_shortlist': None, 'model_selection': None, 'model_exclusion': None, 'task_selection': None, 'runtime_selection': None, 'session_type_dict': {'onnx': 'onnxrt', 'tflite': 'tflitert', 'mxnet': 'tvmdlr'}, 'dataset_type_dict': {'imagenet': 'imagenetv2c'}, 'dataset_loading': None, 'config_range': None, 'enable_logging': True, 'verbose': False, 'experimental_models': False, 'rewrite_results': False, 'with_udp': True, 'flip_test': False, 'model_transformation_dict': None, 'report_perfsim': False, 'tidl_offload': True, 'input_optimization': None, 'settings_file': 'settings_import_on_pc.yaml', 'basic_keys': ['include_files', 'pipeline_type', 'num_frames', 'calibration_frames', 'calibration_iterations', 'configs_path', 'models_path', 'modelartifacts_path', 'datasets_path', 'target_device', 'parallel_devices', 'tensor_bits', 'runtime_options', 'run_import', 'run_inference', 'run_missing', 'detection_thr', 'detection_max', 'save_output', 'model_shortlist', 'model_selection', 'model_exclusion', 'task_selection', 'runtime_selection', 'session_type_dict', 'dataset_type_dict', 'dataset_loading', 'config_range', 'enable_logging', 'verbose', 'experimental_models', 'rewrite_results', 'with_udp', 'flip_test', 'model_transformation_dict', 'report_perfsim', 'tidl_offload', 'input_optimization', 'settings_file'], 'dataset_cache': None, 'quantization_params': <jai_benchmark.config_settings.QuantizationParams object at 0x7f3ddd94fcf8>, 'quantization_params_qat': <jai_benchmark.config_settings.QuantizationParams object at 0x7f3d8a3bde10>}
Traceback (most recent call last):
  File "./scripts/generate_report.py", line 55, in <module>
    tools.run_report(settings)
  File "/home/sefau18/edgeai-benchmark/jai_benchmark/tools/run_report.py", line 100, in run_report
    results_anchor = results_collection[results_max_name]
KeyError: None
-------------------------------------------------------------------