opencv Documentation

Open Source Computer Vision Library

91,206 stars C++ C++ 1 file · ~26,404 tokens #c-plus-plus#computer-vision#deep-learning#image-processing#opencv
RAW Doc

Technical Documentation: opencv/opencv

ℹ️ Provenance: Multi-Tier Ingestion · CodeWiki Reference

Overview (README.md)

OpenCV: Open Source Computer Vision Library

Resources

Contributing

Please read the contribution guidelines before starting work on a pull request.

Summary of the guidelines:

  • One pull request per issue;
  • Choose the right base branch;
  • Include tests and documentation;
  • Clean up "oops" commits before submitting;
  • Follow the coding style guide.

Additional Resources

Doc CMakeLists (`doc/CMakeLists.txt`)

if(NOT BUILD_DOCS)
return()
endif()

Dependencies scheme (* - optional):

javadoc* -> doxygen_javadoc* -> doxygen_cpp ---------> doxygen -> opencv_docs

\ \ / /

\ -> doxygen_python* -> /

---------------------------------------------------------->

find_package(Doxygen)
if(DOXYGEN_FOUND)
if (DOXYGEN_VERSION VERSION_LESS 1.12)
message(WARNING "Found doxygen ${DOXYGEN_VERSION}, version 1.12 is used for testing, there is "
"a chance your documentation will look different or have some limitations.")
endif()
add_custom_target(doxygen)

not documented modules list

set(blacklist "${DOXYGEN_BLACKLIST}")
list(APPEND blacklist "ts" "world")
unset(CMAKE_DOXYGEN_TUTORIAL_CONTRIB_ROOT)
unset(CMAKE_DOXYGEN_TUTORIAL_JS_ROOT)

set(OPENCV_DOCS_EXCLUDE_CUDA ON)
if(";${OPENCV_MODULES_EXTRA};" MATCHES ";cudev;")
set(OPENCV_DOCS_EXCLUDE_CUDA OFF)
list(APPEND CMAKE_DOXYGEN_ENABLED_SECTIONS "CUDA_MODULES")
endif()

gathering headers

set(paths_include)
set(paths_doc)
set(paths_bib)
set(paths_sample)
set(paths_tutorial)
set(paths_hal_interface)
set(refs_main)
set(refs_extra)
set(deps)
foreach(m ${OPENCV_MODULES_MAIN} ${OPENCV_MODULES_EXTRA})
set(the_module "${m}")
if(NOT the_module MATCHES "^opencv_")
set(the_module "opencv_${m}")
endif()
list(FIND blacklist ${m} pos)
if(NOT EXISTS "${OPENCV_MODULE${the_module}LOCATION}/include"
AND NOT EXISTS "${OPENCV_MODULE${the_module}_LOCATION}/doc"
)
set(pos -2) # blacklist
endif()
if(${pos} EQUAL -1)
list(APPEND CMAKE_DOXYGEN_ENABLED_SECTIONS "HAVE_opencv${m}")
# include folder
set(header_dir "${OPENCV_MODULE_opencv${m}LOCATION}/include")
if(EXISTS "${header_dir}")
list(APPEND paths_include "${header_dir}")
list(APPEND deps ${header_dir})
if(OPENCV_DOCS_EXCLUDE_CUDA)
if(EXISTS "${OPENCV_MODULE_opencv${m}LOCATION}/include/opencv2/${m}/cuda")
list(APPEND CMAKE_DOXYGEN_EXCLUDE_LIST "${OPENCV_MODULE_opencv${m}LOCATION}/include/opencv2/${m}/cuda")
endif()
file(GLOB list_cuda_files "${OPENCV_MODULE_opencv${m}LOCATION}/include/opencv2/${m}/cuda.hpp")
if(list_cuda_files)
list(APPEND CMAKE_DOXYGEN_EXCLUDE_LIST ${list_cuda_files})
endif()
endif()
endif()
# doc folder
set(docs_dir "${OPENCV_MODULE_opencv${m}LOCATION}/doc")
if(EXISTS "${docs_dir}")
list(APPEND paths_doc "${docs_dir}")
list(APPEND deps ${docs_dir})
endif()
# sample folder
set(sample_dir "${OPENCV_MODULE_opencv${m}LOCATION}/samples")
if(EXISTS "${sample_dir}")
list(APPEND paths_sample "${sample_dir}")
list(APPEND deps ${sample_dir})
endif()
# tutorial folder
set(tutorial_dir "${OPENCV_MODULE_opencv${m}_LOCATION}/tutorials")
if(EXISTS "${tutorial_dir}")
list(APPEND paths_tutorial "${tutorial_dir}")
list(APPEND deps ${tutorial_dir})

text
# tutorial reference entry
    file(GLOB tutorials RELATIVE "${OPENCV_MODULE_opencv_${m}_LOCATION}" "${tutorial_dir}/*.markdown")
    foreach (t ${tutorials})
      if (NOT DEFINED CMAKE_DOXYGEN_TUTORIAL_CONTRIB_ROOT)
        set(CMAKE_DOXYGEN_TUTORIAL_CONTRIB_ROOT "- @ref tutorial_contrib_root")
        set(tutorial_contrib_root "${CMAKE_CURRENT_BINARY_DIR}/contrib_tutorials.markdown")
        file(WRITE "${tutorial_contrib_root}"
          "Tutorials for contrib modules {#tutorial_contrib_root}\n"
          "=============================\n")
      endif()
      file(STRINGS "${OPENCV_MODULE_opencv_${m}_LOCATION}/${t}" tutorial_id LIMIT_COUNT 1 REGEX ".*{#[^}]+}")
      string(REGEX REPLACE ".*{#([^}]+)}" "\\1" tutorial_id "${tutorial_id}")
      file(APPEND "${tutorial_contrib_root}" "- ${m}. @subpage ${tutorial_id}\n")
    endforeach()
  endif()
  # HAL replacement file
  set(replacement_header "${OPENCV_MODULE_opencv_${m}_LOCATION}/src/hal_replacement.hpp")
  if(EXISTS "${replacement_header}")
    list(APPEND paths_hal_interface "${replacement_header}")
  endif()

  # BiBTeX file
  set(bib_file "${docs_dir}/${m}.bib")
  if(EXISTS "${bib_file}")
    set(paths_bib "${paths_bib} ${bib_file}")
    list(APPEND deps ${bib_file})
  endif()
  # Reference entry
  set(one_ref "\t- ${m}. @ref ${m}\n")
  list(FIND OPENCV_MODULES_EXTRA ${m} _pos)
  if(${_pos} EQUAL -1)
    set(refs_main "${refs_main}${one_ref}")
  else()
    set(refs_extra "${refs_extra}${one_ref}")
  endif()
endif()

endforeach()

fix references

set(ref_header "Module name | Folder\n----------- | ------")

if(refs_main)

set(refs_main "### Main modules\n${ref_header}\n${refs_main}")

endif()

if(refs_extra)

set(refs_extra "### Extra modules\n${ref_header}\n${refs_extra}")

endif()

if(refs_main)
set(refs_main "- Main modules:\n${refs_main}")
endif()
if(refs_extra)
set(refs_extra "- Extra modules:\n${refs_extra}")
endif()

additional config

set(doxyfile "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")
set(rootfile "${CMAKE_CURRENT_BINARY_DIR}/root.markdown")
set(bibfile "${CMAKE_CURRENT_SOURCE_DIR}/opencv.bib")
set(faqfile "${CMAKE_CURRENT_SOURCE_DIR}/faq.markdown")
set(tutorial_path "${CMAKE_CURRENT_SOURCE_DIR}/tutorials")
set(tutorial_py_path "${CMAKE_CURRENT_SOURCE_DIR}/py_tutorials")
set(CMAKE_DOXYGEN_TUTORIAL_JS_ROOT "- @ref tutorial_js_root")
set(tutorial_js_path "${CMAKE_CURRENT_SOURCE_DIR}/js_tutorials")
set(example_path "${CMAKE_SOURCE_DIR}/samples")

set(doxygen_image_path
${CMAKE_CURRENT_SOURCE_DIR}/images
${paths_doc}
${tutorial_path}
${tutorial_py_path}
${tutorial_js_path}
${paths_tutorial}
#${OpenCV_SOURCE_DIR}/samples/data # TODO: need to resolve ambiguous conflicts first
${OpenCV_SOURCE_DIR}
${OpenCV_SOURCE_DIR}/modules # /modules
${OPENCV_EXTRA_MODULES_PATH} # /modules
${OPENCV_DOCS_EXTRA_IMAGE_PATH} # custom variable for user modules
)

set export variables

string(REPLACE ";" " \\n" CMAKE_DOXYGEN_INPUT_LIST "${rootfile} ; ${faqfile} ; ${paths_include} ; ${paths_hal_interface} ; ${paths_doc} ; ${tutorial_path} ; ${tutorial_py_path} ; ${tutorial_js_path} ; ${paths_tutorial} ; ${tutorial_contrib_root}")
string(REPLACE ";" " \\n" CMAKE_DOXYGEN_IMAGE_PATH "${doxygen_image_path}")
string(REPLACE ";" " \\n" CMAKE_DOXYGEN_EXCLUDE_LIST "${CMAKE_DOXYGEN_EXCLUDE_LIST}")
string(REPLACE ";" " " CMAKE_DOXYGEN_ENABLED_SECTIONS "${CMAKE_DOXYGEN_ENABLED_SECTIONS}")

TODO: remove paths_doc from EXAMPLE_PATH after face module tutorials/samples moved to separate folders

string(REPLACE ";" " \\n" CMAKE_DOXYGEN_EXAMPLE_PATH "${example_path} ; ${paths_doc} ; ${paths_sample}")
string(REPLACE ";" " \\n" CMAKE_DOXYGEN_INCLUDE_ROOTS "${paths_include}")
set(CMAKE_DOXYGEN_LAYOUT "${CMAKE_CURRENT_BINARY_DIR}/DoxygenLayout.xml")
set(CMAKE_DOXYGEN_OUTPUT_PATH "doxygen")
set(CMAKE_DOXYGEN_MAIN_REFERENCE "${refs_main}")
set(CMAKE_DOXYGEN_EXTRA_REFERENCE "${refs_extra}")
set(CMAKE_EXTRA_BIB_FILES "${bibfile} ${paths_bib}")
if (CMAKE_DOXYGEN_GENERATE_QHP)
set(CMAKE_DOXYGEN_GENERATE_QHP "YES")
else()
set(CMAKE_DOXYGEN_GENERATE_QHP "NO")
endif()

list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/opencv.ico")
list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/pattern.png")
list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/acircles_pattern.png")
list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/charuco_board_pattern.png")
list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/bodybg.png")

list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/mymath.sty")

list(APPEND CMAKE_DOXYGEN_HTML_FILES "${CMAKE_CURRENT_SOURCE_DIR}/tutorial-utils.js")
string(REPLACE ";" " \\n" CMAKE_DOXYGEN_HTML_FILES "${CMAKE_DOXYGEN_HTML_FILES}")

if (DOXYGEN_DOT_EXECUTABLE)
message(STATUS "Found DOT executable: ${DOXYGEN_DOT_EXECUTABLE}")
set(init_dot_path "${DOXYGEN_DOT_EXECUTABLE}")
set(init_dot_mode "YES")
else()
set(init_dot_path "")
set(init_dot_mode "NO")
endif()
set(OPENCV_DOCS_DOT_PATH "${init_dot_path}" CACHE PATH "Doxygen/DOT_PATH value")
set(OPENCV_DOCS_HAVE_DOT "${init_dot_mode}" CACHE BOOL "Doxygen: build extra diagrams")
set(CMAKECONFIG_DOT_PATH "${OPENCV_DOCS_DOT_PATH}")
set(CMAKECONFIG_HAVE_DOT "${OPENCV_DOCS_HAVE_DOT}")

'png' is good enough for compatibility (but requires +50% storage space)

set(OPENCV_DOCS_DOT_IMAGE_FORMAT "svg" CACHE STRING "Doxygen/DOT_IMAGE_FORMAT value")
set(CMAKECONFIG_DOT_IMAGE_FORMAT "${OPENCV_DOCS_DOT_IMAGE_FORMAT}")

Doxygen 1.8.16 fix: https://github.com/doxygen/doxygen/pull/6870

NO is needed here: https://github.com/opencv/opencv/pull/16039

set(OPENCV_DOCS_INTERACTIVE_SVG "NO" CACHE BOOL "Doxygen/INTERACTIVE_SVG value")
set(CMAKECONFIG_INTERACTIVE_SVG "${OPENCV_DOCS_INTERACTIVE_SVG}")

set(OPENCV_DOCS_DOXYFILE_IN "Doxyfile.in" CACHE PATH "Doxygen configuration template file (Doxyfile.in)")
set(OPENCV_DOCS_DOXYGEN_LAYOUT "DoxygenLayout.xml" CACHE PATH "Doxygen layout file (.xml)")

writing file

configure_file("${OPENCV_DOCS_DOXYGEN_LAYOUT}" DoxygenLayout.xml @ONLY)
configure_file("${OPENCV_DOCS_DOXYFILE_IN}" ${doxyfile} @ONLY)
configure_file(root.markdown.in ${rootfile} @ONLY)

js tutorial assets

set(opencv_tutorial_html_dir "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html")
set(js_tutorials_assets_dir "${CMAKE_CURRENT_SOURCE_DIR}/js_tutorials/js_assets")
set(js_tutorials_assets_deps "")

make sure the build directory exists

file(MAKE_DIRECTORY "${opencv_tutorial_html_dir}")

gather and copy specific files for js tutorials

file(GLOB_RECURSE js_assets "${js_tutorials_assets_dir}/*")
ocv_list_filterout(js_assets "\\.eslintrc.json")
list(APPEND js_assets "${OpenCV_SOURCE_DIR}/samples/cpp/tutorial_code/calib3d/real_time_pose_estimation/Data/box.mp4")

if(BUILD_opencv_js)
set(ocv_js_dir "${CMAKE_BINARY_DIR}/bin")
set(ocv_js "opencv.js")
list(APPEND js_assets "${ocv_js_dir}/${ocv_js}")
elseif(DEFINED OPENCV_JS_LOCATION)
list(APPEND js_assets "${OPENCV_JS_LOCATION}")
endif()

copy haar cascade files

set(haar_cascade_files "")
set(data_harrcascades_path "${OpenCV_SOURCE_DIR}/data/haarcascades/")
list(APPEND js_tutorials_assets_deps "${data_harrcascades_path}/haarcascade_frontalface_default.xml" "${data_harrcascades_path}/haarcascade_eye.xml")
list(APPEND js_assets "${data_harrcascades_path}/haarcascade_frontalface_default.xml" "${data_harrcascades_path}/haarcascade_eye.xml")

foreach(f ${js_assets})
get_filename_component(fname "${f}" NAME)
add_custom_command(OUTPUT "${opencv_tutorial_html_dir}/${fname}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${f}" "${opencv_tutorial_html_dir}/${fname}"
DEPENDS "${f}"
COMMENT "Copying ${fname}"
)
list(APPEND js_tutorials_assets_deps "${f}" "${opencv_tutorial_html_dir}/${fname}")
endforeach()

add_custom_target(
doxygen_cpp
COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile}
DEPENDS ${doxyfile} ${rootfile} ${bibfile} ${deps} ${js_tutorials_assets_deps}
COMMENT "Generate Doxygen documentation"
)

if(NOT DEFINED HAVE_PYTHON_BS4 AND PYTHON_DEFAULT_EXECUTABLE)
# Documentation post-processing tool requires BuautifulSoup Python package
execute_process(COMMAND "${PYTHON_DEFAULT_EXECUTABLE}" -c "import bs4; from bs4 import BeautifulSoup; print(bs4.version)"
RESULT_VARIABLE _result
OUTPUT_VARIABLE _bs4_version
OUTPUT_STRIP_TRAILING_WHITESPACE)

text
if(NOT _result EQUAL 0)
  set(HAVE_PYTHON_BS4 0 CACHE INTERNAL "")
else()
  message(STATUS "Python BeautifulSoup (bs4) version: ${_bs4_version}")
  set(HAVE_PYTHON_BS4 1 CACHE INTERNAL "")
endif()

endif()

if(PYTHON_DEFAULT_EXECUTABLE
AND HAVE_PYTHON_BS4
AND OPENCV_PYTHON_SIGNATURES_FILE
AND TARGET gen_opencv_python_source)
add_custom_target(doxygen_python
COMMAND ${PYTHON_DEFAULT_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON_SIGNATURES_FILE}" "python"
DEPENDS doxygen_cpp gen_opencv_python_source
COMMENT "Inject Python signatures into documentation"
)
endif()

add_dependencies(doxygen doxygen_cpp)

if(TARGET doxygen_python)
add_dependencies(doxygen doxygen_python)
endif()

if(TARGET doxygen_javadoc)
add_dependencies(doxygen_cpp doxygen_javadoc)
endif()

add_dependencies(opencv_docs doxygen)

install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html
DESTINATION "${OPENCV_DOC_INSTALL_PATH}"
COMPONENT "docs" OPTIONAL
${compatible_MESSAGE_NEVER}
)
endif()

Doc Disabled Doc Warnings (`doc/disabled_doc_warnings.txt`)

doxygen citelist build workaround

citelist : .Unexpected new line character.

Doc Opencv Logo (`doc/opencv-logo.md`)

OpenCV logo has been originally designed and contributed to OpenCV by Adi Shavit in 2006. The graphical part consists of three stylized letters O, C, V, colored in the primary R, G, B color components, used by humans and computers to perceive the world. It is shaped in a way to mimic the famous Kanizsa's triangle to emphasize that the prior knowledge and internal processing are at least as important as the actually acquired "raw" data.

The restyled version of the logo has been designed and contributed by xperience.ai in July 2020 for the 20th anniversary of OpenCV.

The logo uses Exo 2 font by Natanael Gama distributed under OFL license.

Higher-resolution version of the logo, as well as SVG version of it, can be obtained at OpenCV Media Kit.

Modules Objc Doc LandingPage Ios (`modules/objc/doc/LandingPage_ios.md`)

This is the documentation for the iOS OpenCV framework.

Quick Start Tutorial

Follow the steps below to get a simple "Hello World" app running on iOS

  • Open Xcode and create a new project from the File > New > Project... menu

  • From template selection dialog select the iOS tab and then select App

  • Enter a name for the project and set Interface to Storyboard and Language to Swift

  • Choose a location to save the project

  • Create a new folder Framework

  • In a Finder window copy the framework file to the newly created folder. The framework should appear in the project navigator as below. (Note: in this case the framework has extension .framework but depending on how you built it or from where you obtained it your framework may have extension .xcframework)

  • In the project navigator select the ViewController.swift file

  • Add an import for the OpenCV framework below the existing import for UIKit

swift, copy
import opencv2
  • Replace the implementation of the viewDidLoad function with the following code
swift, copy
override func viewDidLoad() {
    super.viewDidLoad()
    let white = Scalar(255, 255, 255, 255)
    let black = Scalar(0, 0, 0, 255)
    let m = Mat(rows: 400, cols: Int32(view.frame.width), type: CvType.CV_8UC4, scalar: white)
    Imgproc.putText(img: m, text: "Hello World", org: Point(x: 10, y: 150), fontFace: HersheyFonts.FONT_HERSHEY_DUPLEX, fontScale: 2, color: black)
    let image = m.toUIImage()
    let imageView = UIImageView(image: image)
    self.view.addSubview(imageView)
}

  • The ViewController.swift file should now look like this

  • Launch the app

  • The app home screen should look like this

If that has all worked well then congratulations. If not then feel free to reach out to the OpenCV community for help. Also feel free to contribute fixes and improvements.

Modules Objc Doc LandingPage Osx (`modules/objc/doc/LandingPage_osx.md`)

This is the documentation for the macOS OpenCV framework.

Quick Start Tutorial

Follow the steps below to get a simple "Hello World" app running on macOS

  • Open Xcode and create a new project from the File > New > Project... menu

  • From the template selection dialog select the macOS tab and then select App

  • Enter a name for the project and set Interface to Storyboard and Language to Swift

  • Choose a location to save the project

  • Create a new folder Framework

  • In a Finder window copy the framework file to the newly created folder. The framework should appear in the project navigator as below. (Note: in this case the framework has extension .framework but depending on how you built it or from where you obtained it your framework may have extension .xcframework)

  • Select the project root in the project navigator and on the General tab locate the Frameworks, Libraries and Embedded Content section

  • In the Choose frameworks and libraries to add dialog add the following

    • OpenCL.framework
    • liblapack.tbd
    • libblas.tbd

  • The Frameworks, Libraries and Embedded Content section should now look like this

  • In the project navigator select the ViewController.swift file

  • Add an import for the OpenCV framework below the existing import for Cocoa

swift, copy
import opencv2
  • Replace the implementation of the viewDidLoad function with the following code
swift, copy
override func viewDidLoad() {
    super.viewDidLoad()
    let white = Scalar(255, 255, 255, 255)
    let black = Scalar(0, 0, 0, 255)
    let m = Mat(rows: 400, cols: Int32(view.frame.width), type: CvType.CV_8UC4, scalar: white)
    Imgproc.putText(img: m, text: "Hello World", org: Point(x: 10, y: 250), fontFace: HersheyFonts.FONT_HERSHEY_DUPLEX, fontScale: 2, color: black)
    let image = m.toNSImage()
    let imageView = NSImageView(frame: NSRect(x: 0, y: 0, width: Int(m.cols()), height: Int(m.rows())))
    imageView.image = image
    self.view.addSubview(imageView)
}

  • The ViewController.swift file should now look like this

  • Launch the app

  • The app home screen should look like this

If that has all worked well then congratulations. If not then feel free to reach out to the OpenCV community for help. Also feel free to contribute fixes and improvements.

Doc Tutorials Dnn Dnn Custom Layers Dnn Custom Layers (`doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md`)

Custom deep learning layers support {#tutorial_dnn_custom_layers}

@tableofcontents

@prev_tutorial{tutorial_dnn_javascript}
@next_tutorial{tutorial_dnn_OCR}
Original author Dmitry Kurtaev
Compatibility OpenCV >= 3.4.1

Introduction

Deep learning is a fast-growing area. New approaches to building neural networks
usually introduce new types of layers. These could be modifications of existing
ones or implementation of outstanding research ideas.

OpenCV allows importing and running networks from different deep learning frameworks.
There is a number of the most popular layers. However, you can face a problem that
your network cannot be imported using OpenCV because some layers of your network
can be not implemented in the deep learning engine of OpenCV.

The first solution is to create a feature request at https://github.com/opencv/opencv/issues
mentioning details such as a source of a model and a type of new layer.
The new layer could be implemented if the OpenCV community shares this need.

The second way is to define a custom layer so that OpenCV's deep learning engine
will know how to use it. This tutorial is dedicated to show you a process of deep
learning model's import customization.

Define a custom layer in C++

Deep learning layer is a building block of network's pipeline.
It has connections to input blobs and produces results to output blobs.
There are trained weights and hyper-parameters.
Layers' names, types, weights and hyper-parameters are stored in files are
generated by native frameworks during training. If OpenCV encounters unknown
layer type it throws an exception while trying to read a model:

text
Unspecified error: Can't create layer "layer_name" of type "MyType" in function getLayerInstance

To import the model correctly you have to derive a class from cv::dnn::Layer with
the following methods:

@snippet dnn/custom_layers.hpp A custom layer interface

And register it before the import:

@snippet dnn/custom_layers.hpp Register a custom layer

@note MyType is a type of unimplemented layer from the thrown exception.

Let's see what all the methods do:

  • Constructor

@snippet dnn/custom_layers.hpp MyLayer::MyLayer

Retrieves hyper-parameters from cv::dnn::LayerParams. If your layer has trainable
weights they will be already stored in the Layer's member cv::dnn::Layer::blobs.

  • A static method create

@snippet dnn/custom_layers.hpp MyLayer::create

This method should create an instance of you layer and return cv::Ptr with it.

  • Output blobs' shape computation

@snippet dnn/custom_layers.hpp MyLayer::getMemoryShapes

Returns layer's output shapes depending on input shapes. You may request an extra
memory using internals.

  • Run a layer

@snippet dnn/custom_layers.hpp MyLayer::forward

Implement a layer's logic here. Compute outputs for given inputs.

@note OpenCV manages memory allocated for layers. In the most cases the same memory
can be reused between layers. So your forward implementation should not rely on that
the second invocation of forward will have the same data at outputs and internals.

  • Optional finalize method

@snippet dnn/custom_layers.hpp MyLayer::finalize

The chain of methods is the following: OpenCV deep learning engine calls create
method once, then it calls getMemoryShapes for every created layer, then you
can make some preparations depend on known input dimensions at cv::dnn::Layer::finalize.
After network was initialized only forward method is called for every network's input.

@note Varying input blobs' sizes such height, width or batch size make OpenCV
reallocate all the internal memory. That leads to efficiency gaps. Try to initialize
and deploy models using a fixed batch size and image's dimensions.

Example: custom layer from Caffe

Let's create a custom layer Interp from https://github.com/cdmh/deeplab-public.
It's just a simple resize that takes an input blob of size N x C x Hi x Wi and returns
an output blob of size N x C x Ho x Wo where N is a batch size, C is a number of channels,
Hi x Wi and Ho x Wo are input and output height x width correspondingly.
This layer has no trainable weights but it has hyper-parameters to specify an output size.

In example,

text
layer {
  name: "output"
  type: "Interp"
  bottom: "input"
  top: "output"
  interp_param {
    height: 9
    width: 8
  }
}

This way our implementation can look like:

@snippet dnn/custom_layers.hpp InterpLayer

Next we need to register a new layer type and try to import the model.

@snippet dnn/custom_layers.hpp Register InterpLayer

Example: custom layer from TensorFlow

This is an example of how to import a network with tf.image.resize_bilinear
operation. This is also a resize but with an implementation different from OpenCV's or Interp above.

Let's create a single layer network:

{.py}
inp = tf.placeholder(tf.float32, [2, 3, 4, 5], 'input')
resized = tf.image.resize_bilinear(inp, size=[9, 8], name='resize_bilinear')

OpenCV sees that TensorFlow's graph in the following way:

text
node {
  name: "input"
  op: "Placeholder"
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
}
node {
  name: "resize_bilinear/size"
  op: "Const"
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\t\000\000\000\010\000\000\000"
      }
    }
  }
}
node {
  name: "resize_bilinear"
  op: "ResizeBilinear"
  input: "input:0"
  input: "resize_bilinear/size"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "align_corners"
    value {
      b: false
    }
  }
}
library {
}

Custom layers import from TensorFlow is designed to put all layer's attr into
cv::dnn::LayerParams but input Const blobs into cv::dnn::Layer::blobs.
In our case resize's output shape will be stored in layer's blobs[0].

@snippet dnn/custom_layers.hpp ResizeBilinearLayer

Next we register a layer and try to import the model.

@snippet dnn/custom_layers.hpp Register ResizeBilinearLayer

Define a custom layer in Python

The following example shows how to customize OpenCV's layers in Python.

Let's consider Holistically-Nested Edge Detection
deep learning model. That was trained with one and only difference comparing to
a current version of Caffe framework. Crop
layers that receive two input blobs and crop the first one to match spatial dimensions
of the second one used to crop from the center. Nowadays Caffe's layer does it
from the top-left corner. So using the latest version of Caffe or OpenCV you will
get shifted results with filled borders.

Next we're going to replace OpenCV's Crop layer that makes top-left cropping by
a centric one.

  • Create a class with getMemoryShapes and forward methods

@snippet dnn/edge_detection.py CropLayer

@note Both methods should return lists.

  • Register a new layer.

@snippet dnn/edge_detection.py Register

That's it! We have replaced an implemented OpenCV's layer to a custom one.
You may find a full script in the source code.

Doc Tutorials Dnn Dnn Pytorch Tf Classification Pytorch Cls Model Conversion C Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_classification/pytorch_cls_model_conversion_c_tutorial.md`)

Conversion of PyTorch Classification Models and Launch with OpenCV C++ {#pytorch_cls_c_tutorial_dnn_conversion}

@prev_tutorial{pytorch_cls_tutorial_dnn_conversion}
Original author Anastasia Murzova
Compatibility OpenCV >= 4.5

Goals

In this tutorial you will learn how to:

  • convert PyTorch classification models into ONNX format
  • run converted PyTorch model with OpenCV C/C++ API
  • provide model inference

We will explore the above-listed points by the example of ResNet-50 architecture.

Introduction

Let's briefly view the key concepts involved in the pipeline of PyTorch models transition with OpenCV API. The initial step in conversion of PyTorch models into cv::dnn::Net
is model transferring into ONNX format. ONNX aims at the interchangeability of the neural networks between various frameworks. There is a built-in function in PyTorch for ONNX conversion: torch.onnx.export.
Further the obtained .onnx model is passed into cv::dnn::readNetFromONNX or cv::dnn::readNet.

Requirements

To be able to experiment with the below code you will need to install a set of libraries. We will use a virtual environment with python3.7+ for this:

console
virtualenv -p /usr/bin/python3.7 <env_dir_path>
source <env_dir_path>/bin/activate

For OpenCV-Python building from source, follow the corresponding instructions from the @ref tutorial_py_table_of_contents_setup.

Before you start the installation of the libraries, you can customize the requirements.txt, excluding or including (for example, opencv-python) some dependencies.
The below line initiates requirements installation into the previously activated virtual environment:

console
pip install -r requirements.txt

Practice

In this part we are going to cover the following points:

  1. create a classification model conversion pipeline
  2. provide the inference, process prediction results

Model Conversion Pipeline

The code in this subchapter is located in the samples/dnn/dnn_model_runner module and can be executed with the line:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_resnet50_onnx

The following code contains the description of the below-listed steps:

  1. instantiate PyTorch model
  2. convert PyTorch model into .onnx
python
# initialize PyTorch ResNet-50 model
original_model = models.resnet50(pretrained=True)

# get the path to the converted into ONNX PyTorch model
full_model_path = get_pytorch_onnx_model(original_model)
print("PyTorch ResNet-50 model was successfully converted: ", full_model_path)

get_pytorch_onnx_model(original_model) function is based on torch.onnx.export(...) call:

python
# define the directory for further converted model save
onnx_model_path = "models"
# define the name of further converted model
onnx_model_name = "resnet50.onnx"

# create directory for further converted model
os.makedirs(onnx_model_path, exist_ok=True)

# get full path to the converted model
full_model_path = os.path.join(onnx_model_path, onnx_model_name)

# generate model input
generated_input = Variable(
    torch.randn(1, 3, 224, 224)
)

# model export into ONNX format
torch.onnx.export(
    original_model,
    generated_input,
    full_model_path,
    verbose=True,
    input_names=["input"],
    output_names=["output"],
    opset_version=11
)

After the successful execution of the above code we will get the following output:

console
PyTorch ResNet-50 model was successfully converted: models/resnet50.onnx

The proposed in dnn/samples module dnn_model_runner allows us to reproduce the above conversion steps for the following PyTorch classification models:

  • alexnet
  • vgg11
  • vgg13
  • vgg16
  • vgg19
  • resnet18
  • resnet34
  • resnet50
  • resnet101
  • resnet152
  • squeezenet1_0
  • squeezenet1_1
  • resnext50_32x4d
  • resnext101_32x8d
  • wide_resnet50_2
  • wide_resnet101_2

To obtain the converted model, the following line should be executed:

text
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name <pytorch_cls_model_name> --evaluate False

For the ResNet-50 case the below line should be run:

text
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name resnet50 --evaluate False

The default root directory for the converted model storage is defined in module CommonConfig:

python
@dataclass
class CommonConfig:
    output_data_root_dir: str = "dnn_model_runner/dnn_conversion"

Thus, the converted ResNet-50 will be saved in dnn_model_runner/dnn_conversion/models.

Inference Pipeline

Now we can use models/resnet50.onnx for the inference pipeline using OpenCV C/C++ API. The implemented pipeline can be found in samples/dnn/classification.cpp.
After the build of samples (BUILD_EXAMPLES flag value should be ON), the appropriate example_dnn_classification executable file will be provided.

To provide model inference we will use the below squirrel photo (under CC0 license) corresponding to ImageNet class ID 335:

console
fox squirrel, eastern fox squirrel, Sciurus niger

For the label decoding of the obtained prediction, we also need imagenet_classes.txt file, which contains the full list of the ImageNet classes.

In this tutorial we will run the inference process for the converted PyTorch ResNet-50 model from the build (samples/build) directory:

text
./dnn/example_dnn_classification --model=../dnn/models/resnet50.onnx --input=../data/squirrel_cls.jpg --width=224 --height=224 --rgb=true --scale="0.003921569" --mean="123.675 116.28 103.53" --std="0.229 0.224 0.225" --crop=true --initial_width=256 --initial_height=256 --classes=../data/dnn/classification_classes_ILSVRC2012.txt

Let's explore classification.cpp key points step by step:

  1. read the model with cv::dnn::readNet, initialize the network:
cpp
Net net = readNet(model, config, framework);

The model parameter value is taken from --model key. In our case, it is resnet50.onnx.

  • preprocess input image:
cpp
if (rszWidth != 0 && rszHeight != 0)
{
    resize(frame, frame, Size(rszWidth, rszHeight));
}

// Create a 4D blob from a frame
blobFromImage(frame, blob, scale, Size(inpWidth, inpHeight), mean, swapRB, crop);

// Check std values.
if (std.val[0] != 0.0 && std.val[1] != 0.0 && std.val[2] != 0.0)
{
    // Divide blob by std.
    divide(blob, std, blob);
}

In this step we use cv::dnn::blobFromImage function to prepare model input.
We set Size(rszWidth, rszHeight) with --initial_width=256 --initial_height=256 for the initial image resize as it's described in PyTorch ResNet inference pipeline.

It should be noted that firstly in cv::dnn::blobFromImage mean value is subtracted and only then pixel values are multiplied by scale.
Thus, we use --mean="123.675 116.28 103.53", which is equivalent to [0.485, 0.456, 0.406] multiplied by 255.0 to reproduce the original image preprocessing order for PyTorch classification models:

python
img /= 255.0
img -= [0.485, 0.456, 0.406]
img /= [0.229, 0.224, 0.225]
  • make forward pass:
cpp
net.setInput(blob);
Mat prob = net.forward();
  • process the prediction:
cpp
Point classIdPoint;
double confidence;
minMaxLoc(prob.reshape(1, 1), 0, &confidence, 0, &classIdPoint);
int classId = classIdPoint.x;

Here we choose the most likely object class. The classId result for our case is 335 - fox squirrel, eastern fox squirrel, Sciurus niger:

Doc Tutorials Dnn Dnn Pytorch Tf Classification Pytorch Cls Model Conversion Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_classification/pytorch_cls_model_conversion_tutorial.md`)

Conversion of PyTorch Classification Models and Launch with OpenCV Python {#pytorch_cls_tutorial_dnn_conversion}

@prev_tutorial{tutorial_dnn_OCR}
@next_tutorial{pytorch_cls_c_tutorial_dnn_conversion}
Original author Anastasia Murzova
Compatibility OpenCV >= 4.5

Goals

In this tutorial you will learn how to:

  • convert PyTorch classification models into ONNX format
  • run converted PyTorch model with OpenCV Python API
  • obtain an evaluation of the PyTorch and OpenCV DNN models.

We will explore the above-listed points by the example of the ResNet-50 architecture.

Introduction

Let's briefly view the key concepts involved in the pipeline of PyTorch models transition with OpenCV API. The initial step in conversion of PyTorch models into cv.dnn.Net
is model transferring into ONNX format. ONNX aims at the interchangeability of the neural networks between various frameworks. There is a built-in function in PyTorch for ONNX conversion: torch.onnx.export.
Further the obtained .onnx model is passed into cv.dnn.readNetFromONNX.

Requirements

To be able to experiment with the below code you will need to install a set of libraries. We will use a virtual environment with python3.7+ for this:

console
virtualenv -p /usr/bin/python3.7 <env_dir_path>
source <env_dir_path>/bin/activate

For OpenCV-Python building from source, follow the corresponding instructions from the @ref tutorial_py_table_of_contents_setup.

Before you start the installation of the libraries, you can customize the requirements.txt, excluding or including (for example, opencv-python) some dependencies.
The below line initiates requirements installation into the previously activated virtual environment:

console
pip install -r requirements.txt

Practice

In this part we are going to cover the following points:

  1. create a classification model conversion pipeline and provide the inference
  2. evaluate and test classification models

If you'd like merely to run evaluation or test model pipelines, the "Model Conversion Pipeline" part can be skipped.

Model Conversion Pipeline

The code in this subchapter is located in the dnn_model_runner module and can be executed with the line:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_resnet50

The following code contains the description of the below-listed steps:

  1. instantiate PyTorch model
  2. convert PyTorch model into .onnx
  3. read the transferred network with OpenCV API
  4. prepare input data
  5. provide inference
python
# initialize PyTorch ResNet-50 model
original_model = models.resnet50(pretrained=True)

# get the path to the converted into ONNX PyTorch model
full_model_path = get_pytorch_onnx_model(original_model)

# read converted .onnx model with OpenCV API
opencv_net = cv2.dnn.readNetFromONNX(full_model_path)
print("OpenCV model was successfully read. Layer IDs: \n", opencv_net.getLayerNames())

# get preprocessed image
input_img = get_preprocessed_img("../data/squirrel_cls.jpg")

# get ImageNet labels
imagenet_labels = get_imagenet_labels("../data/dnn/classification_classes_ILSVRC2012.txt")

# obtain OpenCV DNN predictions
get_opencv_dnn_prediction(opencv_net, input_img, imagenet_labels)

# obtain original PyTorch ResNet50 predictions
get_pytorch_dnn_prediction(original_model, input_img, imagenet_labels)

To provide model inference we will use the below squirrel photo (under CC0 license) corresponding to ImageNet class ID 335:

console
fox squirrel, eastern fox squirrel, Sciurus niger

For the label decoding of the obtained prediction, we also need imagenet_classes.txt file, which contains the full list of the ImageNet classes.

Let's go deeper into each step by the example of pretrained PyTorch ResNet-50:

  • instantiate PyTorch ResNet-50 model:
python
# initialize PyTorch ResNet-50 model
original_model = models.resnet50(pretrained=True)
  • convert PyTorch model into ONNX:
python
# define the directory for further converted model save
onnx_model_path = "models"
# define the name of further converted model
onnx_model_name = "resnet50.onnx"

# create directory for further converted model
os.makedirs(onnx_model_path, exist_ok=True)

# get full path to the converted model
full_model_path = os.path.join(onnx_model_path, onnx_model_name)

# generate model input
generated_input = Variable(
    torch.randn(1, 3, 224, 224)
)

# model export into ONNX format
torch.onnx.export(
    original_model,
    generated_input,
    full_model_path,
    verbose=True,
    input_names=["input"],
    output_names=["output"],
    opset_version=11
)

After the successful execution of the above code, we will get models/resnet50.onnx.

  • read the transferred network with cv.dnn.readNetFromONNX passing the obtained in the previous step ONNX model into it:
python
# read converted .onnx model with OpenCV API
opencv_net = cv2.dnn.readNetFromONNX(full_model_path)
  • prepare input data:
python
# read the image
input_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
input_img = input_img.astype(np.float32)

input_img = cv2.resize(input_img, (256, 256))

# define preprocess parameters
mean = np.array([0.485, 0.456, 0.406]) * 255.0
scale = 1 / 255.0
std = [0.229, 0.224, 0.225]

# prepare input blob to fit the model input:
# 1. subtract mean
# 2. scale to set pixel values from 0 to 1
input_blob = cv2.dnn.blobFromImage(
    image=input_img,
    scalefactor=scale,
    size=(224, 224),  # img target size
    mean=mean,
    swapRB=True,  # BGR -> RGB
    crop=True  # center crop
)
# 3. divide by std
input_blob[0] /= np.asarray(std, dtype=np.float32).reshape(3, 1, 1)

In this step we read the image and prepare model input with cv.dnn.blobFromImage function, which returns 4-dimensional blob.
It should be noted that firstly in cv.dnn.blobFromImage mean value is subtracted and only then pixel values are multiplied by scale. Thus, mean is multiplied by 255.0 to reproduce the original image preprocessing order:

python
img /= 255.0
img -= [0.485, 0.456, 0.406]
img /= [0.229, 0.224, 0.225]
  • OpenCV cv.dnn.Net inference:
python
# set OpenCV DNN input
opencv_net.setInput(preproc_img)

# OpenCV DNN inference
out = opencv_net.forward()
print("OpenCV DNN prediction: \n")
print("* shape: ", out.shape)

# get the predicted class ID
imagenet_class_id = np.argmax(out)

# get confidence
confidence = out[0][imagenet_class_id]
print("* class ID: {}, label: {}".format(imagenet_class_id, imagenet_labels[imagenet_class_id]))
print("* confidence: {:.4f}".format(confidence))

After the above code execution we will get the following output:

console
OpenCV DNN prediction:
* shape:  (1, 1000)
* class ID: 335, label: fox squirrel, eastern fox squirrel, Sciurus niger
* confidence: 14.8308
  • PyTorch ResNet-50 model inference:
python
original_net.eval()
preproc_img = torch.FloatTensor(preproc_img)

# inference
out = original_net(preproc_img)
print("\nPyTorch model prediction: \n")
print("* shape: ", out.shape)

# get the predicted class ID
imagenet_class_id = torch.argmax(out, axis=1).item()
print("* class ID: {}, label: {}".format(imagenet_class_id, imagenet_labels[imagenet_class_id]))

# get confidence
confidence = out[0][imagenet_class_id]
print("* confidence: {:.4f}".format(confidence.item()))

After the above code launching we will get the following output:

console
PyTorch model prediction:
* shape:  torch.Size([1, 1000])
* class ID: 335, label: fox squirrel, eastern fox squirrel, Sciurus niger
* confidence: 14.8308

The inference results of the original ResNet-50 model and cv.dnn.Net are equal. For the extended evaluation of the models we can use py_to_py_cls of the dnn_model_runner module. This module part will be described in the next subchapter.

Evaluation of the Models

The proposed in samples/dnn dnn_model_runner module allows to run the full evaluation pipeline on the ImageNet dataset and test execution for the following PyTorch classification models:

  • alexnet
  • vgg11
  • vgg13
  • vgg16
  • vgg19
  • resnet18
  • resnet34
  • resnet50
  • resnet101
  • resnet152
  • squeezenet1_0
  • squeezenet1_1
  • resnext50_32x4d
  • resnext101_32x8d
  • wide_resnet50_2
  • wide_resnet101_2

This list can be also extended with further appropriate evaluation pipeline configuration.

Evaluation Mode

The below line represents running of the module in the evaluation mode:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name <pytorch_cls_model_name>

Chosen from the list classification model will be read into OpenCV cv.dnn.Net object. Evaluation results of PyTorch and OpenCV models (accuracy, inference time, L1) will be written into the log file. Inference time values will be also depicted in a chart to generalize the obtained model information.

Necessary evaluation configurations are defined in the test_config.py and can be modified in accordance with actual paths of data location:

python
@dataclass
class TestClsConfig:
    batch_size: int = 50
    frame_size: int = 224
    img_root_dir: str = "./ILSVRC2012_img_val"
    # location of image-class matching
    img_cls_file: str = "./val.txt"
    bgr_to_rgb: bool = True

To initiate the evaluation of the PyTorch ResNet-50, run the following line:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name resnet50

After script launch, the log file with evaluation data will be generated in dnn_model_runner/dnn_conversion/logs:

console
The model PyTorch resnet50 was successfully obtained and converted to OpenCV DNN resnet50
===== Running evaluation of the model with the following params:
    * val data location: ./ILSVRC2012_img_val
    * log file location: dnn_model_runner/dnn_conversion/logs/PyTorch_resnet50_log.txt

Test Mode

The below line represents running of the module in the test mode, namely it provides the steps for the model inference:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name <pytorch_cls_model_name> --test True --default_img_preprocess <True/False> --evaluate False

Here default_img_preprocess key defines whether you'd like to parametrize the model test process with some particular values or use the default values, for example, scale, mean or std.

Test configuration is represented in test_config.py TestClsModuleConfig class:

python
@dataclass
class TestClsModuleConfig:
    cls_test_data_dir: str = "../data"
    test_module_name: str = "classification"
    test_module_path: str = "classification.py"
    input_img: str = os.path.join(cls_test_data_dir, "squirrel_cls.jpg")
    model: str = ""

    frame_height: str = str(TestClsConfig.frame_size)
    frame_width: str = str(TestClsConfig.frame_size)
    scale: str = "1.0"
    mean: List[str] = field(default_factory=lambda: ["0.0", "0.0", "0.0"])
    std: List[str] = field(default_factory=list)
    crop: str = "False"
    rgb: str = "True"
    rsz_height: str = ""
    rsz_width: str = ""
    classes: str = os.path.join(cls_test_data_dir, "dnn", "classification_classes_ILSVRC2012.txt")

The default image preprocessing options are defined in default_preprocess_config.py. For instance:

python
BASE_IMG_SCALE_FACTOR = 1 / 255.0
PYTORCH_RSZ_HEIGHT = 256
PYTORCH_RSZ_WIDTH = 256

pytorch_resize_input_blob = {
    "mean": ["123.675", "116.28", "103.53"],
    "scale": str(BASE_IMG_SCALE_FACTOR),
    "std": ["0.229", "0.224", "0.225"],
    "crop": "True",
    "rgb": "True",
    "rsz_height": str(PYTORCH_RSZ_HEIGHT),
    "rsz_width": str(PYTORCH_RSZ_WIDTH)
}

The basis of the model testing is represented in samples/dnn/classification.py. classification.py can be executed autonomously with provided converted model in --input and populated parameters for cv.dnn.blobFromImage.

To reproduce from scratch the described in "Model Conversion Pipeline" OpenCV steps with dnn_model_runner execute the below line:

console
python -m dnn_model_runner.dnn_conversion.pytorch.classification.py_to_py_cls --model_name resnet50 --test True --default_img_preprocess True --evaluate False

The network prediction is depicted in the top left corner of the output window:

Doc Tutorials Dnn Dnn Pytorch Tf Classification Tf Cls Model Conversion Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_classification/tf_cls_model_conversion_tutorial.md`)

Conversion of TensorFlow Classification Models and Launch with OpenCV Python {#tf_cls_tutorial_dnn_conversion}
Original author Anastasia Murzova
Compatibility OpenCV >= 4.5

Goals

In this tutorial you will learn how to:

  • obtain frozen graphs of TensorFlow (TF) classification models
  • run converted TensorFlow model with OpenCV Python API
  • obtain an evaluation of the TensorFlow and OpenCV DNN models

We will explore the above-listed points by the example of MobileNet architecture.

Introduction

Let's briefly view the key concepts involved in the pipeline of TensorFlow models transition with OpenCV API. The initial step in conversion of TensorFlow models into cv.dnn.Net
is obtaining the frozen TF model graph. Frozen graph defines the combination of the model graph structure with kept values of the required variables, for example, weights. Usually the frozen graph is saved in protobuf (.pb) files.
After the model .pb file was generated it can be read with cv.dnn.readNetFromTensorflow function.

Requirements

To be able to experiment with the below code you will need to install a set of libraries. We will use a virtual environment with python3.7+ for this:

console
virtualenv -p /usr/bin/python3.7 <env_dir_path>
source <env_dir_path>/bin/activate

For OpenCV-Python building from source, follow the corresponding instructions from the @ref tutorial_py_table_of_contents_setup.

Before you start the installation of the libraries, you can customize the requirements.txt, excluding or including (for example, opencv-python) some dependencies.
The below line initiates requirements installation into the previously activated virtual environment:

console
pip install -r requirements.txt

Practice

In this part we are going to cover the following points:

  1. create a TF classification model conversion pipeline and provide the inference
  2. evaluate and test TF classification models

If you'd like merely to run evaluation or test model pipelines, the "Model Conversion Pipeline" tutorial part can be skipped.

Model Conversion Pipeline

The code in this subchapter is located in the dnn_model_runner module and can be executed with the line:

console
python -m dnn_model_runner.dnn_conversion.tf.classification.py_to_py_mobilenet

The following code contains the description of the below-listed steps:

  1. instantiate TF model
  2. create TF frozen graph
  3. read TF frozen graph with OpenCV API
  4. prepare input data
  5. provide inference
python
# initialize TF MobileNet model
original_tf_model = MobileNet(
    include_top=True,
    weights="imagenet"
)

# get TF frozen graph path
full_pb_path = get_tf_model_proto(original_tf_model)

# read frozen graph with OpenCV API
opencv_net = cv2.dnn.readNetFromTensorflow(full_pb_path)
print("OpenCV model was successfully read. Model layers: \n", opencv_net.getLayerNames())

# get preprocessed image
input_img = get_preprocessed_img("../data/squirrel_cls.jpg")

# get ImageNet labels
imagenet_labels = get_imagenet_labels("../data/dnn/classification_classes_ILSVRC2012.txt")

# obtain OpenCV DNN predictions
get_opencv_dnn_prediction(opencv_net, input_img, imagenet_labels)

# obtain TF model predictions
get_tf_dnn_prediction(original_tf_model, input_img, imagenet_labels)

To provide model inference we will use the below squirrel photo (under CC0 license) corresponding to ImageNet class ID 335:

console
fox squirrel, eastern fox squirrel, Sciurus niger

For the label decoding of the obtained prediction, we also need imagenet_classes.txt file, which contains the full list of the ImageNet classes.

Let's go deeper into each step by the example of pretrained TF MobileNet:

  • instantiate TF model:
python
# initialize TF MobileNet model
original_tf_model = MobileNet(
    include_top=True,
    weights="imagenet"
)
  • create TF frozen graph
python
# define the directory for .pb model
pb_model_path = "models"

# define the name of .pb model
pb_model_name = "mobilenet.pb"

# create directory for further converted model
os.makedirs(pb_model_path, exist_ok=True)

# get model TF graph
tf_model_graph = tf.function(lambda x: tf_model(x))

# get concrete function
tf_model_graph = tf_model_graph.get_concrete_function(
    tf.TensorSpec(tf_model.inputs[0].shape, tf_model.inputs[0].dtype))

# obtain frozen concrete function
frozen_tf_func = convert_variables_to_constants_v2(tf_model_graph)
# get frozen graph
frozen_tf_func.graph.as_graph_def()

# save full tf model
tf.io.write_graph(graph_or_graph_def=frozen_tf_func.graph,
                  logdir=pb_model_path,
                  name=pb_model_name,
                  as_text=False)

After the successful execution of the above code, we will get a frozen graph in models/mobilenet.pb.

  • read TF frozen graph with with cv.dnn.readNetFromTensorflow passing the obtained in the previous step mobilenet.pb into it:
python
# get TF frozen graph path
full_pb_path = get_tf_model_proto(original_tf_model)
  • prepare input data with cv2.dnn.blobFromImage function:
python
# read the image
input_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
input_img = input_img.astype(np.float32)

# define preprocess parameters
mean = np.array([1.0, 1.0, 1.0]) * 127.5
scale = 1 / 127.5

# prepare input blob to fit the model input:
# 1. subtract mean
# 2. scale to set pixel values from 0 to 1
input_blob = cv2.dnn.blobFromImage(
    image=input_img,
    scalefactor=scale,
    size=(224, 224),  # img target size
    mean=mean,
    swapRB=True,  # BGR -> RGB
    crop=True  # center crop
)
print("Input blob shape: {}\n".format(input_blob.shape))

Please, pay attention at the preprocessing order in the cv2.dnn.blobFromImage function. Firstly, the mean value is subtracted and only then pixel values are multiplied by the defined scale.
Therefore, to reproduce the image preprocessing pipeline from the TF mobilenet.preprocess_input function, we multiply mean by 127.5.

As a result, 4-dimensional input_blob was obtained:

Input blob shape: (1, 3, 224, 224)

  • provide OpenCV cv.dnn.Net inference:
python
# set OpenCV DNN input
opencv_net.setInput(preproc_img)

# OpenCV DNN inference
out = opencv_net.forward()
print("OpenCV DNN prediction: \n")
print("* shape: ", out.shape)

# get the predicted class ID
imagenet_class_id = np.argmax(out)

# get confidence
confidence = out[0][imagenet_class_id]
print("* class ID: {}, label: {}".format(imagenet_class_id, imagenet_labels[imagenet_class_id]))
print("* confidence: {:.4f}\n".format(confidence))

After the above code execution we will get the following output:

console
OpenCV DNN prediction:
* shape:  (1, 1000)
* class ID: 335, label: fox squirrel, eastern fox squirrel, Sciurus niger
* confidence: 0.9525
  • provide TF MobileNet inference:
python
# inference
preproc_img = preproc_img.transpose(0, 2, 3, 1)
print("TF input blob shape: {}\n".format(preproc_img.shape))

out = original_net(preproc_img)

print("\nTensorFlow model prediction: \n")
print("* shape: ", out.shape)

# get the predicted class ID
imagenet_class_id = np.argmax(out)
print("* class ID: {}, label: {}".format(imagenet_class_id, imagenet_labels[imagenet_class_id]))

# get confidence
confidence = out[0][imagenet_class_id]
print("* confidence: {:.4f}".format(confidence))

To fit TF model input, input_blob was transposed:

console
TF input blob shape: (1, 224, 224, 3)

TF inference results are the following:

console
TensorFlow model prediction:
* shape:  (1, 1000)
* class ID: 335, label: fox squirrel, eastern fox squirrel, Sciurus niger
* confidence: 0.9525

As it can be seen from the experiments OpenCV and TF inference results are equal.

Evaluation of the Models

The proposed in dnn/samples dnn_model_runner module allows to run the full evaluation pipeline on the ImageNet dataset and test execution for the following TensorFlow classification models:

  • vgg16
  • vgg19
  • resnet50
  • resnet101
  • resnet152
  • densenet121
  • densenet169
  • densenet201
  • inceptionresnetv2
  • inceptionv3
  • mobilenet
  • mobilenetv2
  • nasnetlarge
  • nasnetmobile
  • xception

This list can be also extended with further appropriate evaluation pipeline configuration.

Evaluation Mode

To below line represents running of the module in the evaluation mode:

console
python -m dnn_model_runner.dnn_conversion.tf.classification.py_to_py_cls --model_name <tf_cls_model_name>

Chosen from the list classification model will be read into OpenCV cv.dnn_Net object. Evaluation results of TF and OpenCV models (accuracy, inference time, L1) will be written into the log file. Inference time values will be also depicted in a chart to generalize the obtained model information.

Necessary evaluation configurations are defined in the test_config.py and can be modified in accordance with actual paths of data location::

python
@dataclass
class TestClsConfig:
    batch_size: int = 50
    frame_size: int = 224
    img_root_dir: str = "./ILSVRC2012_img_val"
    # location of image-class matching
    img_cls_file: str = "./val.txt"
    bgr_to_rgb: bool = True

The values from TestClsConfig can be customized in accordance with chosen model.

To initiate the evaluation of the TensorFlow MobileNet, run the following line:

console
python -m dnn_model_runner.dnn_conversion.tf.classification.py_to_py_cls --model_name mobilenet

After script launch, the log file with evaluation data will be generated in dnn_model_runner/dnn_conversion/logs:

console
===== Running evaluation of the model with the following params:
    * val data location: ./ILSVRC2012_img_val
    * log file location: dnn_model_runner/dnn_conversion/logs/TF_mobilenet_log.txt

Test Mode

The below line represents running of the module in the test mode, namely it provides the steps for the model inference:

console
python -m dnn_model_runner.dnn_conversion.tf.classification.py_to_py_cls --model_name <tf_cls_model_name> --test True --default_img_preprocess <True/False> --evaluate False

Here default_img_preprocess key defines whether you'd like to parametrize the model test process with some particular values or use the default values, for example, scale, mean or std.

Test configuration is represented in test_config.py TestClsModuleConfig class:

python
@dataclass
class TestClsModuleConfig:
    cls_test_data_dir: str = "../data"
    test_module_name: str = "classification"
    test_module_path: str = "classification.py"
    input_img: str = os.path.join(cls_test_data_dir, "squirrel_cls.jpg")
    model: str = ""

    frame_height: str = str(TestClsConfig.frame_size)
    frame_width: str = str(TestClsConfig.frame_size)
    scale: str = "1.0"
    mean: List[str] = field(default_factory=lambda: ["0.0", "0.0", "0.0"])
    std: List[str] = field(default_factory=list)
    crop: str = "False"
    rgb: str = "True"
    rsz_height: str = ""
    rsz_width: str = ""
    classes: str = os.path.join(cls_test_data_dir, "dnn", "classification_classes_ILSVRC2012.txt")

The default image preprocessing options are defined in default_preprocess_config.py. For instance, for MobileNet:

python
tf_input_blob = {
    "mean": ["127.5", "127.5", "127.5"],
    "scale": str(1 / 127.5),
    "std": [],
    "crop": "True",
    "rgb": "True"
}

The basis of the model testing is represented in samples/dnn/classification.py. classification.py can be executed autonomously with provided converted model in --input and populated parameters for cv.dnn.blobFromImage.

To reproduce from scratch the described in "Model Conversion Pipeline" OpenCV steps with dnn_model_runner execute the below line:

console
python -m dnn_model_runner.dnn_conversion.tf.classification.py_to_py_cls --model_name mobilenet --test True --default_img_preprocess True --evaluate False

The network prediction is depicted in the top left corner of the output window:

Doc Tutorials Dnn Dnn Pytorch Tf Detection Tf Det Model Conversion Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_detection/tf_det_model_conversion_tutorial.md`)

Conversion of TensorFlow Detection Models and Launch with OpenCV Python {#tf_det_tutorial_dnn_conversion}
Original author Anastasia Murzova
Compatibility OpenCV >= 4.5

Goals

In this tutorial you will learn how to:

  • obtain frozen graphs of TensorFlow (TF) detection models
  • run converted TensorFlow model with OpenCV Python API

We will explore the above-listed points by the example of SSD MobileNetV1.

Introduction

Let's briefly view the key concepts involved in the pipeline of TensorFlow models transition with OpenCV API. The initial step in the conversion of TensorFlow models into cv.dnn.Net
is obtaining the frozen TF model graph. A frozen graph defines the combination of the model graph structure with kept values of the required variables, for example, weights. The frozen graph is saved in protobuf (.pb) files.
There are special functions for reading .pb graphs in OpenCV: cv.dnn.readNetFromTensorflow and cv.dnn.readNet.

Requirements

To be able to experiment with the below code you will need to install a set of libraries. We will use a virtual environment with python3.7+ for this:

console
virtualenv -p /usr/bin/python3.7 <env_dir_path>
source <env_dir_path>/bin/activate

For OpenCV-Python building from source, follow the corresponding instructions from the @ref tutorial_py_table_of_contents_setup.

Before you start the installation of the libraries, you can customize the requirements.txt, excluding or including (for example, opencv-python) some dependencies.
The below line initiates requirements installation into the previously activated virtual environment:

console
pip install -r requirements.txt

Practice

In this part we are going to cover the following points:

  1. create a TF classification model conversion pipeline and provide the inference
  2. provide the inference, process prediction results

Model Preparation

The code in this subchapter is located in the samples/dnn/dnn_model_runner module and can be executed with the below line:

console
python -m dnn_model_runner.dnn_conversion.tf.detection.py_to_py_ssd_mobilenet

The following code contains the steps of the TF SSD MobileNetV1 model retrieval:

python
tf_model_name = 'ssd_mobilenet_v1_coco_2017_11_17'
    graph_extraction_dir = "./"
    frozen_graph_path = extract_tf_frozen_graph(tf_model_name, graph_extraction_dir)
    print("Frozen graph path for {}: {}".format(tf_model_name, frozen_graph_path))

In extract_tf_frozen_graph function we extract the provided in model archive frozen_inference_graph.pb for its further processing:

python
# define model archive name
tf_model_tar = model_name + '.tar.gz'
# define link to retrieve model archive
model_link = DETECTION_MODELS_URL + tf_model_tar

tf_frozen_graph_name = 'frozen_inference_graph'

try:
    urllib.request.urlretrieve(model_link, tf_model_tar)
except Exception:
    print("TF {} was not retrieved: {}".format(model_name, model_link))
    return

print("TF {} was retrieved.".format(model_name))

tf_model_tar = tarfile.open(tf_model_tar)
frozen_graph_path = ""

for model_tar_elem in tf_model_tar.getmembers():
    if tf_frozen_graph_name in os.path.basename(model_tar_elem.name):
        tf_model_tar.extract(model_tar_elem, extracted_model_path)
        frozen_graph_path = os.path.join(extracted_model_path, model_tar_elem.name)
        break
tf_model_tar.close()

After the successful execution of the above code we will get the following output:

console
TF ssd_mobilenet_v1_coco_2017_11_17 was retrieved.
Frozen graph path for ssd_mobilenet_v1_coco_2017_11_17: ./ssd_mobilenet_v1_coco_2017_11_17/frozen_inference_graph.pb

To provide model inference we will use the below double-decker bus photo (under Pexels license):

To initiate the test process we need to provide an appropriate model configuration. We will use ssd_mobilenet_v1_coco.config from TensorFlow Object Detection API.
TensorFlow Object Detection API framework contains helpful mechanisms for object detection model manipulations.

We will use this configuration to provide a text graph representation. To generate .pbtxt we will use the corresponding samples/dnn/tf_text_graph_ssd.py script:

console
python tf_text_graph_ssd.py --input ssd_mobilenet_v1_coco_2017_11_17/frozen_inference_graph.pb --config ssd_mobilenet_v1_coco_2017_11_17/ssd_mobilenet_v1_coco.config --output ssd_mobilenet_v1_coco_2017_11_17.pbtxt

After successful execution ssd_mobilenet_v1_coco_2017_11_17.pbtxt will be created.

Before we run object_detection.py, let's have a look at the default values for the SSD MobileNetV1 test process configuration. They are located in models.yml:

yml
ssd_tf:
  model: "ssd_mobilenet_v1_coco_2017_11_17.pb"
  config: "ssd_mobilenet_v1_coco_2017_11_17.pbtxt"
  mean: [0, 0, 0]
  scale: 1.0
  width: 300
  height: 300
  rgb: true
  classes: "object_detection_classes_coco.txt"
  sample: "object_detection"

To fetch these values we need to provide frozen graph ssd_mobilenet_v1_coco_2017_11_17.pb model and text graph ssd_mobilenet_v1_coco_2017_11_17.pbtxt:

console
python object_detection.py ssd_tf --input ../data/pexels_double_decker_bus.jpg

This line is equivalent to:

console
python object_detection.py --model ssd_mobilenet_v1_coco_2017_11_17.pb --config  ssd_mobilenet_v1_coco_2017_11_17.pbtxt  --input ../data/pexels_double_decker_bus.jpg --width 300 --height 300 --classes ../data/dnn/object_detection_classes_coco.txt

The result is:

There are several helpful parameters, which can be also customized for result corrections: threshold (--thr) and non-maximum suppression (--nms) values.

Doc Tutorials Dnn Dnn Pytorch Tf Segmentation Pytorch Sem Segm Model Conversion Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_segmentation/pytorch_sem_segm_model_conversion_tutorial.md`)

Conversion of PyTorch Segmentation Models and Launch with OpenCV {#pytorch_segm_tutorial_dnn_conversion}

Goals

In this tutorial you will learn how to:

  • convert PyTorch segmentation models
  • run converted PyTorch model with OpenCV
  • obtain an evaluation of the PyTorch and OpenCV DNN models

We will explore the above-listed points by the example of the FCN ResNet-50 architecture.

Introduction

The key points involved in the transition pipeline of the PyTorch classification and segmentation models with OpenCV API are equal. The first step is model transferring into ONNX format with PyTorch torch.onnx.export built-in function.
Further the obtained .onnx model is passed into cv.dnn.readNetFromONNX, which returns cv.dnn.Net object ready for DNN manipulations.

Practice

In this part we are going to cover the following points:

  1. create a segmentation model conversion pipeline and provide the inference
  2. evaluate and test segmentation models

If you'd like merely to run evaluation or test model pipelines, the "Model Conversion Pipeline" part can be skipped.

Model Conversion Pipeline

The code in this subchapter is located in the dnn_model_runner module and can be executed with the line:

python -m dnn_model_runner.dnn_conversion.pytorch.segmentation.py_to_py_fcnresnet50

The following code contains the description of the below-listed steps:

  1. instantiate PyTorch model
  2. convert PyTorch model into .onnx
  3. read the transferred network with OpenCV API
  4. prepare input data
  5. provide inference
  6. get colored masks from predictions
  7. visualize results
python
# initialize PyTorch FCN ResNet-50 model
original_model = models.segmentation.fcn_resnet50(pretrained=True)

# get the path to the converted into ONNX PyTorch model
full_model_path = get_pytorch_onnx_model(original_model)

# read converted .onnx model with OpenCV API
opencv_net = cv2.dnn.readNetFromONNX(full_model_path)
print("OpenCV model was successfully read. Layer IDs: \n", opencv_net.getLayerNames())

# get preprocessed image
img, input_img = get_processed_imgs("test_data/sem_segm/2007_000033.jpg")

# obtain OpenCV DNN predictions
opencv_prediction = get_opencv_dnn_prediction(opencv_net, input_img)

# obtain original PyTorch ResNet50 predictions
pytorch_prediction = get_pytorch_dnn_prediction(original_model, input_img)

pascal_voc_classes, pascal_voc_colors = read_colors_info("test_data/sem_segm/pascal-classes.txt")

# obtain colored segmentation masks
opencv_colored_mask = get_colored_mask(img.shape, opencv_prediction, pascal_voc_colors)
pytorch_colored_mask = get_colored_mask(img.shape, pytorch_prediction, pascal_voc_colors)

# obtain palette of PASCAL VOC colors
color_legend = get_legend(pascal_voc_classes, pascal_voc_colors)

cv2.imshow('PyTorch Colored Mask', pytorch_colored_mask)
cv2.imshow('OpenCV DNN Colored Mask', opencv_colored_mask)
cv2.imshow('Color Legend', color_legend)

cv2.waitKey(0)

To provide the model inference we will use the below picture from the PASCAL VOC validation dataset:

The target segmented result is:

For the PASCAL VOC colors decoding and its mapping with the predicted masks, we also need pascal-classes.txt file, which contains the full list of the PASCAL VOC classes and corresponding colors.

Let's go deeper into each code step by the example of pretrained PyTorch FCN ResNet-50:

  • instantiate PyTorch FCN ResNet-50 model:
python
# initialize PyTorch FCN ResNet-50 model
original_model = models.segmentation.fcn_resnet50(pretrained=True)
  • convert PyTorch model into ONNX format:
python
# define the directory for further converted model save
onnx_model_path = "models"
# define the name of further converted model
onnx_model_name = "fcnresnet50.onnx"

# create directory for further converted model
os.makedirs(onnx_model_path, exist_ok=True)

# get full path to the converted model
full_model_path = os.path.join(onnx_model_path, onnx_model_name)

# generate model input to build the graph
generated_input = Variable(
    torch.randn(1, 3, 500, 500)
)

# model export into ONNX format
torch.onnx.export(
    original_model,
    generated_input,
    full_model_path,
    verbose=True,
    input_names=["input"],
    output_names=["output"],
    opset_version=11
)

The code from this step does not differ from the classification conversion case. Thus, after the successful execution of the above code, we will get models/fcnresnet50.onnx.

  • read the transferred network with cv.dnn.readNetFromONNX passing the obtained in the previous step ONNX model into it:
python
# read converted .onnx model with OpenCV API
opencv_net = cv2.dnn.readNetFromONNX(full_model_path)
  • prepare input data:
python
# read the image
input_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
input_img = input_img.astype(np.float32)

# target image sizes
img_height = input_img.shape[0]
img_width = input_img.shape[1]

# define preprocess parameters
mean = np.array([0.485, 0.456, 0.406]) * 255.0
scale = 1 / 255.0
std = [0.229, 0.224, 0.225]

# prepare input blob to fit the model input:
# 1. subtract mean
# 2. scale to set pixel values from 0 to 1
input_blob = cv2.dnn.blobFromImage(
    image=input_img,
    scalefactor=scale,
    size=(img_width, img_height),  # img target size
    mean=mean,
    swapRB=True,  # BGR -> RGB
    crop=False  # center crop
)
# 3. divide by std
input_blob[0] /= np.asarray(std, dtype=np.float32).reshape(3, 1, 1)

In this step we read the image and prepare model input with cv2.dnn.blobFromImage function, which returns 4-dimensional blob.
It should be noted that firstly in cv2.dnn.blobFromImage mean value is subtracted and only then pixel values are scaled. Thus, mean is multiplied by 255.0 to reproduce the original image preprocessing order:

python
img /= 255.0
img -= [0.485, 0.456, 0.406]
img /= [0.229, 0.224, 0.225]
  • OpenCV cv.dnn_Net inference:
python
# set OpenCV DNN input
opencv_net.setInput(preproc_img)

# OpenCV DNN inference
out = opencv_net.forward()
print("OpenCV DNN segmentation prediction: \n")
print("* shape: ", out.shape)

# get IDs of predicted classes
out_predictions = np.argmax(out[0], axis=0)

After the above code execution we will get the following output:

text
OpenCV DNN segmentation prediction:
* shape:  (1, 21, 500, 500)

Each prediction channel out of 21, where 21 represents the number of PASCAL VOC classes, contains probabilities, which indicate how likely the pixel corresponds to the PASCAL VOC class.

  • PyTorch FCN ResNet-50 model inference:
python
original_net.eval()
preproc_img = torch.FloatTensor(preproc_img)

with torch.no_grad():
    # obtaining unnormalized probabilities for each class
    out = original_net(preproc_img)['out']

print("\nPyTorch segmentation model prediction: \n")
print("* shape: ", out.shape)

# get IDs of predicted classes
out_predictions = out[0].argmax(dim=0)

After the above code launching we will get the following output:

text
PyTorch segmentation model prediction:
* shape:  torch.Size([1, 21, 366, 500])

PyTorch prediction also contains probabilities corresponding to each class prediction.

  • get colored masks from predictions:
python
# convert mask values into PASCAL VOC colors
processed_mask = np.stack([colors[color_id] for color_id in segm_mask.flatten()])

# reshape mask into 3-channel image
processed_mask = processed_mask.reshape(mask_height, mask_width, 3)
processed_mask = cv2.resize(processed_mask, (img_width, img_height), interpolation=cv2.INTER_NEAREST).astype(
    np.uint8)

# convert colored mask from BGR to RGB for compatibility with PASCAL VOC colors
processed_mask = cv2.cvtColor(processed_mask, cv2.COLOR_BGR2RGB)

In this step we map the probabilities from segmentation masks with appropriate colors of the predicted classes. Let's have a look at the results:

For the extended evaluation of the models, we can use py_to_py_segm script of the dnn_model_runner module. This module part will be described in the next subchapter.

Evaluation of the Models

The proposed in dnn/samples dnn_model_runner module allows to run the full evaluation pipeline on the PASCAL VOC dataset and test execution for the following PyTorch segmentation models:

  • FCN ResNet-50
  • FCN ResNet-101

This list can be also extended with further appropriate evaluation pipeline configuration.

Evaluation Mode

The below line represents running of the module in the evaluation mode:

text
python -m dnn_model_runner.dnn_conversion.pytorch.segmentation.py_to_py_segm --model_name <pytorch_segm_model_name>

Chosen from the list segmentation model will be read into OpenCV cv.dnn_Net object. Evaluation results of PyTorch and OpenCV models (pixel accuracy, mean IoU, inference time) will be written into the log file. Inference time values will be also depicted in a chart to generalize the obtained model information.

Necessary evaluation configurations are defined in the test_config.py:

python
@dataclass
class TestSegmConfig:
    frame_size: int = 500
    img_root_dir: str = "./VOC2012"
    img_dir: str = os.path.join(img_root_dir, "JPEGImages/")
    img_segm_gt_dir: str = os.path.join(img_root_dir, "SegmentationClass/")
    # reduced val: https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/data/pascal/seg11valid.txt
    segm_val_file: str = os.path.join(img_root_dir, "ImageSets/Segmentation/seg11valid.txt")
    colour_file_cls: str = os.path.join(img_root_dir, "ImageSets/Segmentation/pascal-classes.txt")

These values can be modified in accordance with chosen model pipeline.

To initiate the evaluation of the PyTorch FCN ResNet-50, run the following line:

text
python -m dnn_model_runner.dnn_conversion.pytorch.segmentation.py_to_py_segm --model_name fcnresnet50

Test Mode

The below line represents running of the module in the test mode, which provides the steps for the model inference:

text
python -m dnn_model_runner.dnn_conversion.pytorch.segmentation.py_to_py_segm --model_name <pytorch_segm_model_name> --test True --default_img_preprocess <True/False> --evaluate False

Here default_img_preprocess key defines whether you'd like to parametrize the model test process with some particular values or use the default values, for example, scale, mean or std.

Test configuration is represented in test_config.py TestSegmModuleConfig class:

python
@dataclass
class TestSegmModuleConfig:
    segm_test_data_dir: str = "test_data/sem_segm"
    test_module_name: str = "segmentation"
    test_module_path: str = "segmentation.py"
    input_img: str = os.path.join(segm_test_data_dir, "2007_000033.jpg")
    model: str = ""

    frame_height: str = str(TestSegmConfig.frame_size)
    frame_width: str = str(TestSegmConfig.frame_size)
    scale: float = 1.0
    mean: List[float] = field(default_factory=lambda: [0.0, 0.0, 0.0])
    std: List[float] = field(default_factory=list)
    crop: bool = False
    rgb: bool = True
    classes: str = os.path.join(segm_test_data_dir, "pascal-classes.txt")

The default image preprocessing options are defined in default_preprocess_config.py:

python
pytorch_segm_input_blob = {
    "mean": ["123.675", "116.28", "103.53"],
    "scale": str(1 / 255.0),
    "std": ["0.229", "0.224", "0.225"],
    "crop": "False",
    "rgb": "True"
}

The basis of the model testing is represented in samples/dnn/segmentation.py. segmentation.py can be executed autonomously with provided converted model in --input and populated parameters for cv2.dnn.blobFromImage.

To reproduce from scratch the described in "Model Conversion Pipeline" OpenCV steps with dnn_model_runner execute the below line:

text
python -m dnn_model_runner.dnn_conversion.pytorch.segmentation.py_to_py_segm --model_name fcnresnet50 --test True --default_img_preprocess True --evaluate False

Doc Tutorials Dnn Dnn Pytorch Tf Segmentation Tf Sem Segm Model Conversion Tutorial (`doc/tutorials/dnn/dnn_pytorch_tf_segmentation/tf_sem_segm_model_conversion_tutorial.md`)

Conversion of TensorFlow Segmentation Models and Launch with OpenCV {#tf_segm_tutorial_dnn_conversion}

Goals

In this tutorial you will learn how to:

  • convert TensorFlow (TF) segmentation models
  • run converted TensorFlow model with OpenCV
  • obtain an evaluation of the TensorFlow and OpenCV DNN models

We will explore the above-listed points by the example of the DeepLab architecture.

Introduction

The key concepts involved in the transition pipeline of the TensorFlow classification and segmentation models with OpenCV API are almost equal excepting the phase of graph optimization. The initial step in conversion of TensorFlow models into cv.dnn.Net
is obtaining the frozen TF model graph. Frozen graph defines the combination of the model graph structure with kept values of the required variables, for example, weights. Usually the frozen graph is saved in protobuf (.pb) files.
To read the generated segmentation model .pb file with cv.dnn.readNetFromTensorflow, it is needed to modify the graph with TF graph transform tool.

Practice

In this part we are going to cover the following points:

  1. create a TF classification model conversion pipeline and provide the inference
  2. evaluate and test TF classification models

If you'd like merely to run evaluation or test model pipelines, the "Model Conversion Pipeline" tutorial part can be skipped.

Model Conversion Pipeline

The code in this subchapter is located in the dnn_model_runner module and can be executed with the line:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.py_to_py_deeplab

TensorFlow segmentation models can be found in TensorFlow Research Models section, which contains the implementations of models on the basis of published research papers.
We will retrieve the archive with the pre-trained TF DeepLabV3 from the below link:

text
http://download.tensorflow.org/models/deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz

The full frozen graph obtaining pipeline is described in deeplab_retrievement.py:

python
def get_deeplab_frozen_graph():
    # define model path to download
    models_url = 'http://download.tensorflow.org/models/'
    mobilenetv2_voctrainval = 'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz'

    # construct model link to download
    model_link = models_url + mobilenetv2_voctrainval

    try:
        urllib.request.urlretrieve(model_link, mobilenetv2_voctrainval)
    except Exception:
        print("TF DeepLabV3 was not retrieved: {}".format(model_link))
        return

    tf_model_tar = tarfile.open(mobilenetv2_voctrainval)

    # iterate the obtained model archive
    for model_tar_elem in tf_model_tar.getmembers():
        # check whether the model archive contains frozen graph
        if TF_FROZEN_GRAPH_NAME in os.path.basename(model_tar_elem.name):
            # extract frozen graph
            tf_model_tar.extract(model_tar_elem, FROZEN_GRAPH_PATH)

    tf_model_tar.close()

After running this script:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.deeplab_retrievement

we will get frozen_inference_graph.pb in deeplab/deeplabv3_mnv2_pascal_trainval.

Before going to the network loading with OpenCV it is needed to optimize the extracted frozen_inference_graph.pb.
To optimize the graph we use TF TransformGraph with default parameters:

python
DEFAULT_OPT_GRAPH_NAME = "optimized_frozen_inference_graph.pb"
DEFAULT_INPUTS = "sub_7"
DEFAULT_OUTPUTS = "ResizeBilinear_3"
DEFAULT_TRANSFORMS = "remove_nodes(op=Identity)" \
                     " merge_duplicate_nodes" \
                     " strip_unused_nodes" \
                     " fold_constants(ignore_errors=true)" \
                     " fold_batch_norms" \
                     " fold_old_batch_norms"


def optimize_tf_graph(
        in_graph,
        out_graph=DEFAULT_OPT_GRAPH_NAME,
        inputs=DEFAULT_INPUTS,
        outputs=DEFAULT_OUTPUTS,
        transforms=DEFAULT_TRANSFORMS,
        is_manual=True,
        was_optimized=True
):
    # ...

    tf_opt_graph = TransformGraph(
        tf_graph,
        inputs,
        outputs,
        transforms
    )

To run graph optimization process, execute the line:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.tf_graph_optimizer --in_graph deeplab/deeplabv3_mnv2_pascal_trainval/frozen_inference_graph.pb

As a result deeplab/deeplabv3_mnv2_pascal_trainval directory will contain optimized_frozen_inference_graph.pb.

After we have obtained the model graphs, let's examine the below-listed steps:

  1. read TF frozen_inference_graph.pb graph
  2. read optimized TF frozen graph with OpenCV API
  3. prepare input data
  4. provide inference
  5. get colored masks from predictions
  6. visualize results
python
# get TF model graph from the obtained frozen graph
deeplab_graph = read_deeplab_frozen_graph(deeplab_frozen_graph_path)

# read DeepLab frozen graph with OpenCV API
opencv_net = cv2.dnn.readNetFromTensorflow(opt_deeplab_frozen_graph_path)
print("OpenCV model was successfully read. Model layers: \n", opencv_net.getLayerNames())

# get processed image
original_img_shape, tf_input_blob, opencv_input_img = get_processed_imgs("test_data/sem_segm/2007_000033.jpg")

# obtain OpenCV DNN predictions
opencv_prediction = get_opencv_dnn_prediction(opencv_net, opencv_input_img)

# obtain TF model predictions
tf_prediction = get_tf_dnn_prediction(deeplab_graph, tf_input_blob)

# get PASCAL VOC classes and colors
pascal_voc_classes, pascal_voc_colors = read_colors_info("test_data/sem_segm/pascal-classes.txt")

# obtain colored segmentation masks
opencv_colored_mask = get_colored_mask(original_img_shape, opencv_prediction, pascal_voc_colors)
tf_colored_mask = get_tf_colored_mask(original_img_shape, tf_prediction, pascal_voc_colors)

# obtain palette of PASCAL VOC colors
color_legend = get_legend(pascal_voc_classes, pascal_voc_colors)

cv2.imshow('TensorFlow Colored Mask', tf_colored_mask)
cv2.imshow('OpenCV DNN Colored Mask', opencv_colored_mask)

cv2.imshow('Color Legend', color_legend)

To provide the model inference we will use the below picture from the PASCAL VOC validation dataset:

The target segmented result is:

For the PASCAL VOC colors decoding and its mapping with the predicted masks, we also need pascal-classes.txt file, which contains the full list of the PASCAL VOC classes and corresponding colors.

Let's go deeper into each step by the example of pretrained TF DeepLabV3 MobileNetV2:

  • read TF frozen_inference_graph.pb graph :
python
# init deeplab model graph
model_graph = tf.Graph()

# obtain
with tf.io.gfile.GFile(frozen_graph_path, 'rb') as graph_file:
    tf_model_graph = GraphDef()
tf_model_graph.ParseFromString(graph_file.read())

with model_graph.as_default():
    tf.import_graph_def(tf_model_graph, name='')
  • read optimized TF frozen graph with OpenCV API:
python
# read DeepLab frozen graph with OpenCV API
opencv_net = cv2.dnn.readNetFromTensorflow(opt_deeplab_frozen_graph_path)
  • prepare input data with cv2.dnn.blobFromImage function:
python
# read the image
input_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
input_img = input_img.astype(np.float32)

# preprocess image for TF model input
tf_preproc_img = cv2.resize(input_img, (513, 513))
tf_preproc_img = cv2.cvtColor(tf_preproc_img, cv2.COLOR_BGR2RGB)

# define preprocess parameters for OpenCV DNN
mean = np.array([1.0, 1.0, 1.0]) * 127.5
scale = 1 / 127.5

# prepare input blob to fit the model input:
# 1. subtract mean
# 2. scale to set pixel values from 0 to 1
input_blob = cv2.dnn.blobFromImage(
    image=input_img,
    scalefactor=scale,
    size=(513, 513),  # img target size
    mean=mean,
    swapRB=True,  # BGR -> RGB
    crop=False  # center crop
)

Please, pay attention at the preprocessing order in the cv2.dnn.blobFromImage function. Firstly, the mean value is subtracted and only then pixel values are multiplied by the defined scale.
Therefore, to reproduce TF image preprocessing pipeline, we multiply mean by 127.5.
Another important point is image preprocessing for TF DeepLab. To pass the image into TF model we need only to construct an appropriate shape, the rest image preprocessing is described in feature_extractor.py and will be invoked automatically.

  • provide OpenCV cv.dnn_Net inference:
python
# set OpenCV DNN input
opencv_net.setInput(preproc_img)

# OpenCV DNN inference
out = opencv_net.forward()
print("OpenCV DNN segmentation prediction: \n")
print("* shape: ", out.shape)

# get IDs of predicted classes
out_predictions = np.argmax(out[0], axis=0)

After the above code execution we will get the following output:

text
OpenCV DNN segmentation prediction:
* shape:  (1, 21, 513, 513)

Each prediction channel out of 21, where 21 represents the number of PASCAL VOC classes, contains probabilities, which indicate how likely the pixel corresponds to the PASCAL VOC class.

  • provide TF model inference:
python
preproc_img = np.expand_dims(preproc_img, 0)

# init TF session
tf_session = Session(graph=model_graph)

input_tensor_name = "ImageTensor:0",
output_tensor_name = "SemanticPredictions:0"

# run inference
out = tf_session.run(
    output_tensor_name,
    feed_dict={input_tensor_name: [preproc_img]}
)

print("TF segmentation model prediction: \n")
print("* shape: ", out.shape)

TF inference results are the following:

text
TF segmentation model prediction:
* shape:  (1, 513, 513)

TensorFlow prediction contains the indexes of corresponding PASCAL VOC classes.

  • transform OpenCV prediction into colored mask:
python
mask_height = segm_mask.shape[0]
mask_width = segm_mask.shape[1]

img_height = original_img_shape[0]
img_width = original_img_shape[1]

# convert mask values into PASCAL VOC colors
processed_mask = np.stack([colors[color_id] for color_id in segm_mask.flatten()])

# reshape mask into 3-channel image
processed_mask = processed_mask.reshape(mask_height, mask_width, 3)
processed_mask = cv2.resize(processed_mask, (img_width, img_height), interpolation=cv2.INTER_NEAREST).astype(
    np.uint8)

# convert colored mask from BGR to RGB
processed_mask = cv2.cvtColor(processed_mask, cv2.COLOR_BGR2RGB)

In this step we map the probabilities from segmentation masks with appropriate colors of the predicted classes. Let's have a look at the results:

  • transform TF prediction into colored mask:
python
colors = np.array(colors)
processed_mask = colors[segm_mask[0]]

img_height = original_img_shape[0]
img_width = original_img_shape[1]

processed_mask = cv2.resize(processed_mask, (img_width, img_height), interpolation=cv2.INTER_NEAREST).astype(
    np.uint8)

# convert colored mask from BGR to RGB for compatibility with PASCAL VOC colors
processed_mask = cv2.cvtColor(processed_mask, cv2.COLOR_BGR2RGB)

The result is:

As a result, we get two equal segmentation masks.

Evaluation of the Models

The proposed in dnn/samples dnn_model_runner module allows to run the full evaluation pipeline on the PASCAL VOC dataset and test execution for the DeepLab MobileNet model.

Evaluation Mode

To below line represents running of the module in the evaluation mode:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.py_to_py_segm

The model will be read into OpenCV cv.dnn_Net object. Evaluation results of TF and OpenCV models (pixel accuracy, mean IoU, inference time) will be written into the log file. Inference time values will be also depicted in a chart to generalize the obtained model information.

Necessary evaluation configurations are defined in the test_config.py:

python
@dataclass
class TestSegmConfig:
    frame_size: int = 500
    img_root_dir: str = "./VOC2012"
    img_dir: str = os.path.join(img_root_dir, "JPEGImages/")
    img_segm_gt_dir: str = os.path.join(img_root_dir, "SegmentationClass/")
    # reduced val: https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/data/pascal/seg11valid.txt
    segm_val_file: str = os.path.join(img_root_dir, "ImageSets/Segmentation/seg11valid.txt")
    colour_file_cls: str = os.path.join(img_root_dir, "ImageSets/Segmentation/pascal-classes.txt")

These values can be modified in accordance with chosen model pipeline.

Test Mode

The below line represents running of the module in the test mode, which provides the steps for the model inference:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.py_to_py_segm --test True --default_img_preprocess <True/False> --evaluate False

Here default_img_preprocess key defines whether you'd like to parametrize the model test process with some particular values or use the default values, for example, scale, mean or std.

Test configuration is represented in test_config.py TestSegmModuleConfig class:

python
@dataclass
class TestSegmModuleConfig:
    segm_test_data_dir: str = "test_data/sem_segm"
    test_module_name: str = "segmentation"
    test_module_path: str = "segmentation.py"
    input_img: str = os.path.join(segm_test_data_dir, "2007_000033.jpg")
    model: str = ""

    frame_height: str = str(TestSegmConfig.frame_size)
    frame_width: str = str(TestSegmConfig.frame_size)
    scale: float = 1.0
    mean: List[float] = field(default_factory=lambda: [0.0, 0.0, 0.0])
    std: List[float] = field(default_factory=list)
    crop: bool = False
    rgb: bool = True
    classes: str = os.path.join(segm_test_data_dir, "pascal-classes.txt")

The default image preprocessing options are defined in default_preprocess_config.py:

python
tf_segm_input_blob = {
    "scale": str(1 / 127.5),
    "mean": ["127.5", "127.5", "127.5"],
    "std": [],
    "crop": "False",
    "rgb": "True"
}

The basis of the model testing is represented in samples/dnn/segmentation.py. segmentation.py can be executed autonomously with provided converted model in --input and populated parameters for cv2.dnn.blobFromImage.

To reproduce from scratch the described in "Model Conversion Pipeline" OpenCV steps with dnn_model_runner execute the below line:

text
python -m dnn_model_runner.dnn_conversion.tf.segmentation.py_to_py_segm --test True --default_img_preprocess True --evaluate False

Doc Tutorials Imgproc Morph Lines Detection Morph Lines Detection (`doc/tutorials/imgproc/morph_lines_detection/morph_lines_detection.md`)

Extract horizontal and vertical lines by using morphological operations {#tutorial_morph_lines_detection}

@tableofcontents

@prev_tutorial{tutorial_hitOrMiss}
@next_tutorial{tutorial_pyramids}
Original author Theodore Tsesmelis
Compatibility OpenCV >= 3.0

Goal

In this tutorial you will learn how to:

  • Apply two very common morphology operators (i.e. Dilation and Erosion), with the creation of custom kernels, in order to extract straight lines on the horizontal and vertical axes. For this purpose, you will use the following OpenCV functions:

    • erode()
    • dilate()
    • getStructuringElement()

    in an example where your goal will be to extract the music notes from a music sheet.

Theory

Morphology Operations

Morphology is a set of image processing operations that process images based on predefined structuring elements known also as kernels. The value of each pixel in the output image is based on a comparison of the corresponding pixel in the input image with its neighbors. By choosing the size and shape of the kernel, you can construct a morphological operation that is sensitive to specific shapes regarding the input image.

Two of the most basic morphological operations are dilation and erosion. Dilation adds pixels to the boundaries of the object in an image, while erosion does exactly the opposite. The amount of pixels added or removed, respectively depends on the size and shape of the structuring element used to process the image. In general the rules followed from these two operations have as follows:

  • Dilation: The value of the output pixel is the maximum value of all the pixels that fall within the structuring element's size and shape. For example in a binary image, if any of the pixels of the input image falling within the range of the kernel is set to the value 1, the corresponding pixel of the output image will be set to 1 as well. The latter applies to any type of image (e.g. grayscale, bgr, etc).

  • Erosion: The vice versa applies for the erosion operation. The value of the output pixel is the minimum value of all the pixels that fall within the structuring element's size and shape. Look the at the example figures below:

Structuring Elements

As it can be seen above and in general in any morphological operation the structuring element used to probe the input image, is the most important part.

A structuring element is a matrix consisting of only 0's and 1's that can have any arbitrary shape and size. Typically are much smaller than the image being processed, while the pixels with values of 1 define the neighborhood. The center pixel of the structuring element, called the origin, identifies the pixel of interest -- the pixel being processed.

For example, the following illustrates a diamond-shaped structuring element of 7x7 size.

A structuring element can have many common shapes, such as lines, diamonds, disks, periodic lines, and circles and sizes. You typically choose a structuring element the same size and shape as the objects you want to process/extract in the input image. For example, to find lines in an image, create a linear structuring element as you will see later.

Code

This tutorial code's is shown lines below.

@add_toggle_cpp
You can also download it from here.
@include samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp
@end_toggle

@add_toggle_java
You can also download it from here.
@include samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java
@end_toggle

@add_toggle_python
You can also download it from here.
@include samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py
@end_toggle

Explanation / Result

Get image from here .

Load Image

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp load_image
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java load_image
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py load_image
@end_toggle

Grayscale

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp gray
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java gray
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py gray
@end_toggle

Grayscale to Binary image

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp bin
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java bin
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py bin
@end_toggle

Output images

Now we are ready to apply morphological operations in order to extract the horizontal and vertical lines and as a consequence to separate the music notes from the music sheet, but first let's initialize the output images that we will use for that reason:

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp init
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java init
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py init
@end_toggle

Structure elements

As we specified in the theory in order to extract the object that we desire, we need to create the corresponding structure element. Since we want to extract the horizontal lines, a corresponding structure element for that purpose will have the following shape:

and in the source code this is represented by the following code snippet:

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp horiz
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java horiz
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py horiz
@end_toggle

The same applies for the vertical lines, with the corresponding structure element:

and again this is represented as follows:

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp vert
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java vert
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py vert
@end_toggle

Refine edges / Result

As you can see we are almost there. However, at that point you will notice that the edges of the notes are a bit rough. For that reason we need to refine the edges in order to obtain a smoother result:

@add_toggle_cpp
@snippet samples/cpp/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.cpp smooth
@end_toggle

@add_toggle_java
@snippet samples/java/tutorial_code/ImgProc/morph_lines_detection/Morphology_3.java smooth
@end_toggle

@add_toggle_python
@snippet samples/python/tutorial_code/imgProc/morph_lines_detection/morph_lines_detection.py smooth
@end_toggle

--- METRICS ---

Explore Related Technologies & Topics

View All 9,111+ Repositories →

Discover similar high-velocity repositories, agent skills, and OpenAPI specifications across the ecosystem.

#c-plus-plus Browse c-plus-plus catalog → #computer-vision Browse computer-vision catalog → #deep-learning Browse deep-learning catalog → #image-processing Browse image-processing catalog → #opencv Browse opencv catalog → ⚡ MCP Servers Model Context Protocol → 🧠 Agent Skills Prompt & rule files → 🔌 OpenAPI Specs REST & Swagger schemas →