Files
opencv/modules/gapi/misc/python/test/test_gapi_stateful_kernel.py
Dmitry Matveev fc5d412ba7 Merge pull request #23597 from dmatveev:dm/gapi_onnx_py_integration
G-API: Integration branch for ONNX & Python-related changes #23597

# Changes overview

## 1. Expose ONNX backend's Normalization and Mean-value parameters in Python

* Since Python G-API bindings rely on `Generic` infer to express Inference, the `Generic` specialization of `onnx::Params` was extended with new methods to control normalization (`/255`) and mean-value; these methods were exposed in the Python bindings
* Found some questionable parts in the existing API which I'd like to review/discuss (see comments)

UPD:
1. Thanks to @TolyaTalamanov normalization inconsistencies have been identified with `squeezenet1.0-9` ONNX model itself; tests using these model were updated to DISABLE normalization and NOT using mean/value.
2. Questionable parts were removed and tests still pass.

### Details (taken from @TolyaTalamanov's comment):

`squeezenet1.0.*onnx` - doesn't require scaling to [0,1] and mean/std because the weights of the first convolution already scaled. ONNX documentation is broken. So the correct approach to use this models is:

1. ONNX: apply preprocessing from the documentation: https://github.com/onnx/models/blob/main/vision/classification/imagenet_preprocess.py#L8-L44 but without normalization step:
```
# DON'T DO IT:
# mean_vec = np.array([0.485, 0.456, 0.406])
# stddev_vec = np.array([0.229, 0.224, 0.225])
# norm_img_data = np.zeros(img_data.shape).astype('float32')
# for i in range(img_data.shape[0]):
#     norm_img_data[i,:,:] = (img_data[i,:,:]/255 - mean_vec[i]) / stddev_vec[i]
#     # add batch channel
#     norm_img_data = norm_img_data.reshape(1, 3, 224, 224).astype('float32')
#     return norm_img_data

# INSTEAD
return img_data.reshape(1, 3, 224, 224)
```

2. G-API: Convert image from BGR to RGB and then pass to `apply` as-is with configuring parameters:
```
net = cv.gapi.onnx.params('squeezenet', model_filename)
net.cfgNormalize('data_0', False)
```
**Note**: Results might be difference because `G-API` doesn't apply central crop but just do resize to model resolution.

---

`squeezenet1.1.*onnx` - requires scaling to [0,1] and mean/std - onnx documentation is correct.
1. ONNX: apply preprocessing from the documentation: https://github.com/onnx/models/blob/main/vision/classification/imagenet_preprocess.py#L8-L44
2. G-API: Convert image from BGR to RGB and then pass to `apply` as-is with configuring parameters:
```
net = cv.gapi.onnx.params('squeezenet', model_filename)
net.cfgNormalize('data_0', True) // default
net.cfgMeanStd('data_0', [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
```
**Note**: Results might be difference because `G-API` doesn't apply central crop but just do resize to model resolution.

## 2. Expose Fluid & kernel package-related functionality in Python

* `cv::gapi::combine()`
* `cv::GKernelPackage::size()` (mainly for testing purposes)
* `cv::gapi::imgproc::fluid::kernels()`

Added a test for the above.

## 3. Fixed issues with Python stateful kernel handling

Fixed error message when `outMeta()` of custom python operation fails.

## 4. Fixed various issues in Python tests

1. `test_gapi_streaming.py` - fixed behavior of Desync test to avoid sporadic issues
2. `test_gapi_infer_onnx.py` - fixed model lookup (it was still using the ONNX Zoo layout but was NOT using the proper env var we use to point to one).

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2023-05-30 17:52:17 +03:00

217 lines
6.4 KiB
Python

#!/usr/bin/env python
import numpy as np
import cv2 as cv
import os
import sys
import unittest
from tests_common import NewOpenCVTests
try:
if sys.version_info[:2] < (3, 0):
raise unittest.SkipTest('Python 2.x is not supported')
class CounterState:
def __init__(self):
self.counter = 0
@cv.gapi.op('stateful_counter',
in_types=[cv.GOpaque.Int],
out_types=[cv.GOpaque.Int])
class GStatefulCounter:
"""Accumulates state counter on every call"""
@staticmethod
def outMeta(desc):
return cv.empty_gopaque_desc()
@cv.gapi.kernel(GStatefulCounter)
class GStatefulCounterImpl:
"""Implementation for GStatefulCounter operation."""
@staticmethod
def setup(desc):
return CounterState()
@staticmethod
def run(value, state):
state.counter += value
return state.counter
class SumState:
def __init__(self):
self.sum = 0
@cv.gapi.op('stateful_sum',
in_types=[cv.GOpaque.Int, cv.GOpaque.Int],
out_types=[cv.GOpaque.Int])
class GStatefulSum:
"""Accumulates sum on every call"""
@staticmethod
def outMeta(lhs_desc, rhs_desc):
return cv.empty_gopaque_desc()
class gapi_sample_pipelines(NewOpenCVTests):
def test_stateful_kernel_single_instance(self):
g_in = cv.GOpaque.Int()
g_out = GStatefulCounter.on(g_in)
comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out))
pkg = cv.gapi.kernels(GStatefulCounterImpl)
nums = [i for i in range(10)]
acc = 0
for v in nums:
acc = comp.apply(cv.gin(v), args=cv.gapi.compile_args(pkg))
self.assertEqual(sum(nums), acc)
def test_stateful_kernel_multiple_instances(self):
# NB: Every counter has his own independent state.
g_in = cv.GOpaque.Int()
g_out0 = GStatefulCounter.on(g_in)
g_out1 = GStatefulCounter.on(g_in)
comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out0, g_out1))
pkg = cv.gapi.kernels(GStatefulCounterImpl)
nums = [i for i in range(10)]
acc0 = acc1 = 0
for v in nums:
acc0, acc1 = comp.apply(cv.gin(v), args=cv.gapi.compile_args(pkg))
ref = sum(nums)
self.assertEqual(ref, acc0)
self.assertEqual(ref, acc1)
def test_stateful_throw_setup(self):
@cv.gapi.kernel(GStatefulCounter)
class GThrowStatefulCounterImpl:
"""Implementation for GStatefulCounter operation
that throw exception in setup method"""
@staticmethod
def setup(desc):
raise Exception('Throw from setup method')
@staticmethod
def run(value, state):
raise Exception('Unreachable')
g_in = cv.GOpaque.Int()
g_out = GStatefulCounter.on(g_in)
comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out))
pkg = cv.gapi.kernels(GThrowStatefulCounterImpl)
with self.assertRaises(Exception): comp.apply(cv.gin(42),
args=cv.gapi.compile_args(pkg))
def test_stateful_reset(self):
g_in = cv.GOpaque.Int()
g_out = GStatefulCounter.on(g_in)
comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out))
pkg = cv.gapi.kernels(GStatefulCounterImpl)
cc = comp.compileStreaming(args=cv.gapi.compile_args(pkg))
cc.setSource(cv.gin(1))
cc.start()
for i in range(1, 10):
_, actual = cc.pull()
self.assertEqual(i, actual)
cc.stop()
cc.setSource(cv.gin(2))
cc.start()
for i in range(2, 10, 2):
_, actual = cc.pull()
self.assertEqual(i, actual)
cc.stop()
def test_stateful_multiple_inputs(self):
@cv.gapi.kernel(GStatefulSum)
class GStatefulSumImpl:
"""Implementation for GStatefulCounter operation."""
@staticmethod
def setup(lhs_desc, rhs_desc):
return SumState()
@staticmethod
def run(lhs, rhs, state):
state.sum+= lhs + rhs
return state.sum
g_in1 = cv.GOpaque.Int()
g_in2 = cv.GOpaque.Int()
g_out = GStatefulSum.on(g_in1, g_in2)
comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out))
pkg = cv.gapi.kernels(GStatefulSumImpl)
lhs_list = [1, 10, 15]
rhs_list = [2, 14, 32]
ref_out = 0
for lhs, rhs in zip(lhs_list, rhs_list):
ref_out += lhs + rhs
gapi_out = comp.apply(cv.gin(lhs, rhs), cv.gapi.compile_args(pkg))
self.assertEqual(ref_out, gapi_out)
def test_stateful_multiple_inputs_throw(self):
@cv.gapi.kernel(GStatefulSum)
class GStatefulSumImplIncorrect:
"""Incorrect implementation for GStatefulCounter operation."""
# NB: setup methods is intentionally
# incorrect - accepts one meta arg instead of two
@staticmethod
def setup(desc):
return SumState()
@staticmethod
def run(lhs, rhs, state):
state.sum+= lhs + rhs
return state.sum
g_in1 = cv.GOpaque.Int()
g_in2 = cv.GOpaque.Int()
g_out = GStatefulSum.on(g_in1, g_in2)
comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out))
pkg = cv.gapi.kernels(GStatefulSumImplIncorrect)
with self.assertRaises(Exception): comp.apply(cv.gin(42, 42),
args=cv.gapi.compile_args(pkg))
except unittest.SkipTest as e:
message = str(e)
class TestSkip(unittest.TestCase):
def setUp(self):
self.skipTest('Skip tests: ' + message)
def test_skip():
pass
pass
if __name__ == '__main__':
NewOpenCVTests.bootstrap()