mirror of
https://github.com/opencv/opencv.git
synced 2026-10-28 08:35:30 +08:00
Merge pull request #30289 from varun-jaiswal17:dnn-int8-cpu-perf
dnn : int8 optimisation - #30289 ### Correctness - **AVX-VNNI 1x1 conv, leftover pixels:** the last `H*W % 8` output pixels read the VNNI-repacked weights in the wrong order. Fixes the wrong top-1 of MobileNetV2 and DenseNet121 int8 with VNNI. - **Padded taps in scalar paths:** the depthwise (AVX2) tail and the NEON kxk scalar path skipped out-of-image taps instead of using the input zero point. The VNNI kxk kernel now never leaves a tile under 8 pixels. ### Speed - **First conv:** quantizing the float image is now parallel and vectorized. A float-input conv with up to 4 channels runs as a 1x1 conv over each pixel's window. First conv 5-6x faster (ResNet50: 0.88 -> 0.16 ms). `OPENCV_DNN_INT8_CONV_WINDOW_PACKING=0` disables the window packing. - **AVX-VNNI depthwise conv:** a new kernel transposes input rows into one row per channel and adds 4 taps for 8 pixels with one `pshufb` + `vpdpbusd`. Depthwise about 2x faster, output bit-identical. - **int8 MatMul at batch 1:** uses all threads instead of one. AlexNet int8: 2.75 -> 1.52 ms. - **Asymmetric padding:** convs with TF "SAME" padding at stride 2 are now fused to int8 instead of running in float between Dequantize and Quantize. Float convs in EfficientNet-Lite4 and MP hand pose: 5 -> 0-1. - **Broadcast int8 Add/Mul:** copies whole contiguous runs in parallel instead of computing an index per element. - **int8 Quantize/Dequantize:** AVX2 fast path for signed int8; per-tensor Q/DQ keep the block layout. ### Tests - Companion PR : opencv/opencv_extra#1436 (test data). ### Results (ms, i9-14900KS , default threads) | Model | 5.x before (ms) | this PR (ms) | ONNX Runtime 1.30 (ms) | Speedup vs 5.x | ORT / this PR | |---|---|---|---|---|---| | DenseNet121 int8 | 109.4 | 14.7 | 16.1 | 7.43x | 1.09x | | PP-OCRv3 det int8 | 65.7 | 12.7 | 33.1 | 5.15x | 2.60x | | SFace int8 | 8.1 | 3.0 | 19.0 | 2.75x | 6.43x | | EfficientNet-Lite4 int8 | 4.1 | 2.9 | 1.9 | 1.40x | 0.67x | | EfficientNet-Lite4 qdq | 4.6 | 3.1 | 2.0 | 1.49x | 0.65x | | MP hand pose int8 | 2.5 | 0.8 | 2.2 | 3.06x | 2.69x | | PPHumanSeg int8 | 2.6 | 1.9 | 5.6 | 1.41x | 3.02x | Co-authored with @Prasadayus ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Varun Jaiswal authored and GitHub committed
1 parent
048f050d5d
commit
56dabe2b19
9 files changed
+663
-114
No files matched your search
@@ -431,6 +431,45 @@ PERF_TEST_P_(DNNTestNetwork, EfficientDet_int8)
|
||||
processNet("dnn/tflite/coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite", "", inp);
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, YuNet_int8)
|
||||
{
|
||||
if (target != DNN_TARGET_CPU || backend != DNN_BACKEND_OPENCV)
|
||||
throw SkipTestException("");
|
||||
processNet("dnn/onnx/models/face_detection_yunet_2023mar_int8.onnx", "", cv::Size(640, 640));
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, SFace_int8)
|
||||
{
|
||||
if (target != DNN_TARGET_CPU || backend != DNN_BACKEND_OPENCV)
|
||||
throw SkipTestException("");
|
||||
processNet("dnn/onnx/models/face_recognition_sface_2021dec_int8.onnx", "", cv::Size(112, 112));
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, DenseNet_121_int8)
|
||||
{
|
||||
if (target != DNN_TARGET_CPU || backend != DNN_BACKEND_OPENCV)
|
||||
throw SkipTestException("");
|
||||
processNet("dnn/onnx/models/densenet-12-int8.onnx", "", cv::Size(224, 224));
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, EfficientNet_int8)
|
||||
{
|
||||
if (target != DNN_TARGET_CPU || backend != DNN_BACKEND_OPENCV)
|
||||
throw SkipTestException("");
|
||||
Mat sample = imread(findDataFile("dnn/dog416.png"));
|
||||
Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(224, 224), Scalar(), true);
|
||||
transposeND(inp, {0, 2, 3, 1}, inp);
|
||||
processNet("dnn/onnx/models/efficientnet-lite4-11-int8.onnx", "", inp);
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, PPOCRv3_int8)
|
||||
{
|
||||
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
|
||||
if (target != DNN_TARGET_CPU || backend != DNN_BACKEND_OPENCV)
|
||||
throw SkipTestException("");
|
||||
processNet("dnn/onnx/models/text_detection_en_ppocrv3_2023may_int8.onnx", "", cv::Size(736, 736));
|
||||
}
|
||||
|
||||
PERF_TEST_P_(DNNTestNetwork, YOLOv8n_TFLite)
|
||||
{
|
||||
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
|
||||
|
||||
@@ -560,14 +560,6 @@ struct ModelFusionQDQ
|
||||
if (wz != 0) all_wzp_zero = false;
|
||||
}
|
||||
|
||||
bool symmetric_pads = true;
|
||||
size_t npads = conv->pads.size();
|
||||
size_t ndims_pad = npads / 2;
|
||||
for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) {
|
||||
if (conv->pads[d] != conv->pads[d + ndims_pad])
|
||||
symmetric_pads = false;
|
||||
}
|
||||
|
||||
// Conv2Int8's VNNI kernel processes K0=8 output channels per
|
||||
// SIMD iteration. When ngroups>1 and Kg<K0, multiple groups map
|
||||
// to the same K0-wide output slot and clobber one another, so
|
||||
@@ -581,7 +573,7 @@ struct ModelFusionQDQ
|
||||
const bool kernelSupportsGrouping =
|
||||
conv->ngroups <= 1 || outChannelsPerGroup >= 8 || isDepthwise;
|
||||
|
||||
if (all_wzp_zero && symmetric_pads && kernelSupportsGrouping) {
|
||||
if (all_wzp_zero && kernelSupportsGrouping) {
|
||||
Mat bias = Mat::zeros(1, outCn, CV_32S);
|
||||
bool biasOk = true;
|
||||
int dq_bias_idx = -1;
|
||||
|
||||
@@ -493,11 +493,13 @@ static void convInt8BlockNEON(const void* inp_, const void* residual_,
|
||||
int zi = zi_base + ofsZYX[i * 3];
|
||||
int yi = yi_base + ofsZYX[i * 3 + 1];
|
||||
int xi = xi_base + ofsZYX[i * 3 + 2];
|
||||
if ((((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) |
|
||||
((unsigned)xi >= (unsigned)Wi_l)) != 0) continue;
|
||||
const int8_t* inptr = inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0;
|
||||
bool outside = (((unsigned)zi >= (unsigned)Di_l) | ((unsigned)yi >= (unsigned)Hi_l) |
|
||||
((unsigned)xi >= (unsigned)Wi_l)) != 0;
|
||||
const int8_t* inptr = outside ? zbuf
|
||||
: inpbaseptr + (((zi * Hi_l) + yi) * Wi_l + xi) * C0;
|
||||
const int inpstep = outside ? 0 : iplanesize_l;
|
||||
const int8_t* wptr_vnni = wbaseptr + (size_t)i * C1Max * K0 * C0;
|
||||
for (int c1i = 0; c1i < cblocks; c1i++, inptr += iplanesize_l) {
|
||||
for (int c1i = 0; c1i < cblocks; c1i++, inptr += inpstep) {
|
||||
const int8_t* wp = wptr_vnni + (size_t)c1i * C0 * K0;
|
||||
for (int c0 = 0; c0 < C0; c0++) {
|
||||
int ival = inputIsU8 ? ((int)(uint8_t)inptr[c0] - 128)
|
||||
@@ -1205,11 +1207,17 @@ static void convInt8BlockDepthwise(const void* inp_, const void* residual_,
|
||||
for (int i = 0; i < ksize; i++) {
|
||||
int zi = zi_base + ofsZYX[i*3], yi = yi_base + ofsZYX[i*3+1],
|
||||
xi = xi_base + ofsZYX[i*3+2];
|
||||
__m256i inp32;
|
||||
if ((unsigned)zi>=(unsigned)Di || (unsigned)yi>=(unsigned)Hi ||
|
||||
(unsigned)xi>=(unsigned)Wi) continue;
|
||||
const int8_t* ip = inpbaseptr + (((zi*Hi)+yi)*Wi+xi)*C0;
|
||||
__m256i inp32 = _mm256_cvtepu8_epi32(
|
||||
_mm_xor_si128(_mm_loadl_epi64((const __m128i*)ip), v_xor));
|
||||
(unsigned)xi>=(unsigned)Wi) {
|
||||
inp32 = inputIsU8
|
||||
? _mm256_set1_epi32(inp_zp)
|
||||
: _mm256_set1_epi32((int)(uint8_t)((uint8_t)inp_zp ^ 0x80u));
|
||||
} else {
|
||||
const int8_t* ip = inpbaseptr + (((zi*Hi)+yi)*Wi+xi)*C0;
|
||||
inp32 = _mm256_cvtepu8_epi32(
|
||||
_mm_xor_si128(_mm_loadl_epi64((const __m128i*)ip), v_xor));
|
||||
}
|
||||
__m256i w32 = _mm256_cvtepi8_epi32(
|
||||
_mm_loadl_epi64((const __m128i*)(wbuf + i*K0)));
|
||||
acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(inp32, w32));
|
||||
@@ -1384,6 +1392,8 @@ static void convInt8BlockVNNI_kxk(const void* inp_, const void* residual_,
|
||||
int min_ntiles = std::max(1, (nthreads * 4 + nkb - 1) / nkb);
|
||||
int TILE = std::max(SPAT_BLOCK_SIZE,
|
||||
((planeblocks_ + min_ntiles - 1) / min_ntiles + SPAT_BLOCK_SIZE - 1) & ~(SPAT_BLOCK_SIZE - 1));
|
||||
while (TILE < planeblocks_ && planeblocks_ % TILE != 0 && planeblocks_ % TILE < SPAT_BLOCK_SIZE)
|
||||
TILE += SPAT_BLOCK_SIZE;
|
||||
int ntiles = (planeblocks_ + TILE - 1) / TILE;
|
||||
int total_tasks = nkb * ntiles;
|
||||
const __m256i v_xor = inputIsU8 ? _mm256_setzero_si256() : _mm256_set1_epi8((char)0x80);
|
||||
@@ -1847,20 +1857,19 @@ static void convInt8BlockVNNI_1x1(const void* inp_, const void* residual_,
|
||||
}
|
||||
|
||||
for (; p < p_end; p++) {
|
||||
alignas(32) int32_t acc[K0];
|
||||
memcpy(acc, biasbuf, K0 * sizeof(int32_t));
|
||||
|
||||
__m256i s = _mm256_load_si256((const __m256i*)biasbuf);
|
||||
const int8_t* inptr_s = inpbaseptr + (size_t)p * C0;
|
||||
const int8_t* wptr = wbaseptr;
|
||||
for (int c1 = 0; c1 < cblocks; c1++, inptr_s += iplanesize, wptr += C0 * K0) {
|
||||
for (int c0 = 0; c0 < C0; c0++) {
|
||||
int iv = inputIsU8 ? (int)(uint8_t)inptr_s[c0]
|
||||
: (int)(uint8_t)((uint8_t)inptr_s[c0] ^ 0x80u);
|
||||
const int8_t* wp = wptr + c0 * K0;
|
||||
for (int k0 = 0; k0 < K0; k0++)
|
||||
acc[k0] += iv * (int)wp[k0];
|
||||
}
|
||||
__m256i wg0 = _mm256_load_si256((const __m256i*)(wptr));
|
||||
__m256i wg1 = _mm256_load_si256((const __m256i*)(wptr + 32));
|
||||
__m256i x0 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)(inptr_s )), v_xor);
|
||||
__m256i x1 = _mm256_xor_si256(_mm256_set1_epi32(*(const int32_t*)(inptr_s + 4)), v_xor);
|
||||
s = _mm256_dpbusd_epi32(s, x0, wg0);
|
||||
s = _mm256_dpbusd_epi32(s, x1, wg1);
|
||||
}
|
||||
alignas(32) int32_t acc[K0];
|
||||
_mm256_store_si256((__m256i*)acc, s);
|
||||
|
||||
int8_t* optr = outptr + (size_t)p * K0;
|
||||
const int8_t* rptr = resptr ? resptr + (size_t)p * K0 : nullptr;
|
||||
@@ -1886,6 +1895,200 @@ static void convInt8BlockVNNI_1x1(const void* inp_, const void* residual_,
|
||||
});
|
||||
}
|
||||
|
||||
static bool convInt8BlockDepthwiseVNNI(const void* inp_, void* out_, const ConvState& cs,
|
||||
const void* weights_, const int* bias,
|
||||
const float* multiplier, int inp_zp, int out_zp,
|
||||
const int8_t* activLUT, bool inputIsU8)
|
||||
{
|
||||
constexpr int C0 = 8, K0 = 8;
|
||||
const MatShape& inpshape = cs.inpshape;
|
||||
const MatShape& outshape = cs.outshape;
|
||||
if (cs.nspatialdims != 2 || inpshape.dims != 5 || outshape.dims != 5)
|
||||
return false;
|
||||
const int kh = cs.kshape[1], kw = cs.kshape[2];
|
||||
const int sy = cs.strides[1], sx = cs.strides[2];
|
||||
const int dy = cs.dilations[1];
|
||||
const int ksize = cs.wshape[2], C1Max = cs.wshape[3];
|
||||
if (cs.dilations[2] != 1 || sx > 4 || ksize != kh * kw)
|
||||
return false;
|
||||
|
||||
const int N = outshape[0], K1 = outshape[1], Ho = outshape[2], Wo = outshape[3];
|
||||
const int K = outshape.channels();
|
||||
const int Hi = inpshape[2], Wi = inpshape[3];
|
||||
const int padT = cs.pads[1], padL = cs.pads[2];
|
||||
const int G = (kw + 3) / 4;
|
||||
const int Wo8 = (Wo + 7) & ~7;
|
||||
const int Wp = std::max(padL + Wi, Wo8 * sx + 4 * G + 4 * sx) + 16;
|
||||
const uint8_t padval = inputIsU8 ? (uint8_t)inp_zp : (uint8_t)((uint8_t)inp_zp ^ 0x80u);
|
||||
|
||||
const int nthreads = std::max(1, cv::getNumThreads());
|
||||
int nbands = std::max(1, (nthreads * 4 + N * K1 - 1) / (N * K1));
|
||||
int band = (Ho + nbands - 1) / nbands;
|
||||
const int rowsBudget = std::max(1, (int)(((size_t)256 << 10) / ((size_t)C0 * Wp)));
|
||||
band = std::max(1, std::min(band, (rowsBudget - (kh - 1) * dy - 1) / sy + 1));
|
||||
nbands = (Ho + band - 1) / band;
|
||||
const int maxRows = (band - 1) * sy + (kh - 1) * dy + 1;
|
||||
|
||||
const int8_t* inp = (const int8_t*)inp_;
|
||||
int8_t* out = (int8_t*)out_;
|
||||
const int8_t* wdata = (const int8_t*)weights_;
|
||||
const int* ofsZYX = cs.coordtab.data();
|
||||
|
||||
parallel_for_(Range(0, N * K1 * nbands), [&](const Range& range) {
|
||||
AutoBuffer<uint8_t> planarBuf((size_t)maxRows * C0 * Wp);
|
||||
uint8_t* planar = planarBuf.data();
|
||||
AutoBuffer<int32_t> wqBuf((size_t)kh * G * C0);
|
||||
int32_t* wq = wqBuf.data();
|
||||
int32_t biasv[K0];
|
||||
float multv[K0];
|
||||
|
||||
alignas(32) int8_t gidx[32];
|
||||
for (int j = 0; j < 4; j++)
|
||||
for (int t4 = 0; t4 < 4; t4++)
|
||||
gidx[j * 4 + t4] = gidx[16 + j * 4 + t4] = (int8_t)(j * sx + t4);
|
||||
const __m256i vgather = _mm256_load_si256((const __m256i*)gidx);
|
||||
const __m256i vtr = _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
|
||||
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
|
||||
const __m128i v_xor = inputIsU8 ? _mm_setzero_si128() : _mm_set1_epi8((char)0x80);
|
||||
const uint8_t xorbyte = inputIsU8 ? 0 : 0x80;
|
||||
const __m256 vzp = _mm256_set1_ps((float)out_zp);
|
||||
|
||||
for (int t = range.start; t < range.end; t++) {
|
||||
int nk = t / nbands, b = t - nk * nbands;
|
||||
int n = nk / K1, k1 = nk - n * K1;
|
||||
int oy0 = b * band, oy1 = std::min(oy0 + band, Ho);
|
||||
int g0 = k1 * K0, n_g = std::min(K0, K - g0);
|
||||
|
||||
memset(wq, 0, sizeof(int32_t) * kh * G * C0);
|
||||
int8_t* wqb = (int8_t*)wq;
|
||||
for (int j = 0; j < n_g; j++) {
|
||||
int g = g0 + j;
|
||||
const int8_t* wg = wdata + (size_t)g * ksize * C1Max * C0 * K0 + (size_t)(g % C0) * K0;
|
||||
for (int i = 0; i < ksize; i++) {
|
||||
int ky = ofsZYX[i * 3 + 1] / dy, kx = ofsZYX[i * 3 + 2];
|
||||
wqb[((ky * G + kx / 4) * C0 + j) * 4 + (kx & 3)] = wg[(size_t)i * C1Max * C0 * K0];
|
||||
}
|
||||
biasv[j] = bias[g];
|
||||
multv[j] = multiplier[g];
|
||||
}
|
||||
for (int j = n_g; j < K0; j++) {
|
||||
biasv[j] = 0;
|
||||
multv[j] = 0.f;
|
||||
}
|
||||
|
||||
const int iy0 = oy0 * sy - padT;
|
||||
const int nrows = (oy1 - 1 - oy0) * sy + (kh - 1) * dy + 1;
|
||||
const int8_t* src = inp + (size_t)(n * K1 + k1) * Hi * Wi * C0;
|
||||
for (int r = 0; r < nrows; r++) {
|
||||
int iy = iy0 + r;
|
||||
uint8_t* prow = planar + (size_t)r * C0 * Wp;
|
||||
if (iy < 0 || iy >= Hi) {
|
||||
memset(prow, padval, (size_t)C0 * Wp);
|
||||
continue;
|
||||
}
|
||||
for (int c = 0; c < C0; c++) {
|
||||
memset(prow + c * Wp, padval, padL);
|
||||
memset(prow + c * Wp + padL + Wi, padval, Wp - padL - Wi);
|
||||
}
|
||||
const int8_t* sp = src + (size_t)iy * Wi * C0;
|
||||
uint8_t* dp = prow + padL;
|
||||
int x = 0;
|
||||
for (; x <= Wi - 8; x += 8, sp += 8 * C0, dp += 8) {
|
||||
__m128i a = _mm_xor_si128(_mm_loadu_si128((const __m128i*)sp), v_xor);
|
||||
__m128i bb = _mm_xor_si128(_mm_loadu_si128((const __m128i*)(sp + 16)), v_xor);
|
||||
__m128i cc = _mm_xor_si128(_mm_loadu_si128((const __m128i*)(sp + 32)), v_xor);
|
||||
__m128i dd = _mm_xor_si128(_mm_loadu_si128((const __m128i*)(sp + 48)), v_xor);
|
||||
__m128i t0 = _mm_unpacklo_epi8(a, bb), t1 = _mm_unpackhi_epi8(a, bb);
|
||||
__m128i t2 = _mm_unpacklo_epi8(cc, dd), t3 = _mm_unpackhi_epi8(cc, dd);
|
||||
__m128i u0 = _mm_unpacklo_epi8(t0, t1), u1 = _mm_unpackhi_epi8(t0, t1);
|
||||
__m128i u2 = _mm_unpacklo_epi8(t2, t3), u3 = _mm_unpackhi_epi8(t2, t3);
|
||||
__m128i v0 = _mm_unpacklo_epi32(u0, u2), v1 = _mm_unpackhi_epi32(u0, u2);
|
||||
__m128i v2 = _mm_unpacklo_epi32(u1, u3), v3 = _mm_unpackhi_epi32(u1, u3);
|
||||
_mm_storel_epi64((__m128i*)dp, v0);
|
||||
_mm_storeh_pd((double*)(dp + Wp), _mm_castsi128_pd(v0));
|
||||
_mm_storel_epi64((__m128i*)(dp + 2 * Wp), v1);
|
||||
_mm_storeh_pd((double*)(dp + 3 * Wp), _mm_castsi128_pd(v1));
|
||||
_mm_storel_epi64((__m128i*)(dp + 4 * Wp), v2);
|
||||
_mm_storeh_pd((double*)(dp + 5 * Wp), _mm_castsi128_pd(v2));
|
||||
_mm_storel_epi64((__m128i*)(dp + 6 * Wp), v3);
|
||||
_mm_storeh_pd((double*)(dp + 7 * Wp), _mm_castsi128_pd(v3));
|
||||
}
|
||||
for (; x < Wi; x++, sp += C0, dp++)
|
||||
for (int c = 0; c < C0; c++)
|
||||
dp[c * Wp] = (uint8_t)sp[c] ^ xorbyte;
|
||||
}
|
||||
|
||||
for (int oy = oy0; oy < oy1; oy++) {
|
||||
const uint8_t* rbase = planar + (size_t)(oy - oy0) * sy * C0 * Wp;
|
||||
int8_t* orow = out + ((size_t)(n * K1 + k1) * Ho + oy) * Wo * K0;
|
||||
for (int ox0 = 0; ox0 < Wo; ox0 += 8) {
|
||||
__m256i s0 = _mm256_set1_epi32(biasv[0]), s1 = _mm256_set1_epi32(biasv[1]);
|
||||
__m256i s2 = _mm256_set1_epi32(biasv[2]), s3 = _mm256_set1_epi32(biasv[3]);
|
||||
__m256i s4 = _mm256_set1_epi32(biasv[4]), s5 = _mm256_set1_epi32(biasv[5]);
|
||||
__m256i s6 = _mm256_set1_epi32(biasv[6]), s7 = _mm256_set1_epi32(biasv[7]);
|
||||
for (int ky = 0; ky < kh; ky++) {
|
||||
const uint8_t* rk = rbase + (size_t)ky * dy * C0 * Wp + (size_t)ox0 * sx;
|
||||
const int32_t* wk = wq + ky * G * C0;
|
||||
for (int g = 0; g < G; g++, rk += 4, wk += C0) {
|
||||
#define DWV_MAC(c) { \
|
||||
const uint8_t* p_ = rk + (c) * Wp; \
|
||||
__m256i x_ = _mm256_inserti128_si256(_mm256_castsi128_si256( \
|
||||
_mm_loadu_si128((const __m128i*)p_)), \
|
||||
_mm_loadu_si128((const __m128i*)(p_ + 4 * sx)), 1); \
|
||||
s##c = _mm256_dpbusd_epi32(s##c, _mm256_shuffle_epi8(x_, vgather), \
|
||||
_mm256_set1_epi32(wk[c])); \
|
||||
}
|
||||
DWV_MAC(0); DWV_MAC(1); DWV_MAC(2); DWV_MAC(3);
|
||||
DWV_MAC(4); DWV_MAC(5); DWV_MAC(6); DWV_MAC(7);
|
||||
#undef DWV_MAC
|
||||
}
|
||||
}
|
||||
|
||||
#define DWV_Q(c) _mm256_cvtps_epi32(_mm256_add_ps(_mm256_mul_ps( \
|
||||
_mm256_cvtepi32_ps(s##c), _mm256_set1_ps(multv[c])), vzp))
|
||||
__m256i p01 = _mm256_packs_epi32(DWV_Q(0), DWV_Q(1));
|
||||
__m256i p23 = _mm256_packs_epi32(DWV_Q(2), DWV_Q(3));
|
||||
__m256i p45 = _mm256_packs_epi32(DWV_Q(4), DWV_Q(5));
|
||||
__m256i p67 = _mm256_packs_epi32(DWV_Q(6), DWV_Q(7));
|
||||
#undef DWV_Q
|
||||
__m256i b03 = inputIsU8 ? _mm256_packus_epi16(p01, p23) : _mm256_packs_epi16(p01, p23);
|
||||
__m256i b47 = inputIsU8 ? _mm256_packus_epi16(p45, p67) : _mm256_packs_epi16(p45, p67);
|
||||
b03 = _mm256_shuffle_epi8(b03, vtr);
|
||||
b47 = _mm256_shuffle_epi8(b47, vtr);
|
||||
__m256i lo = _mm256_unpacklo_epi32(b03, b47);
|
||||
__m256i hi = _mm256_unpackhi_epi32(b03, b47);
|
||||
__m256i px03 = _mm256_permute2x128_si256(lo, hi, 0x20);
|
||||
__m256i px47 = _mm256_permute2x128_si256(lo, hi, 0x31);
|
||||
|
||||
int8_t* optr = orow + (size_t)ox0 * K0;
|
||||
const int npix = std::min(8, Wo - ox0);
|
||||
if (npix == 8 && n_g == K0 && !activLUT) {
|
||||
_mm256_storeu_si256((__m256i*)optr, px03);
|
||||
_mm256_storeu_si256((__m256i*)(optr + 32), px47);
|
||||
} else {
|
||||
alignas(32) int8_t tmp[64];
|
||||
_mm256_store_si256((__m256i*)tmp, px03);
|
||||
_mm256_store_si256((__m256i*)(tmp + 32), px47);
|
||||
for (int j = 0; j < npix; j++) {
|
||||
int8_t* o = optr + j * K0;
|
||||
const int8_t* tj = tmp + j * K0;
|
||||
if (!activLUT)
|
||||
memcpy(o, tj, n_g);
|
||||
else if (inputIsU8)
|
||||
for (int c = 0; c < n_g; c++)
|
||||
o[c] = activLUT[(int)(uint8_t)tj[c]];
|
||||
else
|
||||
for (int c = 0; c < n_g; c++)
|
||||
o[c] = activLUT[(int)tj[c] + 128];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
static void convInt8BlockVNNI(const void* inp_, const void* residual_,
|
||||
void* out_, const ConvState& cs,
|
||||
const void* weightsVNNI_,
|
||||
@@ -2145,12 +2348,22 @@ void convInt8Block(const void* inp_, const void* residual_,
|
||||
// Depthwise convolution has a dedicated kernel.
|
||||
if (cs.depthwise) {
|
||||
const int* dw_bias = biasVNNI_ ? biasVNNI_ : bias;
|
||||
#if CV_AVX_VNNI
|
||||
if (!residual_ && convInt8BlockDepthwiseVNNI(inp_, out_, cs, weights_, dw_bias, multiplier,
|
||||
inp_zp, out_zp, activLUT, inputIsU8))
|
||||
return;
|
||||
#endif
|
||||
convInt8BlockDepthwise(inp_, residual_, out_, cs, weights_,
|
||||
dw_bias, multiplier, inp_zp, out_zp, activLUT, inputIsU8);
|
||||
return;
|
||||
}
|
||||
#if CV_AVX_VNNI
|
||||
if (weightsVNNI_ && biasVNNI_) {
|
||||
bool useVNNI = weightsVNNI_ && biasVNNI_;
|
||||
if (useVNNI && cs.hasPadding()) {
|
||||
const MatShape& os = cs.outshape;
|
||||
useVNNI = os.total() / ((size_t)os[0] * os[1] * os.back()) >= 8;
|
||||
}
|
||||
if (useVNNI) {
|
||||
convInt8BlockVNNI(inp_, residual_, out_, cs, weightsVNNI_,
|
||||
biasVNNI_, multiplier, inp_zp, out_zp, activLUT, inputIsU8);
|
||||
return;
|
||||
|
||||
@@ -14,6 +14,48 @@
|
||||
namespace cv {
|
||||
namespace dnn {
|
||||
|
||||
static void quantizeRowInt8(const float* src, uint8_t* dst, int len, float inv_sc, int zp, bool is_u8)
|
||||
{
|
||||
int j = 0;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
const int VECSZ = VTraits<v_float32>::vlanes();
|
||||
v_float32 v_inv = vx_setall_f32(inv_sc);
|
||||
v_int32 v_zp = vx_setall_s32(zp);
|
||||
for (; j <= len - 4 * VECSZ; j += 4 * VECSZ) {
|
||||
v_int32 q0 = v_add(v_round(v_mul(vx_load(src + j), v_inv)), v_zp);
|
||||
v_int32 q1 = v_add(v_round(v_mul(vx_load(src + j + VECSZ), v_inv)), v_zp);
|
||||
v_int32 q2 = v_add(v_round(v_mul(vx_load(src + j + 2 * VECSZ), v_inv)), v_zp);
|
||||
v_int32 q3 = v_add(v_round(v_mul(vx_load(src + j + 3 * VECSZ), v_inv)), v_zp);
|
||||
v_int16 h0 = v_pack(q0, q1), h1 = v_pack(q2, q3);
|
||||
if (is_u8)
|
||||
v_store(dst + j, v_pack_u(h0, h1));
|
||||
else
|
||||
v_store((schar*)dst + j, v_pack(h0, h1));
|
||||
}
|
||||
#endif
|
||||
for (; j < len; j++) {
|
||||
int q = cvRound(src[j] * inv_sc) + zp;
|
||||
dst[j] = is_u8 ? saturate_cast<uchar>(q) : (uint8_t)saturate_cast<schar>(q);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void copyBytes(uint8_t* dst, const uint8_t* src, int n)
|
||||
{
|
||||
if (n >= 16) {
|
||||
int i = 0;
|
||||
for (; i + 16 <= n; i += 16)
|
||||
memcpy(dst + i, src + i, 16);
|
||||
if (i < n)
|
||||
memcpy(dst + n - 16, src + n - 16, 16);
|
||||
} else if (n >= 8) {
|
||||
memcpy(dst, src, 8);
|
||||
memcpy(dst + n - 8, src + n - 8, 8);
|
||||
} else {
|
||||
for (int i = 0; i < n; i++)
|
||||
dst[i] = src[i];
|
||||
}
|
||||
}
|
||||
|
||||
static MatShape getWpackShapeInt8(const MatShape& wshape, int ngroups, int C0)
|
||||
{
|
||||
CV_Assert(wshape.dims >= 3);
|
||||
@@ -159,6 +201,14 @@ public:
|
||||
Mat activationLUT;
|
||||
Ptr<ActivationLayerInt8> activ;
|
||||
|
||||
Mat quantizedInput;
|
||||
|
||||
bool useWindows = false;
|
||||
Mat winWeights, winWeightsVNNI;
|
||||
ConvState winCs;
|
||||
Mat paddedInput;
|
||||
Mat windowInput;
|
||||
|
||||
bool addResidual;
|
||||
bool inputIsU8;
|
||||
|
||||
@@ -318,24 +368,154 @@ public:
|
||||
const float* src = fp32_nchw.ptr<float>();
|
||||
uint8_t* dst = int8_block.ptr<uint8_t>();
|
||||
|
||||
parallel_for_(Range(0, N * C1), [&](const Range& range) {
|
||||
for (int nc = range.start; nc < range.end; nc++) {
|
||||
int n = nc / C1, c1 = nc % C1;
|
||||
const int PIX_STRIPE = 2048;
|
||||
const int nstripes = (planesize + PIX_STRIPE - 1) / PIX_STRIPE;
|
||||
|
||||
parallel_for_(Range(0, N * C1 * nstripes), [&](const Range& range) {
|
||||
const int BLK = 256;
|
||||
AutoBuffer<uint8_t> qbuf_(C0 * BLK);
|
||||
uint8_t* qbuf = qbuf_.data();
|
||||
for (int t = range.start; t < range.end; t++) {
|
||||
int nc = t / nstripes, stripe = t - nc * nstripes;
|
||||
int n = nc / C1, c1 = nc - n * C1;
|
||||
int c_base = c1 * C0;
|
||||
int nzc = std::min(C0, C - c_base);
|
||||
const float* src_block = src + (n * C + c_base) * planesize;
|
||||
uint8_t* dst_block = dst + (size_t)(n * C1 + c1) * planesize * C0;
|
||||
for (int i = 0; i < planesize; i++) {
|
||||
const float* src_block = src + ((size_t)n * C + c_base) * planesize;
|
||||
uint8_t* dst_block = dst + ((size_t)n * C1 + c1) * planesize * C0;
|
||||
int i1 = std::min((stripe + 1) * PIX_STRIPE, planesize);
|
||||
for (int i = stripe * PIX_STRIPE; i < i1; i += BLK) {
|
||||
int len = std::min(BLK, i1 - i);
|
||||
for (int c = 0; c < nzc; c++)
|
||||
quantizeRowInt8(src_block + (size_t)c * planesize + i, qbuf + c * BLK,
|
||||
len, inv_sc, zp, is_u8);
|
||||
uint8_t* dst_px = dst_block + (size_t)i * C0;
|
||||
if (nzc < C0)
|
||||
memset(dst_px, 0, (size_t)len * C0);
|
||||
for (int c = 0; c < nzc; c++) {
|
||||
float x = src_block[c * planesize + i];
|
||||
int q = cvRound(x * inv_sc) + zp;
|
||||
if (is_u8)
|
||||
dst_block[i * C0 + c] = saturate_cast<uchar>(q);
|
||||
else
|
||||
dst_block[i * C0 + c] = (uint8_t)(int8_t)saturate_cast<schar>(q);
|
||||
const uint8_t* q = qbuf + c * BLK;
|
||||
for (int j = 0; j < len; j++)
|
||||
dst_px[(size_t)j * C0 + c] = q[j];
|
||||
}
|
||||
for (int c = nzc; c < C0; c++)
|
||||
dst_block[i * C0 + c] = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool canUseWindowPacking(const MatShape& outshape, int C0) const
|
||||
{
|
||||
static const bool enabled =
|
||||
utils::getConfigurationParameterBool("OPENCV_DNN_INT8_CONV_WINDOW_PACKING", true);
|
||||
if (!enabled || !float_input || ngroups != 1 || blobs.empty() || C0 != 8 ||
|
||||
wshape0.dims != 4 || outshape.dims != 5)
|
||||
return false;
|
||||
int C = wshape0[1], kh = wshape0[2], kw = wshape0[3];
|
||||
if (C > 4 || kh * kw == 1)
|
||||
return false;
|
||||
size_t Cw8 = (size_t)(kh * kw * C + C0 - 1) / C0 * C0;
|
||||
size_t nbytes = (size_t)outshape[0] * outshape[2] * outshape[3] * Cw8;
|
||||
return nbytes <= ((size_t)64 << 20);
|
||||
}
|
||||
|
||||
void initWindowConv(const MatShape& outshape, int C0)
|
||||
{
|
||||
int K = wshape0[0], C = wshape0[1], kh = wshape0[2], kw = wshape0[3];
|
||||
int Cw8 = (kh * kw * C + C0 - 1) / C0 * C0;
|
||||
if (winWeights.empty()) {
|
||||
const Mat& w = blobs[0];
|
||||
CV_Assert(w.type() == CV_8SC1 && w.isContinuous() && w.total() == (size_t)K * C * kh * kw);
|
||||
Mat w1x1(std::vector<int>{K, Cw8, 1, 1}, CV_8SC1, Scalar(0));
|
||||
const int8_t* src = w.ptr<int8_t>();
|
||||
int8_t* dst = w1x1.ptr<int8_t>();
|
||||
for (int k = 0; k < K; k++)
|
||||
for (int c = 0; c < C; c++)
|
||||
for (int ky = 0; ky < kh; ky++)
|
||||
for (int kx = 0; kx < kw; kx++)
|
||||
dst[(size_t)k * Cw8 + (ky * kw + kx) * C + c] =
|
||||
src[(((size_t)k * C + c) * kh + ky) * kw + kx];
|
||||
repackConvWeightsInt8(w1x1, winWeights, 1, C0);
|
||||
Mat unusedBias;
|
||||
repackWeightsForVNNI(winWeights, 1, K, Cw8, biasInt32, winWeightsVNNI, unusedBias);
|
||||
}
|
||||
int N = outshape[0], Ho = outshape[2], Wo = outshape[3];
|
||||
MatShape wininp = MatShape({N, Cw8, Ho, Wo}, DATA_LAYOUT_NCHW).toLayout(DATA_LAYOUT_BLOCK, C0);
|
||||
winCs.initConv(wininp, MatShape({K, Cw8, 1, 1}, DATA_LAYOUT_UNKNOWN), outshape, 1,
|
||||
{1, 1}, {1, 1}, {0, 0, 0, 0}, AUTO_PAD_NONE, false,
|
||||
FAST_ACTIV_NONE, nullptr, {});
|
||||
}
|
||||
|
||||
void packWindows(const Mat& fp32_nchw, int C0)
|
||||
{
|
||||
MatShape nchw = fp32_nchw.shape();
|
||||
CV_Assert(nchw.dims == 4 && C0 == 8);
|
||||
const int N = nchw[0], C = nchw[1], H = nchw[2], W = nchw[3];
|
||||
const int kh = cs.kshape[1], kw = cs.kshape[2];
|
||||
const int sy = cs.strides[1], sx = cs.strides[2];
|
||||
const int dy = cs.dilations[1], dx = cs.dilations[2];
|
||||
const int padT = cs.pads[1], padL = cs.pads[2];
|
||||
const int Ho = cs.outshape[2], Wo = cs.outshape[3];
|
||||
const int Hp = std::max(padT + H + cs.pads[1 + ConvState::MAX_CONV_DIMS], (Ho - 1) * sy + (kh - 1) * dy + 1);
|
||||
const int Wp = std::max(padL + W + cs.pads[2 + ConvState::MAX_CONV_DIMS], (Wo - 1) * sx + (kw - 1) * dx + 1);
|
||||
const int Cw = kh * kw * C, Cw8 = (Cw + C0 - 1) / C0 * C0, C1w = Cw8 / C0;
|
||||
|
||||
const float inv_sc = input_sc > 0.f ? 1.f / input_sc : 0.f;
|
||||
const int zp = input_zp;
|
||||
const bool is_u8 = inputIsU8;
|
||||
const uint8_t zpbyte = (uint8_t)input_zp;
|
||||
const float* src = fp32_nchw.ptr<float>();
|
||||
const size_t prow = (size_t)Wp * C;
|
||||
|
||||
paddedInput.fit({N * Hp, (int)prow}, CV_8UC1);
|
||||
uint8_t* pbuf = paddedInput.ptr<uint8_t>();
|
||||
parallel_for_(Range(0, N * Hp), [&](const Range& range) {
|
||||
AutoBuffer<uint8_t> qbuf_((size_t)C * W);
|
||||
uint8_t* qbuf = qbuf_.data();
|
||||
for (int t = range.start; t < range.end; t++) {
|
||||
int n = t / Hp, y = t - n * Hp - padT;
|
||||
uint8_t* drow = pbuf + (size_t)t * prow;
|
||||
if (y < 0 || y >= H) {
|
||||
memset(drow, zpbyte, prow);
|
||||
continue;
|
||||
}
|
||||
memset(drow, zpbyte, (size_t)padL * C);
|
||||
memset(drow + (size_t)(padL + W) * C, zpbyte, (size_t)(Wp - padL - W) * C);
|
||||
for (int c = 0; c < C; c++)
|
||||
quantizeRowInt8(src + (((size_t)n * C + c) * H + y) * W, qbuf + (size_t)c * W,
|
||||
W, inv_sc, zp, is_u8);
|
||||
uint8_t* d = drow + (size_t)padL * C;
|
||||
for (int x = 0; x < W; x++)
|
||||
for (int c = 0; c < C; c++)
|
||||
d[x * C + c] = qbuf[(size_t)c * W + x];
|
||||
}
|
||||
});
|
||||
|
||||
MatShape winshape = MatShape({N, Cw8, Ho, Wo}, DATA_LAYOUT_NCHW).toLayout(DATA_LAYOUT_BLOCK, C0);
|
||||
windowInput.fit(winshape, inputIsU8 ? CV_8UC1 : CV_8SC1);
|
||||
uint8_t* wdst = windowInput.ptr<uint8_t>();
|
||||
const size_t plane = (size_t)Ho * Wo * C0;
|
||||
parallel_for_(Range(0, N * Ho), [&](const Range& range) {
|
||||
AutoBuffer<uint8_t> win_(Cw8);
|
||||
uint8_t* win = win_.data();
|
||||
memset(win + Cw, 0, Cw8 - Cw);
|
||||
for (int t = range.start; t < range.end; t++) {
|
||||
int n = t / Ho, oy = t - n * Ho;
|
||||
const uint8_t* srow = pbuf + ((size_t)n * Hp + (size_t)oy * sy) * prow;
|
||||
uint8_t* drow = wdst + (size_t)n * C1w * plane + (size_t)oy * Wo * C0;
|
||||
for (int ox = 0; ox < Wo; ox++) {
|
||||
const uint8_t* s = srow + (size_t)ox * sx * C;
|
||||
for (int ky = 0; ky < kh; ky++) {
|
||||
const uint8_t* sk = s + (size_t)ky * dy * prow;
|
||||
uint8_t* wk = win + ky * kw * C;
|
||||
if (dx == 1) {
|
||||
copyBytes(wk, sk, kw * C);
|
||||
} else {
|
||||
for (int kx = 0; kx < kw; kx++)
|
||||
for (int c = 0; c < C; c++)
|
||||
wk[kx * C + c] = sk[(size_t)kx * dx * C + c];
|
||||
}
|
||||
}
|
||||
uint8_t* d = drow + (size_t)ox * C0;
|
||||
for (int c1 = 0; c1 < C1w; c1++)
|
||||
memcpy(d + c1 * plane, win + c1 * 8, 8);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -348,19 +528,20 @@ public:
|
||||
int ninputs = (int)input_arrs.total();
|
||||
CV_Assert(ninputs >= 1);
|
||||
|
||||
Mat inp_storage;
|
||||
Mat inp_ref = input_arrs.getMat(0);
|
||||
|
||||
MatShape inpshape;
|
||||
if (float_input) {
|
||||
CV_Assert(inp_ref.type() == CV_32F);
|
||||
int C0 = getNetImpl(this)->defaultC0;
|
||||
quantizeInterleaveBlock(inp_ref, inp_storage, C0);
|
||||
MatShape nchw = inp_ref.shape();
|
||||
if (nchw.layout == DATA_LAYOUT_UNKNOWN)
|
||||
nchw.layout = DATA_LAYOUT_NCHW;
|
||||
inpshape = nchw.toLayout(DATA_LAYOUT_BLOCK, getNetImpl(this)->defaultC0);
|
||||
} else {
|
||||
CV_Assert(inp_ref.type() == CV_8SC1 || inp_ref.type() == CV_8UC1);
|
||||
inpshape = inp_ref.shape();
|
||||
}
|
||||
|
||||
const Mat& inp = float_input ? inp_storage : inp_ref;
|
||||
MatShape inpshape = inp.shape();
|
||||
CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK);
|
||||
CV_Assert(inp.type() == CV_8SC1 || inp.type() == CV_8UC1);
|
||||
|
||||
int C0 = inpshape.back();
|
||||
|
||||
@@ -402,7 +583,7 @@ public:
|
||||
MatShape outshape = convInferShape(inpshape, wshape0, emptyKernelShape,
|
||||
ngroups, strides, dilations,
|
||||
pads, auto_pad, ceil_mode);
|
||||
int outtype = inp.type();
|
||||
int outtype = float_input ? (inputIsU8 ? CV_8UC1 : CV_8SC1) : inp_ref.type();
|
||||
|
||||
int outkind = output_arrs.kind();
|
||||
Mat out;
|
||||
@@ -427,9 +608,23 @@ public:
|
||||
if (cs.depthwise) {
|
||||
cs.wshape = getWpackShapeInt8(wshape0, ngroups, C0);
|
||||
}
|
||||
useWindows = canUseWindowPacking(outshape, C0);
|
||||
if (useWindows)
|
||||
initWindowConv(outshape, C0);
|
||||
prevInpshape = inpshape;
|
||||
}
|
||||
|
||||
const Mat* inp = &inp_ref;
|
||||
if (useWindows) {
|
||||
packWindows(inp_ref, C0);
|
||||
inp = &windowInput;
|
||||
} else if (float_input) {
|
||||
quantizeInterleaveBlock(inp_ref, quantizedInput, C0);
|
||||
inp = &quantizedInput;
|
||||
}
|
||||
const Mat& wts = useWindows ? winWeights : weights;
|
||||
const Mat& wtsVNNI = useWindows ? winWeightsVNNI : weightsVNNI;
|
||||
|
||||
const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr<int8_t>() : nullptr;
|
||||
|
||||
const int* effectiveBiasVNNI = inputIsU8
|
||||
@@ -440,9 +635,9 @@ public:
|
||||
? biasBaselineU8.ptr<int>()
|
||||
: biasInt32.ptr<int>();
|
||||
|
||||
convInt8Block(inp.data, resptr, out.data, cs,
|
||||
weights.data,
|
||||
weightsVNNI.empty() ? nullptr : weightsVNNI.data,
|
||||
convInt8Block(inp->data, resptr, out.data, useWindows ? winCs : cs,
|
||||
wts.data,
|
||||
wtsVNNI.empty() ? nullptr : wtsVNNI.data,
|
||||
baselineBias,
|
||||
effectiveBiasVNNI,
|
||||
outMultiplier.ptr<float>(),
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
namespace cv {
|
||||
namespace dnn {
|
||||
|
||||
// A broadcast repeats whole contiguous runs, so the index arithmetic belongs once per run
|
||||
// rather than once per element.
|
||||
static Mat broadcastExpand(const Mat& src, const MatShape& dstShape)
|
||||
{
|
||||
MatShape srcShape = src.shape();
|
||||
@@ -27,27 +29,53 @@ static Mat broadcastExpand(const Mat& src, const MatShape& dstShape)
|
||||
if (srcShape == dstShape)
|
||||
return src;
|
||||
|
||||
MatShape outShape = dstShape;
|
||||
Mat out(outShape, src.type());
|
||||
const int total = (int)out.total();
|
||||
Mat out(dstShape, src.type());
|
||||
if (out.empty())
|
||||
return out;
|
||||
|
||||
std::vector<int> dstStrides(ndst);
|
||||
{ int s = 1; for (int d = ndst-1; d >= 0; d--) { dstStrides[d] = s; s *= dstShape[d]; } }
|
||||
// Innermost axes either all repeat one element, or all match and copy a contiguous run.
|
||||
int d = ndst - 1;
|
||||
int64_t inner = 1;
|
||||
const bool repeat = srcShape[d] == 1 && dstShape[d] != 1;
|
||||
for (; d >= 0 && (repeat ? srcShape[d] == 1 : srcShape[d] == dstShape[d]); d--)
|
||||
inner *= dstShape[d];
|
||||
|
||||
std::vector<int> srcStrides(ndst, 0);
|
||||
{ int s = 1; for (int d = ndst-1; d >= 0; d--) {
|
||||
srcStrides[d] = (srcShape[d] > 1) ? s : 0; s *= srcShape[d]; } }
|
||||
// Axes outside that run whose source extent is 1 repeat the whole run, so it is built
|
||||
// once and doubled. Without this a block layout's C0 axis caps the run at C0 elements.
|
||||
int64_t rep = 1;
|
||||
for (; d >= 0 && srcShape[d] == 1; d--)
|
||||
rep *= dstShape[d];
|
||||
const int64_t block = inner * rep;
|
||||
|
||||
const int nouter = d + 1;
|
||||
std::vector<int64_t> srcStrides(ndst, 0), rowStrides(nouter);
|
||||
{ int64_t s = 1; for (int i = ndst-1; i >= 0; i--) {
|
||||
srcStrides[i] = (srcShape[i] > 1) ? s : 0; s *= srcShape[i]; } }
|
||||
{ int64_t s = 1; for (int i = nouter-1; i >= 0; i--) {
|
||||
rowStrides[i] = s; s *= dstShape[i]; } }
|
||||
|
||||
const int nrows = (int)((int64_t)out.total() / block);
|
||||
const uint8_t* sp = src.ptr<uint8_t>();
|
||||
uint8_t* dp = out.ptr<uint8_t>();
|
||||
for (int i = 0; i < total; i++) {
|
||||
int rem = i, srcIdx = 0;
|
||||
for (int d = 0; d < ndst; d++) {
|
||||
int coord = rem / dstStrides[d]; rem %= dstStrides[d];
|
||||
srcIdx += coord * srcStrides[d];
|
||||
|
||||
parallel_for_(Range(0, nrows), [&](const Range& r) {
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
int64_t off = 0;
|
||||
for (int k = 0; k < nouter; k++)
|
||||
off += (i / rowStrides[k] % dstShape[k]) * srcStrides[k];
|
||||
uint8_t* row = dp + i * block;
|
||||
if (repeat)
|
||||
std::memset(row, sp[off], (size_t)inner);
|
||||
else
|
||||
std::memcpy(row, sp + off, (size_t)inner);
|
||||
for (int64_t done = inner; done < block; ) {
|
||||
const int64_t n = std::min(done, block - done);
|
||||
std::memcpy(row + done, row, (size_t)n);
|
||||
done += n;
|
||||
}
|
||||
}
|
||||
dp[i] = sp[srcIdx];
|
||||
}
|
||||
}, (double)std::max(1, getNumThreads() * 4));
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -117,7 +117,8 @@ public:
|
||||
Mat dstMat = output[0].reshape(1, outerSize);
|
||||
Mat dstMatInt32 = Mat(shape(dstMat), CV_32S);
|
||||
|
||||
const int nstripes = outerSize <= 4 ? 1 : getNumThreads();
|
||||
const size_t nouts = (size_t)outerSize * weightsMat.rows;
|
||||
const int nstripes = (int)std::max<size_t>(1, std::min<size_t>(getNumThreads(), nouts / 16));
|
||||
runInt8Gemm(srcMat, weightsMat, biasMat, outputMultiplier, dstMatInt32, nstripes, output_zp);
|
||||
if (output_type == CV_8U) {
|
||||
dstMatInt32.convertTo(dstMat, output_type, 1, 128);
|
||||
|
||||
@@ -26,30 +26,31 @@ namespace dnn
|
||||
*/
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
template <typename _InpTp>
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((target("avx2")))
|
||||
#endif
|
||||
static void dequantizeLinearChunk_u8_f32_avx2(const uint8_t* src, float* dst,
|
||||
float scale, int zp, int64_t len)
|
||||
static void dequantizeLinearChunk_f32_avx2(const _InpTp* src, float* dst,
|
||||
float scale, int zp, int64_t len)
|
||||
{
|
||||
__m256 vscale = _mm256_set1_ps(scale);
|
||||
__m256i vzp = _mm256_set1_epi32(zp);
|
||||
int64_t j = 0;
|
||||
for (; j <= len - 8; j += 8) {
|
||||
__m128i raw = _mm_loadl_epi64((__m128i*)(src + j));
|
||||
__m256i vi = _mm256_cvtepu8_epi32(raw);
|
||||
vi = _mm256_sub_epi32(vi, vzp);
|
||||
__m256 vf = _mm256_cvtepi32_ps(vi);
|
||||
vf = _mm256_mul_ps(vf, vscale);
|
||||
_mm256_storeu_ps(dst + j, vf);
|
||||
__m128i raw = _mm_loadl_epi64((const __m128i*)(src + j));
|
||||
// Signed input must sign-extend, not zero-extend.
|
||||
__m256i vi = std::is_signed<_InpTp>::value ? _mm256_cvtepi8_epi32(raw)
|
||||
: _mm256_cvtepu8_epi32(raw);
|
||||
__m256 vf = _mm256_cvtepi32_ps(_mm256_sub_epi32(vi, vzp));
|
||||
_mm256_storeu_ps(dst + j, _mm256_mul_ps(vf, vscale));
|
||||
}
|
||||
for (; j < len; j++)
|
||||
dst[j] = (float)(src[j] - zp) * scale;
|
||||
dst[j] = (float)((int)src[j] - zp) * scale;
|
||||
}
|
||||
|
||||
static void dequantizeLinearFast_u8_f32_avx2(const uint8_t* inp, float* out,
|
||||
float scale, int zp,
|
||||
int64_t total)
|
||||
template <typename _InpTp>
|
||||
static void dequantizeLinearFast_f32_avx2(const _InpTp* inp, float* out,
|
||||
float scale, int zp, int64_t total)
|
||||
{
|
||||
const int64_t block = 1024;
|
||||
int64_t nblocks = (total + block - 1) / block;
|
||||
@@ -58,7 +59,7 @@ static void dequantizeLinearFast_u8_f32_avx2(const uint8_t* inp, float* out,
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
int64_t ofs = i * block;
|
||||
int64_t len = std::min(block, total - ofs);
|
||||
dequantizeLinearChunk_u8_f32_avx2(inp + ofs, out + ofs, scale, zp, len);
|
||||
dequantizeLinearChunk_f32_avx2(inp + ofs, out + ofs, scale, zp, len);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -290,14 +291,19 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp_,
|
||||
|
||||
// Fast path: per-tensor dequantization uint8→float with AVX2 + proper parallelism
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F
|
||||
&& checkHardwareSupport(CV_CPU_AVX2)) {
|
||||
if (block_size == 0 && sz_a == 1 && outtype == CV_32F && sctype == CV_32F
|
||||
&& (inptype == CV_8U || inptype == CV_8S) && checkHardwareSupport(CV_CPU_AVX2)) {
|
||||
float sc = reinterpret_cast<const float*>(scale.data)[0];
|
||||
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
int64_t total = nslices * slice_size;
|
||||
dequantizeLinearFast_u8_f32_avx2(reinterpret_cast<const uint8_t*>(inp.data),
|
||||
reinterpret_cast<float*>(out.data),
|
||||
sc, zpval, total);
|
||||
if (inptype == CV_8U) {
|
||||
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
dequantizeLinearFast_f32_avx2(reinterpret_cast<const uint8_t*>(inp.data),
|
||||
reinterpret_cast<float*>(out.data), sc, zpval, total);
|
||||
} else {
|
||||
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const int8_t*>(zp.data)[0];
|
||||
dequantizeLinearFast_f32_avx2(reinterpret_cast<const int8_t*>(inp.data),
|
||||
reinterpret_cast<float*>(out.data), sc, zpval, total);
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@@ -447,6 +453,31 @@ public:
|
||||
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
|
||||
}
|
||||
|
||||
// Block layout reorders channels, so only a scalar scale survives it unchanged;
|
||||
// a per-channel or blocked one would be indexed against the wrong axis.
|
||||
bool isPerTensor() const
|
||||
{
|
||||
if (block_size != 0 || this->inputs.size() < 2)
|
||||
return false;
|
||||
Net::Impl* netimpl_ = getNetImpl(this);
|
||||
return netimpl_ && netimpl_->isConstArg(this->inputs[1]) &&
|
||||
netimpl_->argTensor(this->inputs[1]).total() == 1;
|
||||
}
|
||||
|
||||
int getLayouts(const std::vector<DataLayout>& actualInputs,
|
||||
std::vector<DataLayout>& desiredInputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<DataLayout>& outputs) const CV_OVERRIDE
|
||||
{
|
||||
size_t ninputs = actualInputs.size();
|
||||
CV_Assert(ninputs >= 1u);
|
||||
desiredInputs.assign(ninputs, DATA_LAYOUT_UNKNOWN);
|
||||
const DataLayout keep = isPerTensor() ? actualInputs[0] : DATA_LAYOUT_UNKNOWN;
|
||||
desiredInputs[0] = keep;
|
||||
outputs.assign(requiredOutputs, keep);
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<MatShape> &outputs,
|
||||
|
||||
@@ -17,39 +17,52 @@ namespace cv
|
||||
namespace dnn
|
||||
{
|
||||
|
||||
// Fast path for per-tensor quantization: float → uint8 with AVX2
|
||||
// Fast path for per-tensor quantization: float -> uint8 or int8 with AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
template <typename _OutTp>
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((target("avx2")))
|
||||
#endif
|
||||
static void quantizeLinearChunk_f32_u8_avx2(const float* src, uint8_t* dst,
|
||||
float scale, float zp_f,
|
||||
int64_t len)
|
||||
static void quantizeLinearChunk_f32_avx2(const float* src, _OutTp* dst,
|
||||
float scale, int zp_i, int64_t len)
|
||||
{
|
||||
const int lo_i = (int)std::numeric_limits<_OutTp>::lowest();
|
||||
const int hi_i = (int)std::numeric_limits<_OutTp>::max();
|
||||
__m256 vscale = _mm256_set1_ps(scale);
|
||||
// The zero-point is integral. Round x/scale FIRST, then add the integer zp;
|
||||
// adding an odd zp before rounding would flip round-half-to-even ties
|
||||
int zp_i = cvRound(zp_f);
|
||||
// Clamp in float: a value outside int32 range converts to INT_MIN, which then
|
||||
// saturates to the wrong end. Bounds carry the zero point so it can be added after.
|
||||
__m256 vlo = _mm256_set1_ps((float)(lo_i - zp_i));
|
||||
__m256 vhi = _mm256_set1_ps((float)(hi_i - zp_i));
|
||||
__m256i vzp = _mm256_set1_epi32(zp_i);
|
||||
|
||||
int64_t j = 0;
|
||||
for (; j <= len - 8; j += 8) {
|
||||
__m256 v = _mm256_loadu_ps(src + j);
|
||||
__m256i vi = _mm256_cvtps_epi32(_mm256_div_ps(v, vscale)); // round half-to-even
|
||||
vi = _mm256_add_epi32(vi, vzp); // + integer zero-point
|
||||
__m128i lo = _mm256_castsi256_si128(vi);
|
||||
__m128i hi = _mm256_extracti128_si256(vi, 1);
|
||||
__m128i packed16 = _mm_packs_epi32(lo, hi);
|
||||
__m128i packed8 = _mm_packus_epi16(packed16, packed16); // saturates to [0,255]
|
||||
__m256 v = _mm256_div_ps(_mm256_loadu_ps(src + j), vscale);
|
||||
// max then min, so a NaN takes the second operand and lands inside the range.
|
||||
v = _mm256_min_ps(_mm256_max_ps(v, vlo), vhi);
|
||||
// Round before adding the zero point; an odd one would flip half-to-even ties.
|
||||
__m256i vi = _mm256_add_epi32(_mm256_cvtps_epi32(v), vzp);
|
||||
__m128i packed16 = _mm_packs_epi32(_mm256_castsi256_si128(vi),
|
||||
_mm256_extracti128_si256(vi, 1));
|
||||
__m128i packed8 = std::is_signed<_OutTp>::value
|
||||
? _mm_packs_epi16(packed16, packed16)
|
||||
: _mm_packus_epi16(packed16, packed16);
|
||||
_mm_storel_epi64((__m128i*)(dst + j), packed8);
|
||||
}
|
||||
for (; j < len; j++)
|
||||
dst[j] = saturate_cast<uint8_t>(cvRound(src[j] / scale) + zp_i);
|
||||
// Same clamp as the vector loop: out of range cvRound gives INT_MIN and the zero point
|
||||
// then overflows. A NaN fails both compares and lands on the low bound, as MAXPS does.
|
||||
const float flo = (float)(lo_i - zp_i), fhi = (float)(hi_i - zp_i);
|
||||
for (; j < len; j++) {
|
||||
float v = src[j] / scale;
|
||||
v = v > flo ? v : flo;
|
||||
v = v < fhi ? v : fhi;
|
||||
dst[j] = (_OutTp)(cvRound(v) + zp_i);
|
||||
}
|
||||
}
|
||||
|
||||
static void quantizeLinearFast_f32_u8_avx2(const float* inp, uint8_t* out,
|
||||
float scale, float zp_f,
|
||||
int64_t total)
|
||||
template <typename _OutTp>
|
||||
static void quantizeLinearFast_f32_avx2(const float* inp, _OutTp* out,
|
||||
float scale, int zp_i, int64_t total)
|
||||
{
|
||||
const int64_t block = 1024;
|
||||
int64_t nblocks = (total + block - 1) / block;
|
||||
@@ -58,7 +71,7 @@ static void quantizeLinearFast_f32_u8_avx2(const float* inp, uint8_t* out,
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
int64_t ofs = i * block;
|
||||
int64_t len = std::min(block, total - ofs);
|
||||
quantizeLinearChunk_f32_u8_avx2(inp + ofs, out + ofs, scale, zp_f, len);
|
||||
quantizeLinearChunk_f32_avx2(inp + ofs, out + ofs, scale, zp_i, len);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -286,16 +299,23 @@ static void quantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
|
||||
}
|
||||
}
|
||||
|
||||
// Fast path: per-tensor quantization float→uint8 with AVX2 + proper parallelism
|
||||
// Fast path: per-tensor quantization float to uint8 or int8 with AVX2 + parallelism
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
if (block_size == 0 && sz_a == 1 && inptype == CV_32F && outtype == CV_8U && sctype == CV_32F
|
||||
&& checkHardwareSupport(CV_CPU_AVX2)) {
|
||||
if (block_size == 0 && sz_a == 1 && inptype == CV_32F && sctype == CV_32F
|
||||
&& (outtype == CV_8U || outtype == CV_8S) && checkHardwareSupport(CV_CPU_AVX2)) {
|
||||
float scale_f32 = reinterpret_cast<const float*>(scale.data)[0];
|
||||
float zp_f = zp.empty() ? 0.f : (float)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
int64_t total = nslices * slice_size;
|
||||
quantizeLinearFast_f32_u8_avx2(reinterpret_cast<const float*>(inp.data),
|
||||
if (outtype == CV_8U) {
|
||||
int zp_i = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
quantizeLinearFast_f32_avx2(reinterpret_cast<const float*>(inp.data),
|
||||
reinterpret_cast<uint8_t*>(out.data),
|
||||
scale_f32, zp_f, total);
|
||||
scale_f32, zp_i, total);
|
||||
} else {
|
||||
int zp_i = zp.empty() ? 0 : (int)reinterpret_cast<const int8_t*>(zp.data)[0];
|
||||
quantizeLinearFast_f32_avx2(reinterpret_cast<const float*>(inp.data),
|
||||
reinterpret_cast<int8_t*>(out.data),
|
||||
scale_f32, zp_i, total);
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@@ -430,6 +450,31 @@ public:
|
||||
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
|
||||
}
|
||||
|
||||
// Block layout reorders channels, so only a scalar scale survives it unchanged;
|
||||
// a per-channel or blocked one would be indexed against the wrong axis.
|
||||
bool isPerTensor() const
|
||||
{
|
||||
if (block_size != 0 || this->inputs.size() < 2)
|
||||
return false;
|
||||
Net::Impl* netimpl_ = getNetImpl(this);
|
||||
return netimpl_ && netimpl_->isConstArg(this->inputs[1]) &&
|
||||
netimpl_->argTensor(this->inputs[1]).total() == 1;
|
||||
}
|
||||
|
||||
int getLayouts(const std::vector<DataLayout>& actualInputs,
|
||||
std::vector<DataLayout>& desiredInputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<DataLayout>& outputs) const CV_OVERRIDE
|
||||
{
|
||||
size_t ninputs = actualInputs.size();
|
||||
CV_Assert(ninputs >= 1u);
|
||||
desiredInputs.assign(ninputs, DATA_LAYOUT_UNKNOWN);
|
||||
const DataLayout keep = isPerTensor() ? actualInputs[0] : DATA_LAYOUT_UNKNOWN;
|
||||
desiredInputs[0] = keep;
|
||||
outputs.assign(requiredOutputs, keep);
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<MatShape> &outputs,
|
||||
|
||||
@@ -2421,6 +2421,11 @@ TEST_P(Test_ONNX_layers, Quantized_Eltwise_Broadcast)
|
||||
testONNXModels("quantized_eltwise_broadcast");
|
||||
}
|
||||
|
||||
TEST_P(Test_ONNX_layers, Quantized_BatchNorm_Mul_Add)
|
||||
{
|
||||
testONNXModels("quantized_bn_mul_add");
|
||||
}
|
||||
|
||||
TEST_P(Test_ONNX_layers, Quantized_LeakyReLU)
|
||||
{
|
||||
testONNXModels("quantized_leaky_relu");
|
||||
|
||||
Reference in new issue
Block a user