卷积神经网络在 ARM-CPU 上的推断计算综述

// using Gas, add -mfpu=neon to the assembler command line
// compilation: arm-none-linux-gnueabi-as -mfpu=neon asm.s
.text
.arm
.global double_elements
double_elements:
vadd.i32 q0,q0,q0
bx       lr
.end

// specify a target processor that supports NEON instructions
// armasm --cpu=Cortex-A8 asm.s
AREA RO, CODE, READONLY
ARM
EXPORT double_elements
double_elements
VADD.I32 Q0, Q0, Q0
BX       LR
END

// NEON intrinsics
// same syntax for GNU & RVCT

// NEON intrinsics with GCC
// arm-none-linux-gnueabi-gcc -mfpu=neon intrinsic.c
// Depending on your toolchain, you might also have to add 
// -mfloat-abi=softfp to indicate to the compiler that 
// NEON variables must be passed in general purpose registers.

// NEON intrinsics with RVCT
// armcc --cpu=Cortex-A9 intrinsic.c

#include <arm_neon.h>

uint32x4_t double_elements(uint32x4_t input)
{
    return(vaddq_u32(input, input));
}

// GCC
// arm-none-linux-gnueabi-gcc -mfpu=neon -ftree-vectorize -ftree-vectorizer-verbose=1 -c vectorized.c

// RVCT
// armcc --cpu=Cortex-A9 -O3 -Otime --vectorize --fpmode=fast --remarks -c vectorized.c

// 使用 __restrict 保证指针 pa, pb 没有在内存中没有交叠的地方
void add_ints(int * __restrict pa, int * __restrict pb, unsigned int n, int x)
{
    unsigned int i;
    // n & ~3 使 n 的低2位为0，既 n 为4的倍数。假设此处 int* 是32位整型指针，n 为4，则
    // 编译器会将下面的循环使用 VADD.I32 Q, Q, Q 进行优化，Q 为128位寄存器
    for(i = 0; i < (n & ~3); i++) pa[i] = pb[i] + x;
}

/* Header regrouping all the NEON functions */
#include "arm_compute/runtime/NEON/functions/NEAbsoluteDifference.h"
#include "arm_compute/runtime/NEON/functions/NEAccumulate.h"
#include "arm_compute/runtime/NEON/functions/NEActivationLayer.h"
#include "arm_compute/runtime/NEON/functions/NEArithmeticAddition.h"
#include "arm_compute/runtime/NEON/functions/NEArithmeticSubtraction.h"
#include "arm_compute/runtime/NEON/functions/NEBatchNormalizationLayer.h"
#include "arm_compute/runtime/NEON/functions/NEBitwiseAnd.h"
#include "arm_compute/runtime/NEON/functions/NEBitwiseNot.h"
#include "arm_compute/runtime/NEON/functions/NEBitwiseOr.h"
#include "arm_compute/runtime/NEON/functions/NEBitwiseXor.h"
#include "arm_compute/runtime/NEON/functions/NEBox3x3.h"
#include "arm_compute/runtime/NEON/functions/NECannyEdge.h"
#include "arm_compute/runtime/NEON/functions/NEChannelCombine.h"
#include "arm_compute/runtime/NEON/functions/NEChannelExtract.h"
#include "arm_compute/runtime/NEON/functions/NEColorConvert.h"
#include "arm_compute/runtime/NEON/functions/NEConvolution.h"
#include "arm_compute/runtime/NEON/functions/NEConvolutionLayer.h"
#include "arm_compute/runtime/NEON/functions/NEDepthConcatenate.h"
#include "arm_compute/runtime/NEON/functions/NEDepthConvert.h"
#include "arm_compute/runtime/NEON/functions/NEDerivative.h"
#include "arm_compute/runtime/NEON/functions/NEDilate.h"
#include "arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h"
#include "arm_compute/runtime/NEON/functions/NEEqualizeHistogram.h"
#include "arm_compute/runtime/NEON/functions/NEErode.h"
#include "arm_compute/runtime/NEON/functions/NEFastCorners.h"
#include "arm_compute/runtime/NEON/functions/NEFillBorder.h"
#include "arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h"
#include "arm_compute/runtime/NEON/functions/NEGEMM.h"
#include "arm_compute/runtime/NEON/functions/NEGEMMInterleave4x4.h"
#include "arm_compute/runtime/NEON/functions/NEGEMMLowp.h"
#include "arm_compute/runtime/NEON/functions/NEGEMMTranspose1xW.h"
......

#include "arm_compute/runtime/NEON/NEFunctions.h"
#include "arm_compute/core/Types.h"
#include "utils/Utils.h"
#include <time.h>
#include <iostream>

using namespace arm_compute;
using namespace utils;

void main_mobilenets(int argc, const char **argv)
{
    ARM_COMPUTE_UNUSED(argc);
    ARM_COMPUTE_UNUSED(argv);
    clock_t start, end;
    double cpu_time_used;
    
    // The src tensor should contain the input image
    Tensor src;
    
    // The weights and biases tensors for MobileNets
    Tensor conv1_w;
    Tensor conv1_b;
    Tensor conv1_bn_b;
    Tensor conv1_bn_s;
    Tensor conv1_bn_rm;
    Tensor conv1_bn_riv;
    
    Tensor out_conv1;
    Tensor out_conv1_bn;
    Tensor out_conv1_bn_relu;
    
    NEConvolutionLayer conv1;
    NEBatchNormalizationLayer conv1_bn;
    NEActivationLayer conv1_bn_relu;
    
    /* [Initialize tensors] */
    // Initialize src tensor
    constexpr unsigned int width_src_image  = 160;
    constexpr unsigned int height_src_image = 160;
    constexpr unsigned int ifm_src_img      = 3;
    const TensorShape src_shape(width_src_image, height_src_image, ifm_src_img);
    src.allocator()->init(TensorInfo(src_shape, 1, DataType::F32));
    
    // Initialize tensors of conv1
    constexpr unsigned int kernel_x_conv1 = 3;
    constexpr unsigned int kernel_y_conv1 = 3;
    constexpr unsigned int ofm_conv1      = 16;
    const TensorShape weights_shape_conv1(kernel_x_conv1, kernel_y_conv1, src_shape.z(), ofm_conv1);
    const TensorShape biases_shape_conv1(weights_shape_conv1[3]);
    const TensorShape out_shape_conv1(src_shape.x() / 2, src_shape.y() / 2, weights_shape_conv1[3]);
    conv1_w.allocator()->init(TensorInfo(weights_shape_conv1, 1, DataType::F32));
    conv1_b.allocator()->init(TensorInfo(biases_shape_conv1, 1, DataType::F32));
    out_conv1.allocator()->init(TensorInfo(out_shape_conv1, 1, DataType::F32));
    
    // Initialize tensors of conv1_bn
    const TensorShape b_shape_conv1_bn(weights_shape_conv1[3]);
    const TensorShape s_shape_conv1_bn(weights_shape_conv1[3]);
    const TensorShape rm_shape_conv1_bn(weights_shape_conv1[3]);
    const TensorShape riv_shape_conv1_bn(weights_shape_conv1[3]);
    const TensorShape out_shape_conv1_bn(out_shape_conv1);
    conv1_bn_b.allocator()->init(TensorInfo(b_shape_conv1_bn, 1, DataType::F32));
    conv1_bn_s.allocator()->init(TensorInfo(s_shape_conv1_bn, 1, DataType::F32));
    conv1_bn_rm.allocator()->init(TensorInfo(rm_shape_conv1_bn, 1, DataType::F32));
    conv1_bn_riv.allocator()->init(TensorInfo(riv_shape_conv1_bn, 1, DataType::F32));
    out_conv1_bn.allocator()->init(TensorInfo(out_shape_conv1_bn, 1, DataType::F32));
    
    // Initialize tensors of conv1_bn_relu
    const TensorShape out_shape_conv1_bn_relu(out_shape_conv1);
    out_conv1_bn_relu.allocator()->init(TensorInfo(out_shape_conv1_bn_relu, 1, DataType::F32));
    /* -----------------------End: [Initialize tensors] */
    
    /* [Configure functions] */
    conv1.configure(&src, &conv1_w, &conv1_b, &out_conv1, PadStrideInfo(2, 2, 1, 1));
    conv1_bn.configure(&out_conv1, &out_conv1_bn, &conv1_bn_rm, &conv1_bn_riv, &conv1_bn_b, &conv1_bn_s, 0.0001);
    conv1_bn_relu.configure(&out_conv1_bn, &out_conv1_bn_relu, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU));
    /* -----------------------End: [Configure functions] */
    
    /* [Allocate tensors] */
    // Now that the padding requirements are known we can allocate the images:
    src.allocator()->allocate();
    conv1_w.allocator()->allocate();
    conv1_b.allocator()->allocate();
    out_conv1.allocator()->allocate();
    conv1_bn_b.allocator()->allocate();
    conv1_bn_s.allocator()->allocate();
    conv1_bn_rm.allocator()->allocate();
    conv1_bn_riv.allocator()->allocate();
    out_conv1_bn.allocator()->allocate();
    out_conv1_bn_relu.allocator()->allocate();
    /* -----------------------End: [Allocate tensors] */
    
    /* [Initialize weights and biases tensors] */
    // Once the tensors have been allocated, the src, weights and biases tensors can be initialized
    // ...
    /* -----------------------[Initialize weights and biases tensors] */
    
    /* [Execute the functions] */
    start = clock();
    conv1.run();
    conv1_bn.run();
    conv1_bn_relu.run();
    end = clock();
    cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
    std::cout << "time: " << cpu_time_used * 1000 << "ms" << std::endl;
    /* -----------------------End: [Execute the functions] */
}

/** Main program for mobilenets test
*
* The example implements the following mobilenets architecture:
*
*
* @param[in] argc Number of arguments
* @param[in] argv Arguments
*/
int main(int argc, const char **argv)
{
    return utils::run_example(argc, argv, main_mobilenets);
}

// 读取非加密文件
ncnn::Net net;
net.load_param("alexnet.param");
net.load_model("alexnet.bin");
// 读取加密文件
ncnn::Net net;
net.load_param_bin("alexnet.param.bin");
net.load_model("alexnet.bin");

static int detect_squeezenet(const cv::Mat& bgr, std::vector<float>& cls_scores)
{
    ncnn::Net squeezenet;
    squeezenet.load_param("squeezenet_v1.1.param");    // 加载网络参数文件
    squeezenet.load_model("squeezenet_v1.1.bin");    // 加载网络结构文件

    // 将 OpenCV2 的输入图像进行格式转换
    ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, 227, 227);

    const float mean_vals[3] = {104.f, 117.f, 123.f};
    in.substract_mean_normalize(mean_vals, 0);

    ncnn::Extractor ex = squeezenet.create_extractor();
    ex.set_light_mode(true);

    ex.input("data", in);    // 输入推断数据

    ncnn::Mat out;
    ex.extract("prob", out);    // 获得推断结果

    cls_scores.resize(out.c);
    for (int j=0; j<out.c; j++)
    {
        const float* prob = out.data + out.cstep * j;
        cls_scores[j] = prob[0];
    }
    return 0;
}

// -----CUDA code-----
extern "C" __global__ void myadd__kernel0(float* __restrict__ C, float* __restrict__ A, float* __restrict__ B, int n) {
  if (((int)blockIdx.x) < ((n + -127) / 64)) {
    C[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))] = 
        (A[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))] + B[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))]);
  } else {
    if ((((int)blockIdx.x) * 64) < (n - ((int)threadIdx.x))) {
      C[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))] = 
        (A[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))] + B[((((int)blockIdx.x) * 64) + ((int)threadIdx.x))]);
    }
  }
}

fadd_cl = tvm.build(s, [A, B, C], "opencl", name="myadd")
ctx = tvm.cl(0)
n = 1024
a = tvm.nd.array(np.random.uniform(size=n).astype(A.dtype), ctx)
b = tvm.nd.array(np.random.uniform(size=n).astype(B.dtype), ctx)
c = tvm.nd.array(np.zeros(n, dtype=C.dtype), ctx)
fadd_cl(a, b, c)

// ------opencl code------
__kernel void myadd__kernel0(__global float* restrict C, __global float* restrict A, __global float* restrict B, int n) {
  if (((int)get_group_id(0)) < ((n + -127) / 64)) {
    C[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))] = 
        (A[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))] + B[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))]);
  } else {
    if ((((int)get_group_id(0)) * 64) < (n - ((int)get_local_id(0)))) {
      C[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))] = 
        (A[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))] + B[((((int)get_group_id(0)) * 64) + ((int)get_local_id(0)))]);
    }
  }
}