tiny-cnn执行过程分析(MNIST)

在http://blog.csdn.net/fengbingchun/article/details/50573841中以MNIST为例对tiny-cnn的使用进行了介绍，下面对其执行过程进行分析：

支持两种损失函数：(1)、mean squared error(均方差)；(2)、cross entropy(交叉熵)。在MNIST中使用的是mean squared error，代码段：

// mean-squared-error loss function for regression
class mse {
public:static float_t f(float_t y, float_t t) {return (y - t) * (y - t) / 2;}static float_t df(float_t y, float_t t) {return y - t;}
};

支持六种激活函数：(1)、tanh；(2)、sigmoid；(3)、softmax；(4)、rectifiedlinear(relu)；(5)、leaky relu；(6)、identity。MNIST中使用的是tanh，代码段：

class tan_h : public function {
public:float_t f(const vec_t& v, size_t i) const override {const float_t ep = std::exp(v[i]);const float_t em = std::exp(-v[i]); return (ep - em) / (ep + em);}// fast approximation of tanh (improve 2-3% speed in LeNet-5)/*float_t f(float_t x) const {const float_t x2 = x * x;x *= 1.0 + x2 * (0.1653 + x2 * 0.0097);return x / std::sqrt(1.0 + x * x);// invsqrt(static_cast<float>(1.0 + x * x));}*/float_t df(float_t y) const override { return 1.0 - sqr(y); }std::pair<float_t, float_t> scale() const override { return std::make_pair(-0.8, 0.8); }

设计CNN结构，用于MNIST，与LeNet-5结构相似，去除了F6层：

输入层Input：图像大小32*32，神经元数量32*32=1024，代码段：

    const int width = header.num_cols + 2 * x_padding;const int height = header.num_rows + 2 * y_padding;std::vector<uint8_t> image_vec(header.num_rows * header.num_cols);ifs.read((char*) &image_vec[0], header.num_rows * header.num_cols);dst.resize(width * height, scale_min);for (size_t y = 0; y < header.num_rows; y++)for (size_t x = 0; x < header.num_cols; x++)dst[width * (y + y_padding) + x + x_padding]= (image_vec[y * header.num_cols + x] / 255.0) * (scale_max - scale_min) + scale_min;

C1层：卷积窗大小5*5，输出特征图数量6，卷积窗种类6，输出特征图大小28*28，可训练参数5*5*6+6=156，神经元数量28*28*6=4704；

S2层：卷积窗大小2*2，输出下采样图数量6，卷积窗种类6，输出下采样图大小14*14，可训练参数1*6+6=12，神经元数量14*14*6=1176；

C3层：卷积窗大小5*5，输出特征图数量16，卷积窗种类16，输出特征图大小10*10，可训练参数6*16*5*5+16=2416，神经元数量10*10*16=1600；

S4层：卷积窗大小2*2，输出下采样图数量16，卷积窗种类16，输出下采样图大小5*5，可训练参数1*16+16=32，神经元数量5*5*16=400；

C5层：卷积窗大小5*5，输出特征图数量120，卷积窗种类120，输出特征图大小1*1，可训练参数5*5*16*120+120=48120，神经元数量1*120=120；

输出层Output：输出特征图数量10，卷积窗种类10，输出特征图大小1*1，可训练参数120*10+10=1210，神经元数量1*10=10。

原有MNIST图像大小为28*28，此处为32*32，上下左右各填补2个像素，填补的像素取值为-1，其它像素取值范围为[-1,1]。

权值和阈值(偏置)初始化：权值采用均匀随机数产生，阈值均赋0。

C1层权值，初始化范围[sqrt(6.0/(25+150)), sqrt(6.0/(25+150))]；

S2层权值，初始化范围[sqrt(6.0/(4+1)), - sqrt(6.0/(4+1))]；

C3层权值，初始化范围[sqrt(6.0/(150+400)), - sqrt(6.0/(150+400))]；

S4层权值，初始化范围[sqrt(6.0/(4+1)), - sqrt(6.0/(4+1))]；

C5层权值，初始化范围[sqrt(6.0/(400+3000)), - sqrt(6.0/(400+3000))]；

输出层权值，初始化范围[sqrt(6.0/(120+10)),  -sqrt(6.0/(120+10))]。

前向传播：

C1层代码段：

        vec_t &a = a_[worker_index]; // w*xvec_t &out = output_[worker_index]; // outputconst vec_t &in = *(prev_out_padded_[worker_index]); // inputstd::fill(a.begin(), a.end(), (float_t)0.0);for_i(parallelize_, out_.depth_, [&](int o) {for (layer_size_t inc = 0; inc < in_.depth_; inc++) {if (!tbl_.is_connected(o, inc)) continue;const float_t *pw = &this->W_[weight_.get_index(0, 0, in_.depth_ * o + inc)];const float_t *pi = &in[in_padded_.get_index(0, 0, inc)];float_t *pa = &a[out_.get_index(0, 0, o)];for (layer_size_t y = 0; y < out_.height_; y++) {for (layer_size_t x = 0; x < out_.width_; x++) {const float_t * ppw = pw;const float_t * ppi = pi + (y * h_stride_) * in_padded_.width_ + x * w_stride_;float_t sum = (float_t)0.0;// should be optimized for small kernel(3x3,5x5)for (layer_size_t wy = 0; wy < weight_.height_; wy++) {for (layer_size_t wx = 0; wx < weight_.width_; wx++) {sum += *ppw++ * ppi[wy * in_padded_.width_ + wx];}}pa[y * out_.width_ + x] += sum;}}}if (!this->b_.empty()) {float_t *pa = &a[out_.get_index(0, 0, o)];float_t b = this->b_[o];std::for_each(pa, pa + out_.width_ * out_.height_, [&](float_t& f) { f += b; });}});for_i(parallelize_, out_size_, [&](int i) {out[i] = h_.f(a, i);});

S2层代码段：

        vec_t& a = a_[index];for_i(parallelize_, out_size_, [&](int i) {const wi_connections& connections = out2wi_[i];a[i] = 0.0;for (auto connection : connections)// 13.1%a[i] += W_[connection.first] * in[connection.second]; // 3.2%a[i] *= scale_factor_;a[i] += b_[out2bias_[i]];});for_i(parallelize_, out_size_, [&](int i) {output_[index][i] = h_.f(a, i);});

C3层、C5层代码段与C1层相同。

S4层代码段与S2层相同。

输出层代码段：

        vec_t &a = a_[index];vec_t &out = output_[index];for_i(parallelize_, out_size_, [&](int i) {a[i] = 0.0;for (layer_size_t c = 0; c < in_size_; c++) {a[i] += W_[c*out_size_ + i] * in[c];}if (has_bias_)a[i] += b_[i];});for_i(parallelize_, out_size_, [&](int i) {out[i] = h_.f(a, i);});

反向传播：

输出层代码段：

        vec_t delta(out_dim());const activation::function& h = layers_.tail()->activation_function();if (is_canonical_link(h)) {for_i(out_dim(), [&](int i){ delta[i] = out[i] - t[i]; });} else {vec_t dE_dy = gradient<E>(out, t);// delta = dE/da = (dE/dy) * (dy/da)for (size_t i = 0; i < out_dim(); i++) {vec_t dy_da = h.df(out, i);delta[i] = vectorize::dot(&dE_dy[0], &dy_da[0], out_dim());}}

C5层代码段：

        const vec_t& prev_out = prev_->output(index);const activation::function& prev_h = prev_->activation_function();vec_t& prev_delta = prev_delta_[index];vec_t& dW = dW_[index];vec_t& db = db_[index];for (layer_size_t c = 0; c < this->in_size_; c++) {// propagate delta to previous layer// prev_delta[c] += current_delta[r] * W_[c * out_size_ + r]prev_delta[c] = vectorize::dot(&curr_delta[0], &W_[c*out_size_], out_size_);prev_delta[c] *= prev_h.df(prev_out[c]);}for_(parallelize_, 0, (size_t)out_size_, [&](const blocked_range& r) {// accumulate weight-step using delta// dW[c * out_size + i] += current_delta[i] * prev_out[c]for (layer_size_t c = 0; c < in_size_; c++)vectorize::muladd(&curr_delta[r.begin()], prev_out[c], r.end() - r.begin(), &dW[c*out_size_ + r.begin()]);if (has_bias_) {for (int i = r.begin(); i < r.end(); i++)db[i] += curr_delta[i];}});

S4层代码段：

        const vec_t& prev_out = *(prev_out_padded_[index]);const activation::function& prev_h = prev_->activation_function();vec_t* prev_delta = (pad_type_ == padding::same) ? &prev_delta_padded_[index] : &prev_delta_[index];vec_t& dW = dW_[index];vec_t& db = db_[index];std::fill(prev_delta->begin(), prev_delta->end(), (float_t)0.0);// propagate delta to previous layerfor_i(in_.depth_, [&](int inc) {for (layer_size_t outc = 0; outc < out_.depth_; outc++) {if (!tbl_.is_connected(outc, inc)) continue;const float_t *pw = &this->W_[weight_.get_index(0, 0, in_.depth_ * outc + inc)];const float_t *pdelta_src = &curr_delta[out_.get_index(0, 0, outc)];float_t *pdelta_dst = &(*prev_delta)[in_padded_.get_index(0, 0, inc)];for (layer_size_t y = 0; y < out_.height_; y++) {for (layer_size_t x = 0; x < out_.width_; x++) {const float_t * ppw = pw;const float_t ppdelta_src = pdelta_src[y * out_.width_ + x];float_t * ppdelta_dst = pdelta_dst + y * h_stride_ * in_padded_.width_ + x * w_stride_;for (layer_size_t wy = 0; wy < weight_.height_; wy++) {for (layer_size_t wx = 0; wx < weight_.width_; wx++) {ppdelta_dst[wy * in_padded_.width_ + wx] += *ppw++ * ppdelta_src;}}}}}});for_i(parallelize_, in_padded_.size(), [&](int i) {(*prev_delta)[i] *= prev_h.df(prev_out[i]);});// accumulate dwfor_i(in_.depth_, [&](int inc) {for (layer_size_t outc = 0; outc < out_.depth_; outc++) {if (!tbl_.is_connected(outc, inc)) continue;for (layer_size_t wy = 0; wy < weight_.height_; wy++) {for (layer_size_t wx = 0; wx < weight_.width_; wx++) {float_t dst = 0.0;const float_t * prevo = &prev_out[in_padded_.get_index(wx, wy, inc)];const float_t * delta = &curr_delta[out_.get_index(0, 0, outc)];for (layer_size_t y = 0; y < out_.height_; y++) {dst += vectorize::dot(prevo + y * in_padded_.width_, delta + y * out_.width_, out_.width_);}dW[weight_.get_index(wx, wy, in_.depth_ * outc + inc)] += dst;}}}});// accumulate dbif (!db.empty()) {for (layer_size_t outc = 0; outc < out_.depth_; outc++) {const float_t *delta = &curr_delta[out_.get_index(0, 0, outc)];db[outc] += std::accumulate(delta, delta + out_.width_ * out_.height_, (float_t)0.0);}}

C3层代码段：

        const vec_t& prev_out = prev_->output(index);const activation::function& prev_h = prev_->activation_function();vec_t& prev_delta = prev_delta_[index];for_(parallelize_, 0, (size_t)in_size_, [&](const blocked_range& r) {for (int i = r.begin(); i != r.end(); i++) {const wo_connections& connections = in2wo_[i];float_t delta = 0.0;for (auto connection : connections) delta += W_[connection.first] * current_delta[connection.second]; // 40.6%prev_delta[i] = delta * scale_factor_ * prev_h.df(prev_out[i]); // 2.1%}});for_(parallelize_, 0, weight2io_.size(), [&](const blocked_range& r) {for (int i = r.begin(); i < r.end(); i++) {const io_connections& connections = weight2io_[i];float_t diff = 0.0;for (auto connection : connections) // 11.9%diff += prev_out[connection.first] * current_delta[connection.second];dW_[index][i] += diff * scale_factor_;}});for (size_t i = 0; i < bias2out_.size(); i++) {const std::vector<layer_size_t>& outs = bias2out_[i];float_t diff = 0.0;for (auto o : outs)diff += current_delta[o];    db_[index][i] += diff;} 

S2层、输入层代码段与S4层相同。

C1层代码段与C3层相同。

权值和偏置更新代码段：

    void update(const vec_t& dW, const vec_t& /*Hessian*/, vec_t &W) {vec_t& g = get<0>(W);for_i(W.size(), [&](int i) {g[i] += dW[i] * dW[i];W[i] -= alpha * dW[i] / (std::sqrt(g[i]) + eps);});}

对MNIST中的60000个训练样本，依次执行上面的操作，并更新权值和偏置。

每此循环执行完60000个训练样本，会对10000个测试样本，进行测试，获得识别率。

共迭代30次，然后将最终的权值、偏置等相关参数保持到指定的文件中。