吴恩达老师深度学习视频课笔记：逻辑回归公式推导及C++实现

逻辑回归(Logistic Regression)是一个二分分类算法。逻辑回归的目标是最小化其预测与训练数据之间的误差。为了训练逻辑回归模型中的参数w和b，需要定义一个成本函数(cost function)。

成本函数(cost function)：它是针对整个训练集的。衡量参数w和b在整个训练集上的效果。

损失函数或误差函数(loss function or error function)：它是针对单个训练样本进行定义的。可以用来衡量算法的效果，衡量预测输出值与实际值有多接近。

梯度下降法的核心是最小化成本函数。使用梯度下降法可以找到一个函数的局部极小值。

关于逻辑回归的介绍可以参考： http://blog.csdn.net/fengbingchun/article/details/78283675

关于梯度下降法的介绍可以参考： http://blog.csdn.net/fengbingchun/article/details/75351323

关于激活函数sigmoid函数的介绍可以参考： http://blog.csdn.net/fengbingchun/article/details/73848734

关于MNIST数据集的介绍可以参考：  http://blog.csdn.net/fengbingchun/article/details/49611549

以下截图来自吴恩达老师深度学习视频课：

以下code是完全按照上面的推导公式进行实现的，训练数据集为从MNIST中train中随机选取的0、1各10个图像；测试数据集为从MNIST中test中随机选取的0、1各10个图像，如下图，其中第一排前10个0用于训练，后10个0用于测试；第二排前10个1用于训练，后10个1用于测试：

logistic_regression2.hpp:

#ifndef FBC_SRC_NN_LOGISTIC_REGRESSION2_HPP_
#define FBC_SRC_NN_LOGISTIC_REGRESSION2_HPP_#include <vector>
#include <string>namespace ANN {template<typename T>
class LogisticRegression2 { // two categories
public:LogisticRegression2() = default;int init(const T* data, const T* labels, int train_num, int feature_length, T learning_rate = 0.00001, int iterations = 10000);int train(const std::string& model);int load_model(const std::string& model);T predict(const T* data, int feature_length) const; // y = 1/(1+exp(-(wx+b)))private:int store_model(const std::string& model) const;T calculate_sigmoid(T value) const; // y = 1/(1+exp(-value))T calculate_z(const std::vector<T>& feature) const;std::vector<std::vector<T>> x; // training setstd::vector<T> y; // ground truth labelsint iterations = 1000;int m = 0; // train samples numint feature_length = 0;T alpha = (T)0.00001; // learning ratestd::vector<T> w; // weightsT b = (T)0.; // threshold
}; // class LogisticRegression2} // namespace ANN#endif // FBC_SRC_NN_LOGISTIC_REGRESSION2_HPP_

logistic_regression2.cpp:

#include "logistic_regression2.hpp"
#include <fstream>
#include <algorithm>
#include <random>
#include <cmath>
#include "common.hpp"namespace ANN {template<typename T>
int LogisticRegression2<T>::init(const T* data, const T* labels, int train_num, int feature_length, T learning_rate, int iterations)
{if (train_num < 2) {fprintf(stderr, "logistic regression train samples num is too little: %d\n", train_num);return -1;}if (learning_rate <= 0) {fprintf(stderr, "learning rate must be greater 0: %f\n", learning_rate);return -1;}if (iterations <= 0) {fprintf(stderr, "number of iterations cannot be zero or a negative number: %d\n", iterations);return -1;}this->alpha = learning_rate;this->iterations = iterations;this->m = train_num;this->feature_length = feature_length;this->x.resize(train_num);this->y.resize(train_num);for (int i = 0; i < train_num; ++i) {const T* p = data + i * feature_length;this->x[i].resize(feature_length);for (int j = 0; j < feature_length; ++j) {this->x[i][j] = p[j];}this->y[i] = labels[i];}return 0;
}template<typename T>
T LogisticRegression2<T>::calculate_z(const std::vector<T>& feature) const
{T z{ 0. };for (int i = 0; i < this->feature_length; ++i) {z += w[i] * feature[i];}z += b;return z;
}template<typename T>
int LogisticRegression2<T>::train(const std::string& model)
{CHECK(x.size() == y.size());w.resize(this->feature_length, (T)0.);std::random_device rd;std::mt19937 generator(rd());std::uniform_real_distribution<T> distribution(-0.1, 0.1);for (int i = 0; i < this->feature_length; ++i) {w[i] = distribution(generator);}b = distribution(generator);for (int iter = 0; iter < this->iterations; ++iter) {T J = (T)0., db = (T)0.;std::vector<T> dw(this->feature_length, (T)0.);std::vector<T> z(this->m, (T)0), a(this->m, (T)0), dz(this->m, (T)0);for (int i = 0; i < this->m; ++i) {z[i] = calculate_z(x[i]); // z(i)=w^T*x(i)+ba[i] = calculate_sigmoid(z[i]); // a(i)= 1/(1+e^(-z(i)))J += -(y[i] * std::log(a[i]) + (1 - y[i] * std::log(1 - a[i]))); // J+=-[y(i)*loga(i)+(1-y(i))*log(1-a(i))]dz[i] = a[i] - y[i]; // dz(i) = a(i)-y(i)for (int j = 0; j < this->feature_length; ++j) {dw[j] += x[i][j] * dz[i]; // dw(i)+=x(i)(j)*dz(i)}db += dz[i]; // db+=dz(i)}J /= this->m;for (int j = 0; j < this->feature_length; ++j) {dw[j] /= m;}db /= m;for (int j = 0; j < this->feature_length; ++j) {w[j] -= this->alpha * dw[j];}b -= this->alpha*db;}CHECK(store_model(model) == 0);return 0;
}template<typename T>
int LogisticRegression2<T>::load_model(const std::string& model)
{std::ifstream file;file.open(model.c_str(), std::ios::binary);if (!file.is_open()) {fprintf(stderr, "open file fail: %s\n", model.c_str());return -1;}int length{ 0 };file.read((char*)&length, sizeof(length));this->w.resize(length);this->feature_length = length;file.read((char*)this->w.data(), sizeof(T)*this->w.size());file.read((char*)&this->b, sizeof(T));file.close();return 0;
}template<typename T>
T LogisticRegression2<T>::predict(const T* data, int feature_length) const
{CHECK(feature_length == this->feature_length);T value{ (T)0. };for (int t = 0; t < this->feature_length; ++t) {value += data[t] * this->w[t];}value += this->b;return (calculate_sigmoid(value));
}template<typename T>
int LogisticRegression2<T>::store_model(const std::string& model) const
{std::ofstream file;file.open(model.c_str(), std::ios::binary);if (!file.is_open()) {fprintf(stderr, "open file fail: %s\n", model.c_str());return -1;}int length = w.size();file.write((char*)&length, sizeof(length));file.write((char*)w.data(), sizeof(T) * w.size());file.write((char*)&b, sizeof(T));file.close();return 0;
}template<typename T>
T LogisticRegression2<T>::calculate_sigmoid(T value) const
{return ((T)1 / ((T)1 + exp(-value)));
}template class LogisticRegression2<float>;
template class LogisticRegression2<double>;} // namespace ANN

main.cpp:

#include "funset.hpp"
#include <iostream>
#include "perceptron.hpp"
#include "BP.hpp""
#include "CNN.hpp"
#include "linear_regression.hpp"
#include "naive_bayes_classifier.hpp"
#include "logistic_regression.hpp"
#include "common.hpp"
#include "knn.hpp"
#include "decision_tree.hpp"
#include "pca.hpp"
#include <opencv2/opencv.hpp>
#include "logistic_regression2.hpp"// ================================ logistic regression =====================
int test_logistic_regression2_train()
{const std::string image_path{ "E:/GitCode/NN_Test/data/images/digit/handwriting_0_and_1/" };cv::Mat data, labels;for (int i = 1; i < 11; ++i) {const std::vector<std::string> label{ "0_", "1_" };for (const auto& value : label) {std::string name = std::to_string(i);name = image_path + value + name + ".jpg";cv::Mat image = cv::imread(name, 0);if (image.empty()) {fprintf(stderr, "read image fail: %s\n", name.c_str());return -1;}data.push_back(image.reshape(0, 1));}}data.convertTo(data, CV_32F);std::unique_ptr<float[]> tmp(new float[20]);for (int i = 0; i < 20; ++i) {if (i % 2 == 0) tmp[i] = 0.f;else tmp[i] = 1.f;}labels = cv::Mat(20, 1, CV_32FC1, tmp.get());ANN::LogisticRegression2<float> lr;const float learning_rate{ 0.0001f };const int iterations{ 10000 };int ret = lr.init((float*)data.data, (float*)labels.data, data.rows, data.cols);if (ret != 0) {fprintf(stderr, "logistic regression init fail: %d\n", ret);return -1;}const std::string model{ "E:/GitCode/NN_Test/data/logistic_regression2.model" };ret = lr.train(model);if (ret != 0) {fprintf(stderr, "logistic regression train fail: %d\n", ret);return -1;}return 0;
}int test_logistic_regression2_predict()
{const std::string image_path{ "E:/GitCode/NN_Test/data/images/digit/handwriting_0_and_1/" };cv::Mat data, labels, result;for (int i = 11; i < 21; ++i) {const std::vector<std::string> label{ "0_", "1_" };for (const auto& value : label) {std::string name = std::to_string(i);name = image_path + value + name + ".jpg";cv::Mat image = cv::imread(name, 0);if (image.empty()) {fprintf(stderr, "read image fail: %s\n", name.c_str());return -1;}data.push_back(image.reshape(0, 1));}}data.convertTo(data, CV_32F);std::unique_ptr<int[]> tmp(new int[20]);for (int i = 0; i < 20; ++i) {if (i % 2 == 0) tmp[i] = 0;else tmp[i] = 1;}labels = cv::Mat(20, 1, CV_32SC1, tmp.get());CHECK(data.rows == labels.rows);const std::string model{ "E:/GitCode/NN_Test/data/logistic_regression2.model" };ANN::LogisticRegression2<float> lr;int ret = lr.load_model(model);if (ret != 0) {fprintf(stderr, "load logistic regression model fail: %d\n", ret);return -1;}for (int i = 0; i < data.rows; ++i) {float probability = lr.predict((float*)(data.row(i).data), data.cols);fprintf(stdout, "probability: %.6f, ", probability);if (probability > 0.5) fprintf(stdout, "predict result: 1, ");else fprintf(stdout, "predict result: 0, ");fprintf(stdout, "actual result: %d\n", ((int*)(labels.row(i).data))[0]);}return 0;
}

测试结果如下：由执行结果可知，测试图像全部分类正确。由于w和b初始值是随机产生的，因此每次执行的结果多少有些差异。

GitHub： https://github.com/fengbingchun/NN_Test