Caffe源码中io文件分析

Caffe源码(caffe version commit: 09868ac , date: 2015.08.15)中有一些重要的头文件，这里介绍下include/caffe/util/io.hpp文件的内容：

1.      include文件：

(1)、<google/protobuf/message.h>：关于protobuf的介绍可以参考：http://blog.csdn.net/fengbingchun/article/details/49977903

(2)、<caffe/blob.hpp>：此文件的介绍可以参考：http://blog.csdn.net/fengbingchun/article/details/59106613

(3)、<caffe/common.hpp>：此文件的介绍可以参考：http://blog.csdn.net/fengbingchun/article/details/54955236

(4)、<caffe/proto/caffe.pb.h>：此文件的介绍可以参考：http://blog.csdn.net/fengbingchun/article/details/55267162

2.      <mkstemp.h>文件：

此文件是来自于libc/sysdeps/posix/tempname.c，此文件中包含一个静态全局常量letters和一个基于TMPL生成临时文件名的函数mkstemp。此文件在源码中没有用到，仅在test文件中用到。此文件仅在Linux下使用，在Windows下直接使用会有问题，而且在新版的Caffe中，已将此文件移除。

3.      函数：

此文件中的函数主要作用包括：读取prototxt文本文件并解析(反序列化)；生成prototxt文本文件即序列化；读取caffemodel二进制文件并解析(反序列化)；生成caffemodel二进制文件即序列化；读取二进制文件、图像文件到Datum类；对Datum进行解码；将图像读取到cv::Mat；解码Datum到cv::Mat；将cv::Mat转换为Datum。

Datum既可以直接存储解码前的数据，也可以存储解码后的数据。Datum支持两种数据类型string和float。

<caffe/util/io.hpp>文件的详细介绍如下：

#ifndef CAFFE_UTIL_IO_H_
#define CAFFE_UTIL_IO_H_#include <unistd.h>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>#include "google/protobuf/message.h"#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"#include "mkstemp.h"namespace caffe {using ::google::protobuf::Message;// 生成临时文件，在windows下不能直接调用此文件
// 此函数在新版的Caffe中，在windows下可以直接调用
inline void MakeTempFilename(string* temp_filename) {temp_filename->clear();*temp_filename = "/tmp/caffe_test.XXXXXX";char* temp_filename_cstr = new char[temp_filename->size() + 1];// NOLINT_NEXT_LINE(runtime/printf)strcpy(temp_filename_cstr, temp_filename->c_str());int fd = mkstemp(temp_filename_cstr);CHECK_GE(fd, 0) << "Failed to open a temporary file at: " << *temp_filename;
#ifndef _MSC_VERclose(fd);
#else_close(fd);
#endif*temp_filename = temp_filename_cstr;delete[] temp_filename_cstr;
}// 生成临时目录，在windows下不能直接调用此文件
// 此函数在新版的Caffe中，在windows下可以直接调用
inline void MakeTempDir(string* temp_dirname) {temp_dirname->clear();*temp_dirname = "/tmp/caffe_test.XXXXXX";char* temp_dirname_cstr = new char[temp_dirname->size() + 1];// NOLINT_NEXT_LINE(runtime/printf)strcpy(temp_dirname_cstr, temp_dirname->c_str());//char* mkdtemp_result = mkdtemp(temp_dirname_cstr);
#ifndef _MSC_VER char* mkdtemp_result = mkdtemp(temp_dirname_cstr);
#elseerrno_t mkdtemp_result = _mktemp_s(temp_dirname_cstr, sizeof(temp_dirname_cstr));
#endifCHECK(mkdtemp_result != NULL)<< "Failed to create a temporary directory at: " << *temp_dirname;*temp_dirname = temp_dirname_cstr;delete[] temp_dirname_cstr;
}// 读取prototxt文本文件并解析(反序列化)
bool ReadProtoFromTextFile(const char* filename, Message* proto);
// 读取prototxt文本文件并解析(反序列化)
inline bool ReadProtoFromTextFile(const string& filename, Message* proto) {return ReadProtoFromTextFile(filename.c_str(), proto);
}
// 读取prototxt文本文件并解析(反序列化)
inline void ReadProtoFromTextFileOrDie(const char* filename, Message* proto) {CHECK(ReadProtoFromTextFile(filename, proto));
}
// 读取prototxt文本文件并解析(反序列化)
inline void ReadProtoFromTextFileOrDie(const string& filename, Message* proto) {ReadProtoFromTextFileOrDie(filename.c_str(), proto);
}// 生成prototxt文本文件，即序列化
void WriteProtoToTextFile(const Message& proto, const char* filename);
// 生成prototxt文本文件，即序列化
inline void WriteProtoToTextFile(const Message& proto, const string& filename) {WriteProtoToTextFile(proto, filename.c_str());
}// 读取caffemodel二进制文件并解析(反序列化)
bool ReadProtoFromBinaryFile(const char* filename, Message* proto);
// 读取caffemodel二进制文件并解析(反序列化)
inline bool ReadProtoFromBinaryFile(const string& filename, Message* proto) {return ReadProtoFromBinaryFile(filename.c_str(), proto);
}
// 读取caffemodel二进制文件并解析(反序列化)
inline void ReadProtoFromBinaryFileOrDie(const char* filename, Message* proto) {CHECK(ReadProtoFromBinaryFile(filename, proto));
}
// 读取caffemodel二进制文件并解析(反序列化)
inline void ReadProtoFromBinaryFileOrDie(const string& filename, Message* proto) {ReadProtoFromBinaryFileOrDie(filename.c_str(), proto);
}// 生成caffemodel二进制文件，即序列化
void WriteProtoToBinaryFile(const Message& proto, const char* filename);
// 生成caffemodel二进制文件，即序列化
inline void WriteProtoToBinaryFile(const Message& proto, const string& filename) {WriteProtoToBinaryFile(proto, filename.c_str());
}// 注：以下的Datum是定义在caffe.proto中的一个message，其字段有channels、height、width、data、label、float_data、encoded
// 读取二进制数据文件到Datum类
bool ReadFileToDatum(const string& filename, const int label, Datum* datum);
// 读取二进制数据文件到Datum类
inline bool ReadFileToDatum(const string& filename, Datum* datum) {return ReadFileToDatum(filename, -1, datum);
}// 读取图像文件到Datum类
bool ReadImageToDatum(const string& filename, const int label,const int height, const int width, const bool is_color,const std::string & encoding, Datum* datum);
// 读取图像文件到Datum类
inline bool ReadImageToDatum(const string& filename, const int label,const int height, const int width, const bool is_color, Datum* datum) {return ReadImageToDatum(filename, label, height, width, is_color,"", datum);
}
// 读取图像文件到Datum类
inline bool ReadImageToDatum(const string& filename, const int label,const int height, const int width, Datum* datum) {return ReadImageToDatum(filename, label, height, width, true, datum);
}
// 读取图像文件到Datum类
inline bool ReadImageToDatum(const string& filename, const int label,const bool is_color, Datum* datum) {return ReadImageToDatum(filename, label, 0, 0, is_color, datum);
}
// 读取图像文件到Datum类
inline bool ReadImageToDatum(const string& filename, const int label, Datum* datum) {return ReadImageToDatum(filename, label, 0, 0, true, datum);
}
// 读取图像文件到Datum类
inline bool ReadImageToDatum(const string& filename, const int label,const std::string & encoding, Datum* datum) {return ReadImageToDatum(filename, label, 0, 0, true, encoding, datum);
}// 对Datum进行解码
bool DecodeDatumNative(Datum* datum);
// 对Datum进行解码
bool DecodeDatum(Datum* datum, bool is_color);// 将图像读取到cv::Mat(可选择读取灰度图还是彩色图、可进行图像缩放操作)
cv::Mat ReadImageToCVMat(const string& filename,const int height, const int width, const bool is_color);
// 将图像读取到cv::Mat
cv::Mat ReadImageToCVMat(const string& filename,const int height, const int width);
// 将图像读取到cv::Mat
cv::Mat ReadImageToCVMat(const string& filename,const bool is_color);
// 将图像读取到cv::Mat
cv::Mat ReadImageToCVMat(const string& filename);// 解码Datum到cv::Mat
cv::Mat DecodeDatumToCVMatNative(const Datum& datum);
// 解码Datum到cv::Mat
cv::Mat DecodeDatumToCVMat(const Datum& datum, bool is_color);// 将cv::Mat转换为Datum
void CVMatToDatum(const cv::Mat& cv_img, Datum* datum);}  // namespace caffe#endif   // CAFFE_UTIL_IO_H_

在caffe.proto文件中，有一个message是与io相关的，如下：

message Datum { // 存储图像数据，可以解码前的即编码，也可以是解码后的optional int32 channels = 1; // 图像通道数optional int32 height = 2; // 图像高optional int32 width = 3; // 图像宽// the actual image data, in bytesoptional bytes data = 4; // 以std::string类型存储图像数据optional int32 label = 5; // 当前图像对应的label值// Optionally, the datum could also hold float data.repeated float float_data = 6; // 以float类型存储图像数据// If true data contains an encoded image that need to be decodedoptional bool encoded = 7 [default = false]; // 是否是编码数据
}

io的测试代码如下：

#include "funset.hpp"
#include <string>
#include <vector>
#include "common.hpp"static bool ReadImageToDatumReference(const std::string& filename, const int label,const int height, const int width, const bool is_color, caffe::Datum* datum)
{cv::Mat cv_img;int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);cv::Mat cv_img_origin = cv::imread(filename, cv_read_flag);if (!cv_img_origin.data) {fprintf(stderr, "Could not open or find file: %s\n", filename.c_str());return false;}if (height > 0 && width > 0)cv::resize(cv_img_origin, cv_img, cv::Size(width, height));elsecv_img = cv_img_origin;int num_channels = (is_color ? 3 : 1);datum->set_channels(num_channels);datum->set_height(cv_img.rows);datum->set_width(cv_img.cols);datum->set_label(label);datum->clear_data();datum->clear_float_data();std::string* datum_string = datum->mutable_data();if (is_color) {for (int c = 0; c < num_channels; ++c) {for (int h = 0; h < cv_img.rows; ++h) {for (int w = 0; w < cv_img.cols; ++w) {datum_string->push_back(static_cast<char>(cv_img.at<cv::Vec3b>(h, w)[c]));}}}} else {  // Faster than repeatedly testing is_color for each pixel w/i loopfor (int h = 0; h < cv_img.rows; ++h) {for (int w = 0; w < cv_img.cols; ++w) {datum_string->push_back(static_cast<char>(cv_img.at<uchar>(h, w)));}}}return true;
}static int CompareDatumMat(const caffe::Datum& datum1, const caffe::Datum& datum2)
{if (datum1.channels() != datum2.channels() || datum1.height() != datum2.height() ||datum1.width() != datum2.width() || datum1.data().size() != datum2.data().size()) {fprintf(stderr, "these values should be equal\n");return -1;}const std::string& data1 = datum1.data();const std::string& data2 = datum2.data();for (int i = 0; i < datum1.data().size(); ++i) {if (data1[i] != data2[i]) {fprintf(stderr, "their data should be equal\n");return -1;}}return 0;
}static int CompareDatumMat(const caffe::Datum& datum, const cv::Mat& mat)
{if (datum.channels() != mat.channels() || datum.height() != mat.rows || datum.width() != mat.cols) {fprintf(stderr, "these values should be equal\n");return -1;}const std::string& datum_data = datum.data();int index = 0;for (int c = 0; c < mat.channels(); ++c) {for (int h = 0; h < mat.rows; ++h) {for (int w = 0; w < mat.cols; ++w) {if (datum_data[index++] != static_cast<char>(mat.at<cv::Vec3b>(h, w)[c])) {fprintf(stderr, "their data should be equal\n");return -1;}}}}return 0;
}static int CompareDatumMat(const cv::Mat& mat1, const cv::Mat& mat2)
{if (mat1.channels() != mat2.channels() || mat1.rows != mat2.rows || mat1.cols != mat2.cols) {fprintf(stderr, "these values should be equal\n");return -1;}for (int c = 0; c < mat1.channels(); ++c) {for (int h = 0; h < mat1.rows; ++h) {for (int w = 0; w < mat1.cols; ++w) {if (mat1.at<cv::Vec3b>(h, w)[c] != mat2.at<cv::Vec3b>(h, w)[c]) {fprintf(stderr, "their data should be equal\n");return -1;}}}}return 0;
}int test_caffe_util_io()
{std::string filename{ "E:/GitCode/Caffe_Test/test_data/images/a.jpg" };// 1. caffe::ReadImageToDatumcaffe::Datum datum1;caffe::ReadImageToDatum(filename, 0, &datum1);fprintf(stderr, "datum1: channels: %d, height: %d, width: %d\n",datum1.channels(), datum1.height(), datum1.width());// 2. test ReadImageToDatumReferencecaffe::Datum datum2, datum_ref2;caffe::ReadImageToDatum(filename, 0, 0, 0, true, &datum2);ReadImageToDatumReference(filename, 0, 0, 0, true, &datum_ref2);if(CompareDatumMat(datum2, datum_ref2) != 0) return -1;// 3. test ReadImageToDatumReferenceResizedcaffe::Datum datum3, datum_ref3;caffe::ReadImageToDatum(filename, 0, 100, 200, true, &datum3);ReadImageToDatumReference(filename, 0, 100, 200, true, &datum_ref3);if (CompareDatumMat(datum3, datum_ref3) != 0) return -1;// 4. test ReadImageToDatumContentcaffe::Datum datum4;caffe::ReadImageToDatum(filename, 0, &datum4);cv::Mat cv_img = caffe::ReadImageToCVMat(filename);if (CompareDatumMat(datum4, cv_img) != 0) return -1;// 5. test CVMatToDatumContentcv_img = caffe::ReadImageToCVMat(filename);caffe::Datum datum5;caffe::CVMatToDatum(cv_img, &datum5);caffe::Datum datum_ref5;caffe::ReadImageToDatum(filename, 0, &datum_ref5);if (CompareDatumMat(datum5, datum_ref5) != 0) return -1;// 6. test ReadFileToDatumcaffe::Datum datum6;if (!caffe::ReadFileToDatum(filename, &datum6)) {fprintf(stderr, "read file to datum fail\n");return -1;}fprintf(stderr, "datum encoded: %d; datum label: %d, datum size: %d\n",datum6.encoded(), datum6.label(), datum6.data().size());// 7. test DecodeDatumcaffe::Datum datum7;caffe::ReadFileToDatum(filename, &datum7);if (!caffe::DecodeDatum(&datum7, true)) return -1;if(caffe::DecodeDatum(&datum7, true)) return -1;caffe::Datum datum_ref7;ReadImageToDatumReference(filename, 0, 0, 0, true, &datum_ref7);if (CompareDatumMat(datum7, datum_ref7) != 0) return -1;// 8. test DecodeDatumToCVMatContentcaffe::Datum datum8;if (!caffe::ReadImageToDatum(filename, 0, std::string("jpg"), &datum8)) return -1;cv::Mat cv_img8 = caffe::DecodeDatumToCVMat(datum8, true);cv::Mat cv_img_ref = caffe::ReadImageToCVMat(filename);// if (CompareDatumMat(cv_img8, cv_img_ref) != 0) return -1; // Note: some values are not equal// 9. read prototxt and parsestd::string solver_prototxt{ "E:/GitCode/Caffe_Test/test_data/model/mnist/lenet_solver.prototxt" };caffe::SolverParameter solver_param;if (!caffe::ReadProtoFromTextFile(solver_prototxt.c_str(), &solver_param)) {fprintf(stderr, "parse solver.prototxt fail\n");return -1;}// 10. write prototxt to text filestd::string save_prototxt{"E:/GitCode/Caffe_Test/test_data/test.prototxt"};caffe::WriteProtoToTextFile(solver_param, save_prototxt);return 0;
}

测试结果如下：