mirror of
https://github.com/ZLMediaKit/ZLMediaKit.git
synced 2024-10-31 00:37:39 +08:00
http文件服务器mmap方案采用共享方式
This commit is contained in:
ziyue
parent
72caa43c97
commit
ba213346bc
@ -9,16 +9,23 @@
|
||||
*/
|
||||
|
||||
#include <csignal>
|
||||
#include "HttpBody.h"
|
||||
#include "Util/util.h"
|
||||
#include "Util/File.h"
|
||||
#include "Util/uv_errno.h"
|
||||
#include "Util/logger.h"
|
||||
#include "Util/onceToken.h"
|
||||
#include "HttpClient.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
#include <sys/sendfile.h>
|
||||
#endif
|
||||
|
||||
#include "Util/File.h"
|
||||
#include "Util/logger.h"
|
||||
#include "Util/onceToken.h"
|
||||
#include "Util/util.h"
|
||||
#include "Util/uv_errno.h"
|
||||
|
||||
#include "HttpBody.h"
|
||||
#include "HttpClient.h"
|
||||
#include "Common/macros.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
#define ENABLE_MMAP
|
||||
@ -29,7 +36,7 @@ using namespace toolkit;
|
||||
|
||||
namespace mediakit {
|
||||
|
||||
HttpStringBody::HttpStringBody(string str){
|
||||
HttpStringBody::HttpStringBody(string str) {
|
||||
_str = std::move(str);
|
||||
}
|
||||
|
||||
@ -39,84 +46,92 @@ ssize_t HttpStringBody::remainSize() {
|
||||
|
||||
Buffer::Ptr HttpStringBody::readData(size_t size) {
|
||||
size = MIN((size_t)remainSize(), size);
|
||||
if(!size){
|
||||
if (!size) {
|
||||
//没有剩余字节了
|
||||
return nullptr;
|
||||
}
|
||||
auto ret = std::make_shared<BufferString>(_str,_offset,size);
|
||||
auto ret = std::make_shared<BufferString>(_str, _offset, size);
|
||||
_offset += size;
|
||||
return ret;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
|
||||
HttpFileBody::HttpFileBody(const string &filePath, bool use_mmap) {
|
||||
std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
|
||||
#ifdef ENABLE_MMAP
|
||||
static std::shared_ptr<char> getSharedMmap(const string &file_path, const std::shared_ptr<FILE> &fp, uint64_t max_size) {
|
||||
static mutex s_mtx;
|
||||
static unordered_map<string /*file_path*/, weak_ptr<char> /*mmap*/> s_shared_mmap;
|
||||
|
||||
{
|
||||
lock_guard<mutex> lck(s_mtx);
|
||||
auto it = s_shared_mmap.find(file_path);
|
||||
if (it != s_shared_mmap.end()) {
|
||||
auto ret = it->second.lock();
|
||||
if (ret) {
|
||||
//命中mmap缓存
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int fd = fileno(fp.get());
|
||||
if (fd < 0) {
|
||||
WarnL << "fileno failed:" << get_uv_errmsg(false);
|
||||
return nullptr;
|
||||
}
|
||||
auto ptr = (char *)mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
WarnL << "mmap " << file_path << " failed:" << get_uv_errmsg(false);
|
||||
return nullptr;
|
||||
}
|
||||
std::shared_ptr<char> ret(ptr, [max_size, fp](char *ptr) { munmap(ptr, max_size); });
|
||||
{
|
||||
lock_guard<mutex> lck(s_mtx);
|
||||
s_shared_mmap[file_path] = ret;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
HttpFileBody::HttpFileBody(const string &file_path, bool use_mmap) {
|
||||
_fp.reset(fopen(file_path.data(), "rb"), [](FILE *fp) {
|
||||
if (fp) {
|
||||
fclose(fp);
|
||||
}
|
||||
});
|
||||
if (!fp) {
|
||||
init(fp, 0, 0, use_mmap);
|
||||
} else {
|
||||
init(fp, 0, File::fileSize(fp.get()), use_mmap);
|
||||
if (!_fp) {
|
||||
//文件不存在
|
||||
_read_to = -1;
|
||||
return;
|
||||
}
|
||||
_read_to = File::fileSize(_fp.get());
|
||||
#ifdef ENABLE_MMAP
|
||||
if (use_mmap && _read_to) {
|
||||
_map_addr = getSharedMmap(file_path, _fp, _read_to);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void HttpFileBody::setRange(uint64_t offset, uint64_t max_size) {
|
||||
CHECK(offset <= _read_to && max_size + offset <= _read_to);
|
||||
_read_to = max_size + offset;
|
||||
_file_offset = offset;
|
||||
if (_fp && !_map_addr) {
|
||||
fseek64(_fp.get(), _file_offset, SEEK_SET);
|
||||
}
|
||||
}
|
||||
|
||||
HttpFileBody::HttpFileBody(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap) {
|
||||
init(fp, offset, max_size, use_mmap);
|
||||
}
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
#include <sys/sendfile.h>
|
||||
#endif
|
||||
|
||||
int HttpFileBody::sendFile(int fd) {
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
static onceToken s_token([]() {
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
});
|
||||
static onceToken s_token([]() { signal(SIGPIPE, SIG_IGN); });
|
||||
off_t off = _file_offset;
|
||||
return sendfile(fd, fileno(_fp.get()), &off, _max_size);
|
||||
return sendfile(fd, fileno(_fp.get()), &off, _read_to - _file_offset);
|
||||
#else
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
void HttpFileBody::init(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap) {
|
||||
_fp = fp;
|
||||
_max_size = max_size;
|
||||
_file_offset = offset;
|
||||
#ifdef ENABLE_MMAP
|
||||
if (use_mmap) {
|
||||
do {
|
||||
if (!_fp) {
|
||||
//文件不存在
|
||||
break;
|
||||
}
|
||||
int fd = fileno(fp.get());
|
||||
if (fd < 0) {
|
||||
WarnL << "fileno failed:" << get_uv_errmsg(false);
|
||||
break;
|
||||
}
|
||||
auto ptr = (char *) mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, offset);
|
||||
if (ptr == MAP_FAILED) {
|
||||
WarnL << "mmap failed:" << get_uv_errmsg(false);
|
||||
break;
|
||||
}
|
||||
_map_addr.reset(ptr, [max_size, fp](char *ptr) {
|
||||
munmap(ptr, max_size);
|
||||
});
|
||||
} while (false);
|
||||
}
|
||||
#endif
|
||||
if (!_map_addr && offset && fp.get()) {
|
||||
//未映射,那么fseek设置偏移量
|
||||
fseek64(fp.get(), offset, SEEK_SET);
|
||||
}
|
||||
}
|
||||
|
||||
class BufferMmap : public Buffer{
|
||||
class BufferMmap : public Buffer {
|
||||
public:
|
||||
typedef std::shared_ptr<BufferMmap> Ptr;
|
||||
BufferMmap(const std::shared_ptr<char> &map_addr, size_t offset, size_t size) {
|
||||
@ -124,103 +139,96 @@ public:
|
||||
_data = map_addr.get() + offset;
|
||||
_size = size;
|
||||
}
|
||||
~BufferMmap() override{};
|
||||
~BufferMmap() override {};
|
||||
//返回数据长度
|
||||
char *data() const override {
|
||||
return _data;
|
||||
}
|
||||
size_t size() const override{
|
||||
return _size;
|
||||
}
|
||||
char *data() const override { return _data; }
|
||||
size_t size() const override { return _size; }
|
||||
|
||||
private:
|
||||
std::shared_ptr<char> _map_addr;
|
||||
char *_data;
|
||||
size_t _size;
|
||||
std::shared_ptr<char> _map_addr;
|
||||
};
|
||||
|
||||
ssize_t HttpFileBody::remainSize() {
|
||||
return _max_size - _offset;
|
||||
return _read_to - _file_offset;
|
||||
}
|
||||
|
||||
Buffer::Ptr HttpFileBody::readData(size_t size) {
|
||||
size = MIN((size_t)remainSize(),size);
|
||||
if(!size){
|
||||
size = MIN((size_t)remainSize(), size);
|
||||
if (!size) {
|
||||
//没有剩余字节了
|
||||
return nullptr;
|
||||
}
|
||||
if(!_map_addr){
|
||||
//fread模式
|
||||
if (!_map_addr) {
|
||||
// fread模式
|
||||
ssize_t iRead;
|
||||
auto ret = _pool.obtain2();
|
||||
ret->setCapacity(size + 1);
|
||||
do{
|
||||
do {
|
||||
iRead = fread(ret->data(), 1, size, _fp.get());
|
||||
}while(-1 == iRead && UV_EINTR == get_uv_error(false));
|
||||
} while (-1 == iRead && UV_EINTR == get_uv_error(false));
|
||||
|
||||
if(iRead > 0){
|
||||
if (iRead > 0) {
|
||||
//读到数据了
|
||||
ret->setSize(iRead);
|
||||
_offset += iRead;
|
||||
_file_offset += iRead;
|
||||
return std::move(ret);
|
||||
}
|
||||
//读取文件异常，文件真实长度小于声明长度
|
||||
_offset = _max_size;
|
||||
_file_offset = _read_to;
|
||||
WarnL << "read file err:" << get_uv_errmsg();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//mmap模式
|
||||
auto ret = std::make_shared<BufferMmap>(_map_addr,_offset,size);
|
||||
_offset += size;
|
||||
// mmap模式
|
||||
auto ret = std::make_shared<BufferMmap>(_map_addr, _file_offset, size);
|
||||
_file_offset += size;
|
||||
return ret;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
HttpMultiFormBody::HttpMultiFormBody(const HttpArgs &args,const string &filePath,const string &boundary){
|
||||
std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
|
||||
if(fp){
|
||||
fclose(fp);
|
||||
}
|
||||
});
|
||||
if(!fp){
|
||||
|
||||
HttpMultiFormBody::HttpMultiFormBody(const HttpArgs &args, const string &filePath, const string &boundary) {
|
||||
_fileBody = std::make_shared<HttpFileBody>(filePath);
|
||||
if (_fileBody->remainSize() < 0) {
|
||||
throw std::invalid_argument(StrPrinter << "open file failed：" << filePath << " " << get_uv_errmsg());
|
||||
}
|
||||
_fileBody = std::make_shared<HttpFileBody>(fp, 0, File::fileSize(fp.get()));
|
||||
|
||||
auto fileName = filePath;
|
||||
auto pos = filePath.rfind('/');
|
||||
if(pos != string::npos){
|
||||
if (pos != string::npos) {
|
||||
fileName = filePath.substr(pos + 1);
|
||||
}
|
||||
_bodyPrefix = multiFormBodyPrefix(args,boundary,fileName);
|
||||
_bodyPrefix = multiFormBodyPrefix(args, boundary, fileName);
|
||||
_bodySuffix = multiFormBodySuffix(boundary);
|
||||
_totalSize = _bodyPrefix.size() + _bodySuffix.size() + _fileBody->remainSize();
|
||||
_totalSize = _bodyPrefix.size() + _bodySuffix.size() + _fileBody->remainSize();
|
||||
}
|
||||
|
||||
ssize_t HttpMultiFormBody::remainSize() {
|
||||
return _totalSize - _offset;
|
||||
}
|
||||
|
||||
Buffer::Ptr HttpMultiFormBody::readData(size_t size){
|
||||
if(_bodyPrefix.size()){
|
||||
Buffer::Ptr HttpMultiFormBody::readData(size_t size) {
|
||||
if (_bodyPrefix.size()) {
|
||||
auto ret = std::make_shared<BufferString>(_bodyPrefix);
|
||||
_offset += _bodyPrefix.size();
|
||||
_bodyPrefix.clear();
|
||||
return ret;
|
||||
}
|
||||
|
||||
if(_fileBody->remainSize()){
|
||||
if (_fileBody->remainSize()) {
|
||||
auto ret = _fileBody->readData(size);
|
||||
if(!ret){
|
||||
if (!ret) {
|
||||
//读取文件出现异常，提前中断
|
||||
_offset = _totalSize;
|
||||
}else{
|
||||
} else {
|
||||
_offset += ret->size();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
if(_bodySuffix.size()){
|
||||
if (_bodySuffix.size()) {
|
||||
auto ret = std::make_shared<BufferString>(_bodySuffix);
|
||||
_offset = _totalSize;
|
||||
_bodySuffix.clear();
|
||||
@ -230,7 +238,7 @@ Buffer::Ptr HttpMultiFormBody::readData(size_t size){
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
string HttpMultiFormBody::multiFormBodySuffix(const string &boundary){
|
||||
string HttpMultiFormBody::multiFormBodySuffix(const string &boundary) {
|
||||
string MPboundary = string("--") + boundary;
|
||||
string endMPboundary = MPboundary + "--";
|
||||
_StrPrinter body;
|
||||
@ -238,21 +246,23 @@ string HttpMultiFormBody::multiFormBodySuffix(const string &boundary){
|
||||
return std::move(body);
|
||||
}
|
||||
|
||||
string HttpMultiFormBody::multiFormContentType(const string &boundary){
|
||||
string HttpMultiFormBody::multiFormContentType(const string &boundary) {
|
||||
return StrPrinter << "multipart/form-data; boundary=" << boundary;
|
||||
}
|
||||
|
||||
string HttpMultiFormBody::multiFormBodyPrefix(const HttpArgs &args,const string &boundary,const string &fileName){
|
||||
string HttpMultiFormBody::multiFormBodyPrefix(const HttpArgs &args, const string &boundary, const string &fileName) {
|
||||
string MPboundary = string("--") + boundary;
|
||||
_StrPrinter body;
|
||||
for(auto &pr : args){
|
||||
for (auto &pr : args) {
|
||||
body << MPboundary << "\r\n";
|
||||
body << "Content-Disposition: form-data; name=\"" << pr.first << "\"\r\n\r\n";
|
||||
body << pr.second << "\r\n";
|
||||
}
|
||||
body << MPboundary << "\r\n";
|
||||
body << "Content-Disposition: form-data; name=\"" << "file" << "\";filename=\"" << fileName << "\"\r\n";
|
||||
body << "Content-Type: application/octet-stream\r\n\r\n" ;
|
||||
body << "Content-Disposition: form-data; name=\""
|
||||
<< "file"
|
||||
<< "\";filename=\"" << fileName << "\"\r\n";
|
||||
body << "Content-Type: application/octet-stream\r\n\r\n";
|
||||
return std::move(body);
|
||||
}
|
||||
|
||||
@ -268,4 +278,4 @@ Buffer::Ptr HttpBufferBody::readData(size_t size) {
|
||||
return Buffer::Ptr(std::move(_buffer));
|
||||
}
|
||||
|
||||
}//namespace mediakit
|
||||
} // namespace mediakit
|
||||
|
||||
@ -104,32 +104,32 @@ private:
|
||||
/**
|
||||
* 文件类型的content
|
||||
*/
|
||||
class HttpFileBody : public HttpBody{
|
||||
class HttpFileBody : public HttpBody {
|
||||
public:
|
||||
typedef std::shared_ptr<HttpFileBody> Ptr;
|
||||
|
||||
/**
|
||||
* 构造函数
|
||||
* @param fp 文件句柄，文件的偏移量必须为0
|
||||
* @param offset 相对文件头的偏移量
|
||||
* @param max_size 最大读取字节数，未判断是否大于文件真实大小
|
||||
* @param file_path 文件路径
|
||||
* @param use_mmap 是否使用mmap方式访问文件
|
||||
*/
|
||||
HttpFileBody(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap = true);
|
||||
HttpFileBody(const std::string &file_path, bool use_mmap = true);
|
||||
~HttpFileBody() override = default;
|
||||
|
||||
ssize_t remainSize() override ;
|
||||
/**
|
||||
* 设置读取范围
|
||||
* @param offset 相对文件头的偏移量
|
||||
* @param max_size 最大读取字节数
|
||||
*/
|
||||
void setRange(uint64_t offset, uint64_t max_size);
|
||||
|
||||
ssize_t remainSize() override;
|
||||
toolkit::Buffer::Ptr readData(size_t size) override;
|
||||
int sendFile(int fd) override;
|
||||
|
||||
private:
|
||||
void init(const std::shared_ptr<FILE> &fp,size_t offset,size_t max_size, bool use_mmap);
|
||||
|
||||
private:
|
||||
size_t _max_size;
|
||||
size_t _offset = 0;
|
||||
size_t _file_offset = 0;
|
||||
int64_t _read_to = 0;
|
||||
uint64_t _file_offset = 0;
|
||||
std::shared_ptr<FILE> _fp;
|
||||
std::shared_ptr<char> _map_addr;
|
||||
toolkit::ResourcePool<toolkit::BufferRaw> _pool;
|
||||
|
||||
@ -582,13 +582,8 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader,
|
||||
const string &filePath,
|
||||
bool use_mmap) const {
|
||||
StrCaseMap &httpHeader = const_cast<StrCaseMap &>(responseHeader);
|
||||
std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
|
||||
if (fp) {
|
||||
fclose(fp);
|
||||
}
|
||||
});
|
||||
|
||||
if (!fp) {
|
||||
auto fileBody = std::make_shared<HttpFileBody>(filePath, use_mmap);
|
||||
if (fileBody->remainSize() < 0) {
|
||||
//打开文件失败
|
||||
GET_CONFIG(string, notFound, Http::kNotFound);
|
||||
GET_CONFIG(string, charSet, Http::kCharSet);
|
||||
@ -600,29 +595,23 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader,
|
||||
}
|
||||
|
||||
auto &strRange = const_cast<StrCaseMap &>(requestHeader)["Range"];
|
||||
size_t iRangeStart = 0;
|
||||
size_t iRangeEnd = 0;
|
||||
size_t fileSize = File::fileSize(fp.get());
|
||||
|
||||
int code;
|
||||
if (strRange.size() == 0) {
|
||||
//全部下载
|
||||
code = 200;
|
||||
iRangeEnd = fileSize - 1;
|
||||
} else {
|
||||
int code = 200;
|
||||
if (!strRange.empty()) {
|
||||
//分节下载
|
||||
code = 206;
|
||||
iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data());
|
||||
iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data());
|
||||
auto iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data());
|
||||
auto iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data());
|
||||
auto fileSize = fileBody->remainSize();
|
||||
if (iRangeEnd == 0) {
|
||||
iRangeEnd = fileSize - 1;
|
||||
}
|
||||
//设置文件范围
|
||||
fileBody->setRange(iRangeStart, iRangeEnd - iRangeStart + 1);
|
||||
//分节下载返回Content-Range头
|
||||
httpHeader.emplace("Content-Range", StrPrinter << "bytes " << iRangeStart << "-" << iRangeEnd << "/" << fileSize << endl);
|
||||
}
|
||||
|
||||
//回复文件
|
||||
HttpBody::Ptr fileBody = std::make_shared<HttpFileBody>(fp, iRangeStart, iRangeEnd - iRangeStart + 1, use_mmap);
|
||||
(*this)(code, httpHeader, fileBody);
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user