/* * Copyright (c) 2016 The ZLMediaKit project authors. All Rights Reserved. * * This file is part of ZLMediaKit(https://github.com/xia-chu/ZLMediaKit). * * Use of this source code is governed by MIT license that can be found in the * LICENSE file in the root of the source tree. All contributing project authors * may be found in the AUTHORS file in the root of the source tree. */ #include #include "Util/base64.h" #include "HttpClient.h" #include "Common/config.h" namespace mediakit { void HttpClient::sendRequest(const string &strUrl, float fTimeOutSec) { _aliveTicker.resetTime(); _url = strUrl; auto protocol = FindField(strUrl.data(), NULL, "://"); uint16_t defaultPort; bool isHttps; if (strcasecmp(protocol.data(), "http") == 0) { defaultPort = 80; isHttps = false; } else if (strcasecmp(protocol.data(), "https") == 0) { defaultPort = 443; isHttps = true; } else { auto strErr = StrPrinter << "非法的http url:" << strUrl << endl; throw std::invalid_argument(strErr); } auto host = FindField(strUrl.data(), "://", "/"); if (host.empty()) { host = FindField(strUrl.data(), "://", NULL); } _path = FindField(strUrl.data(), host.data(), NULL); if (_path.empty()) { _path = "/"; } auto pos = host.find('@'); if (pos != string::npos) { //去除?后面的字符串 auto authStr = host.substr(0, pos); host = host.substr(pos + 1, host.size()); _header.emplace("Authorization", "Basic " + encodeBase64(authStr)); } auto host_header = host; uint16_t port = atoi(FindField(host.data(), ":", NULL).data()); if (port <= 0) { //默认端口 port = defaultPort; } else { //服务器域名 host = FindField(host.data(), NULL, ":"); } _header.emplace("Host", host_header); _header.emplace("Tools", kServerName); _header.emplace("Connection", "keep-alive"); _header.emplace("Accept", "*/*"); _header.emplace("Accept-Language", "zh-CN,zh;q=0.8"); _header.emplace("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"); if (_body && _body->remainSize()) { _header.emplace("Content-Length", to_string(_body->remainSize())); _header.emplace("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } bool bChanged = (_lastHost != host + ":" + to_string(port)) || (_isHttps != isHttps); _lastHost = host + ":" + to_string(port); _isHttps = isHttps; _fTimeOutSec = fTimeOutSec; auto cookies = HttpCookieStorage::Instance().get(_lastHost, _path); _StrPrinter printer; for (auto &cookie : cookies) { printer << cookie->getKey() << "=" << cookie->getVal() << ";"; } if (!printer.empty()) { printer.pop_back(); _header.emplace("Cookie", printer); } if (!alive() || bChanged) { startConnect(host, port, fTimeOutSec); } else { SockException ex; onConnect_l(ex); } } void HttpClient::clear() { _url.clear(); _header.clear(); _body.reset(); _method.clear(); _path.clear(); _aliveTicker.resetTime(); _chunkedSplitter.reset(); _fTimeOutSec = 0; clearResponse(); } void HttpClient::clearResponse() { _recvedBodySize = 0; _totalBodySize = 0; _parser.Clear(); _chunkedSplitter = nullptr; HttpRequestSplitter::reset(); } void HttpClient::setMethod(string method) { _method = std::move(method); } void HttpClient::setHeader(HttpHeader header) { _header = std::move(header); } HttpClient &HttpClient::addHeader(string key, string val, bool force) { if (!force) { _header.emplace(std::move(key), std::move(val)); } else { _header[std::move(key)] = std::move(val); } return *this; } void HttpClient::setBody(string body) { _body.reset(new HttpStringBody(std::move(body))); } void HttpClient::setBody(HttpBody::Ptr body) { _body = std::move(body); } const Parser &HttpClient::response() const { return _parser; } const string &HttpClient::getUrl() const { return _url; } void HttpClient::onConnect(const SockException &ex) { onConnect_l(ex); } void HttpClient::onConnect_l(const SockException &ex) { _aliveTicker.resetTime(); if (ex) { onDisconnect(ex); return; } clearResponse(); _StrPrinter printer; printer << _method + " " << _path + " HTTP/1.1\r\n"; for (auto &pr : _header) { printer << pr.first + ": "; printer << pr.second + "\r\n"; } SockSender::send(printer << "\r\n"); onFlush(); } void HttpClient::onRecv(const Buffer::Ptr &pBuf) { _aliveTicker.resetTime(); HttpRequestSplitter::input(pBuf->data(), pBuf->size()); } void HttpClient::onErr(const SockException &ex) { if (ex.getErrCode() == Err_eof && _totalBodySize < 0) { //如果Content-Length未指定 但服务器断开链接 //则认为本次http请求完成 onResponseCompleted_l(); } onDisconnect(ex); } ssize_t HttpClient::onRecvHeader(const char *data, size_t len) { _parser.Parse(data); if (_parser.Url() == "302" || _parser.Url() == "301") { auto newUrl = _parser["Location"]; if (newUrl.empty()) { shutdown(SockException(Err_shutdown, "未找到Location字段(跳转url)")); return 0; } if (onRedirectUrl(newUrl, _parser.Url() == "302")) { HttpClient::clear(); setMethod("GET"); HttpClient::sendRequest(newUrl, _fTimeOutSec); return 0; } } checkCookie(_parser.getHeader()); _totalBodySize = onResponseHeader(_parser.Url(), _parser.getHeader()); if (!_parser["Content-Length"].empty()) { //有Content-Length字段时忽略onResponseHeader的返回值 _totalBodySize = atoll(_parser["Content-Length"].data()); } if (_parser["Transfer-Encoding"] == "chunked") { //如果Transfer-Encoding字段等于chunked,则认为后续的content是不限制长度的 _totalBodySize = -1; _chunkedSplitter = std::make_shared([this](const char *data, size_t len) { if (len > 0) { auto recvedBodySize = _recvedBodySize + len; onResponseBody(data, len, recvedBodySize, SIZE_MAX); _recvedBodySize = recvedBodySize; } else { onResponseCompleted_l(); } }); } if (_totalBodySize == 0) { //后续没content,本次http请求结束 onResponseCompleted_l(); return 0; } //当_totalBodySize != 0时到达这里,代表后续有content //虽然我们在_totalBodySize >0 时知道content的确切大小, //但是由于我们没必要等content接收完毕才回调onRecvContent(因为这样浪费内存并且要多次拷贝数据) //所以返回-1代表我们接下来分段接收content _recvedBodySize = 0; return -1; } void HttpClient::onRecvContent(const char *data, size_t len) { if (_chunkedSplitter) { _chunkedSplitter->input(data, len); return; } auto recvedBodySize = _recvedBodySize + len; if (_totalBodySize < 0) { //不限长度的content,最大支持SIZE_MAX个字节 onResponseBody(data, len, recvedBodySize, SIZE_MAX); _recvedBodySize = recvedBodySize; return; } //固定长度的content if (recvedBodySize < (size_t) _totalBodySize) { //content还未接收完毕 onResponseBody(data, len, recvedBodySize, _totalBodySize); _recvedBodySize = recvedBodySize; return; } //content接收完毕 onResponseBody(data, _totalBodySize - _recvedBodySize, _totalBodySize, _totalBodySize); bool biggerThanExpected = recvedBodySize > (size_t) _totalBodySize; onResponseCompleted_l(); if (biggerThanExpected) { //声明的content数据比真实的小,那么我们只截取前面部分的并断开链接 shutdown(SockException(Err_shutdown, "http response content size bigger than expected")); } } void HttpClient::onFlush() { _aliveTicker.resetTime(); GET_CONFIG(uint32_t, sendBufSize, Http::kSendBufSize); while (_body && _body->remainSize() && !isSocketBusy()) { auto buffer = _body->readData(sendBufSize); if (!buffer) { //数据发送结束或读取数据异常 break; } if (send(buffer) <= 0) { //发送数据失败,不需要回滚数据,因为发送前已经通过isSocketBusy()判断socket可写 //所以发送缓存区肯定未满,该buffer肯定已经写入socket break; } } } void HttpClient::onManager() { if (_aliveTicker.elapsedTime() > 3 * 1000 && _totalBodySize < 0 && !_chunkedSplitter) { //如果Content-Length未指定 但接收数据超时 //则认为本次http请求完成 onResponseCompleted_l(); } if (_fTimeOutSec > 0 && _aliveTicker.elapsedTime() > _fTimeOutSec * 1000) { //超时 shutdown(SockException(Err_timeout, "http request timeout")); } } void HttpClient::onResponseCompleted_l() { onResponseCompleted(); } void HttpClient::checkCookie(HttpClient::HttpHeader &headers) { //Set-Cookie: IPTV_SERVER=8E03927B-CC8C-4389-BC00-31DBA7EC7B49;expires=Sun, Sep 23 2018 15:07:31 GMT;path=/index/api/ for (auto it_set_cookie = headers.find("Set-Cookie"); it_set_cookie != headers.end(); ++it_set_cookie) { auto key_val = Parser::parseArgs(it_set_cookie->second, ";", "="); HttpCookie::Ptr cookie = std::make_shared(); cookie->setHost(_lastHost); int index = 0; auto arg_vec = split(it_set_cookie->second, ";"); for (string &key_val : arg_vec) { auto key = FindField(key_val.data(), NULL, "="); auto val = FindField(key_val.data(), "=", NULL); if (index++ == 0) { cookie->setKeyVal(key, val); continue; } if (key == "path") { cookie->setPath(val); continue; } if (key == "expires") { cookie->setExpires(val, headers["Date"]); continue; } } if (!(*cookie)) { //无效的cookie continue; } HttpCookieStorage::Instance().set(cookie); } } } /* namespace mediakit */