Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Support http 1.1 deflate compression

Browse Source
This commit is contained in:
Dennis Luxen 2011-02-11 16:12:37 +00:00
parent e48b47f1ec
commit d0547f3d69
4 changed files with 355 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Server/BasicDatastructures.h
View File

@ -20,7 +20,6 @@ or see http://www.gnu.org/licenses/agpl.txt.
#ifndef BASIC_DATASTRUCTURES_H #ifndef BASIC_DATASTRUCTURES_H
#define BASIC_DATASTRUCTURES_H #define BASIC_DATASTRUCTURES_H
#include <string> #include <string>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
@ -39,8 +38,18 @@ const char crlf[] = { '\r', '\n' };
struct Header { struct Header {
std::string name; std::string name;
std::string value; std::string value;
void Clear() {
name.clear();
value.clear();
}
}; };
enum CompressionType {
none,
gzipRFC1952,
deflateRFC1951
} Compression;
struct Request { struct Request {
std::string uri; std::string uri;
}; };
@ -55,8 +64,19 @@ struct Reply {
std::vector<Header> headers; std::vector<Header> headers;
std::string content; std::string content;
std::vector<boost::asio::const_buffer> toBuffers(); std::vector<boost::asio::const_buffer> toBuffers();
std::vector<boost::asio::const_buffer> HeaderstoBuffers();
static Reply stockReply(status_type status); static Reply stockReply(status_type status);
void setSize(unsigned size) {
for (std::size_t i = 0; i < headers.size(); ++i) {
Header& h = headers[i];
if("Content-Length" == h.name) {
std::stringstream sizeString;
sizeString << size;
h.value = sizeString.str();
}
}
}
}; };
boost::asio::const_buffer ToBuffer(Reply::status_type status) { boost::asio::const_buffer ToBuffer(Reply::status_type status) {
@ -96,6 +116,21 @@ std::vector<boost::asio::const_buffer> Reply::toBuffers(){
return buffers; return buffers;
} }
std::vector<boost::asio::const_buffer> Reply::HeaderstoBuffers(){
std::vector<boost::asio::const_buffer> buffers;
buffers.push_back(ToBuffer(status));
for (std::size_t i = 0; i < headers.size(); ++i) {
Header& h = headers[i];
// std::cout << h.name << ": " << h.value << std::endl;
buffers.push_back(boost::asio::buffer(h.name));
buffers.push_back(boost::asio::buffer(seperators));
buffers.push_back(boost::asio::buffer(h.value));
buffers.push_back(boost::asio::buffer(crlf));
}
buffers.push_back(boost::asio::buffer(crlf));
return buffers;
}
Reply Reply::stockReply(Reply::status_type status) { Reply Reply::stockReply(Reply::status_type status) {
Reply rep; Reply rep;
rep.status = status; rep.status = status;

81
Server/Connection.h
View File

@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/licenses/agpl.txt. or see http://www.gnu.org/licenses/agpl.txt.
*/ */
#ifndef CONNECTION_H #ifndef CONNECTION_H
#define CONNECTION_H #define CONNECTION_H
@ -29,10 +29,14 @@ or see http://www.gnu.org/licenses/agpl.txt.
#include <boost/noncopyable.hpp> #include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp> #include <boost/enable_shared_from_this.hpp>
#include "../DataStructures/Util.h"
#include "BasicDatastructures.h" #include "BasicDatastructures.h"
#include "RequestHandler.h" #include "RequestHandler.h"
#include "RequestParser.h" #include "RequestParser.h"
#include "zlib.h"
namespace http { namespace http {
/// Represents a single connection from a client. /// Represents a single connection from a client.
@ -52,12 +56,40 @@ public:
private: private:
void handleRead(const boost::system::error_code& e, std::size_t bytes_transferred) { void handleRead(const boost::system::error_code& e, std::size_t bytes_transferred) {
if (!e) { if (!e) {
CompressionType compressionType(none);
boost::tribool result; boost::tribool result;
boost::tie(result, boost::tuples::ignore) = requestParser.Parse( request, incomingDataBuffer.data(), incomingDataBuffer.data() + bytes_transferred); boost::tie(result, boost::tuples::ignore) = requestParser.Parse( request, incomingDataBuffer.data(), incomingDataBuffer.data() + bytes_transferred, &compressionType);
if (result) { if (result) {
// std::cout << "----" << std::endl;
// if(compressionType == gzipRFC1952)
// std::cout << "[debug] supports gzip" << std::endl;
// if(compressionType == deflateRFC1951)
// std::cout << "[debug] Supports deflate" << std::endl;
requestHandler.handle_request(request, reply); requestHandler.handle_request(request, reply);
boost::asio::async_write(TCPsocket, reply.toBuffers(), strand.wrap( boost::bind(&Connection::handleWrite, this->shared_from_this(), boost::asio::placeholders::error)));
if(compressionType == deflateRFC1951) {
Header compressionHeader;
compressionHeader.name = "Content-Encoding";
compressionHeader.value = "deflate";
reply.headers.insert(reply.headers.begin(), compressionHeader); //push_back(compressionHeader);
std::vector<unsigned char> compressed;
// double time = get_timestamp();
compressCharArray(reply.content.c_str(), strlen(reply.content.c_str()), compressed);
// std::cout << "[info] compression took " << get_timestamp() - time << " seconds." << std::endl;
reply.setSize(compressed.size());
std::vector<boost::asio::const_buffer> outputBuffer = reply.HeaderstoBuffers();
outputBuffer.push_back(boost::asio::buffer(compressed));
// std::cout << "[debug] outbuffer.size(): " << outputBuffer.size() << std::endl;
boost::asio::async_write(TCPsocket, outputBuffer, strand.wrap( boost::bind(&Connection::handleWrite, this->shared_from_this(), boost::asio::placeholders::error)));
} else {
boost::asio::async_write(TCPsocket, reply.toBuffers(), strand.wrap( boost::bind(&Connection::handleWrite, this->shared_from_this(), boost::asio::placeholders::error)));
}
} else if (!result) { } else if (!result) {
reply = Reply::stockReply(Reply::badRequest); reply = Reply::stockReply(Reply::badRequest);
boost::asio::async_write(TCPsocket, reply.toBuffers(), strand.wrap( boost::bind(&Connection::handleWrite, this->shared_from_this(), boost::asio::placeholders::error))); boost::asio::async_write(TCPsocket, reply.toBuffers(), strand.wrap( boost::bind(&Connection::handleWrite, this->shared_from_this(), boost::asio::placeholders::error)));
@ -80,6 +112,49 @@ private:
// destructor closes the socket. // destructor closes the socket.
} }
void compressCharArray(const void *in_data, size_t in_data_size, std::vector<unsigned char> &buffer) {
// std::vector<unsigned char> buffer;
const size_t BUFSIZE = 128 * 1024;
unsigned char temp_buffer[BUFSIZE];
z_stream strm;
strm.zalloc = 0;
strm.zfree = 0;
strm.next_in = (unsigned char *)(in_data);
strm.avail_in = in_data_size;
strm.next_out = temp_buffer;
strm.avail_out = BUFSIZE;
deflateInit(&strm, Z_BEST_SPEED);
while (strm.avail_in != 0) {
int res = deflate(&strm, Z_NO_FLUSH);
assert(res == Z_OK);
if (strm.avail_out == 0) {
buffer.insert(buffer.end(), temp_buffer, temp_buffer + BUFSIZE);
strm.next_out = temp_buffer;
strm.avail_out = BUFSIZE;
}
}
int deflate_res = Z_OK;
while (deflate_res == Z_OK) {
if (strm.avail_out == 0) {
buffer.insert(buffer.end(), temp_buffer, temp_buffer + BUFSIZE);
strm.next_out = temp_buffer;
strm.avail_out = BUFSIZE;
}
deflate_res = deflate(&strm, Z_FINISH);
}
assert(deflate_res == Z_STREAM_END);
buffer.insert(buffer.end(), temp_buffer, temp_buffer + BUFSIZE - strm.avail_out);
deflateEnd(&strm);
// out_data.swap(buffer);
}
boost::asio::io_service::strand strand; boost::asio::io_service::strand strand;
boost::asio::ip::tcp::socket TCPsocket; boost::asio::ip::tcp::socket TCPsocket;
RequestHandler& requestHandler; RequestHandler& requestHandler;

2
Server/RequestHandler.h
View File

@ -42,7 +42,7 @@ public:
void handle_request(const Request& req, Reply& rep){ void handle_request(const Request& req, Reply& rep){
//parse command //parse command
std::string request(req.uri); std::string request(req.uri);
std::string command; std::string command;
std::size_t firstAmpPosition = request.find_first_of("&"); std::size_t firstAmpPosition = request.find_first_of("&");
command = request.substr(1,firstAmpPosition-1); command = request.substr(1,firstAmpPosition-1);

466
Server/RequestParser.h
View File

@ -29,241 +29,253 @@ namespace http {
class RequestParser { class RequestParser {
public: public:
RequestParser() : state_(method_start) { } RequestParser() : state_(method_start) { }
void Reset() { state_ = method_start; } void Reset() { state_ = method_start; }
boost::tuple<boost::tribool, char*> Parse(Request& req, char* begin, char* end) { boost::tuple<boost::tribool, char*> Parse(Request& req, char* begin, char* end, CompressionType * compressionType) {
while (begin != end) { while (begin != end) {
boost::tribool result = consume(req, *begin++); boost::tribool result = consume(req, *begin++, compressionType);
if (result || !result){ if (result || !result){
return boost::make_tuple(result, begin); return boost::make_tuple(result, begin);
} }
} }
boost::tribool result = boost::indeterminate; boost::tribool result = boost::indeterminate;
return boost::make_tuple(result, begin); return boost::make_tuple(result, begin);
} }
private: private:
boost::tribool consume(Request& req, char input) { boost::tribool consume(Request& req, char input, CompressionType * compressionType) {
switch (state_) { switch (state_) {
case method_start: case method_start:
if (!isChar(input) || isCTL(input) || isTSpecial(input)) { if (!isChar(input) || isCTL(input) || isTSpecial(input)) {
return false; return false;
} else { } else {
state_ = method; state_ = method;
return boost::indeterminate; return boost::indeterminate;
} }
case method: case method:
if (input == ' ') { if (input == ' ') {
state_ = uri; state_ = uri;
return boost::indeterminate; return boost::indeterminate;
} else if (!isChar(input) || isCTL(input) || isTSpecial(input)) { } else if (!isChar(input) || isCTL(input) || isTSpecial(input)) {
return false; return false;
} else { } else {
return boost::indeterminate; return boost::indeterminate;
} }
case uri_start: case uri_start:
if (isCTL(input)) { if (isCTL(input)) {
return false; return false;
} else { } else {
state_ = uri; state_ = uri;
req.uri.push_back(input); req.uri.push_back(input);
return boost::indeterminate; return boost::indeterminate;
} }
case uri: case uri:
if (input == ' ') { if (input == ' ') {
state_ = http_version_h; state_ = http_version_h;
return boost::indeterminate; return boost::indeterminate;
} else if (isCTL(input)) { } else if (isCTL(input)) {
return false; return false;
} else { } else {
req.uri.push_back(input); req.uri.push_back(input);
return boost::indeterminate; return boost::indeterminate;
} }
case http_version_h: case http_version_h:
if (input == 'H') { if (input == 'H') {
state_ = http_version_t_1; state_ = http_version_t_1;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_t_1: case http_version_t_1:
if (input == 'T') { if (input == 'T') {
state_ = http_version_t_2; state_ = http_version_t_2;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_t_2: case http_version_t_2:
if (input == 'T') { if (input == 'T') {
state_ = http_version_p; state_ = http_version_p;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_p: case http_version_p:
if (input == 'P') { if (input == 'P') {
state_ = http_version_slash; state_ = http_version_slash;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_slash: case http_version_slash:
if (input == '/') { if (input == '/') {
state_ = http_version_major_start; state_ = http_version_major_start;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_major_start: case http_version_major_start:
if (isDigit(input)) { if (isDigit(input)) {
state_ = http_version_major; state_ = http_version_major;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_major: case http_version_major:
if (input == '.') { if (input == '.') {
state_ = http_version_minor_start; state_ = http_version_minor_start;
return boost::indeterminate; return boost::indeterminate;
} else if (isDigit(input)) { } else if (isDigit(input)) {
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_minor_start: case http_version_minor_start:
if (isDigit(input)) { if (isDigit(input)) {
state_ = http_version_minor; state_ = http_version_minor;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case http_version_minor: case http_version_minor:
if (input == '\r') { if (input == '\r') {
state_ = expecting_newline_1; state_ = expecting_newline_1;
return boost::indeterminate; return boost::indeterminate;
} else if (isDigit(input)) { } else if (isDigit(input)) {
return boost::indeterminate; return boost::indeterminate;
} }
else { else {
return false; return false;
} }
case expecting_newline_1: case expecting_newline_1:
if (input == '\n') { if (input == '\n') {
state_ = header_line_start; state_ = header_line_start;
return boost::indeterminate; return boost::indeterminate;
} else { } else {
return false; return false;
} }
case header_line_start: case header_line_start:
if (input == '\r') { if (input == '\r') {
state_ = expecting_newline_3; state_ = expecting_newline_3;
return boost::indeterminate; return boost::indeterminate;
} else if (!isChar(input) || isCTL(input) || isTSpecial(input)) { } else if (!isChar(input) || isCTL(input) || isTSpecial(input)) {
return false; return false;
} else { } else {
state_ = header_name; state_ = header_name;
return boost::indeterminate; if(header.name == "Accept-Encoding") {
} if(header.value.find("deflate") != std::string::npos)
case header_lws: *compressionType = deflateRFC1951;
if (input == '\r') { // if(header.value.find("gzip") != std::string::npos)
state_ = expecting_newline_2; // *compressionType = gzipRFC1952;
return boost::indeterminate; }
} else if (input == ' ' || input == '\t') { header.Clear();
return boost::indeterminate; header.name.push_back(input);
} return boost::indeterminate;
else if (isCTL(input)) { }
return false; case header_lws:
} else { if (input == '\r') {
state_ = header_value; state_ = expecting_newline_2;
return boost::indeterminate; return boost::indeterminate;
} } else if (input == ' ' || input == '\t') {
case header_name: return boost::indeterminate;
if (input == ':') { }
state_ = space_before_header_value; else if (isCTL(input)) {
return boost::indeterminate; return false;
} else if (!isChar(input) || isCTL(input) || isTSpecial(input)) { } else {
return false; state_ = header_value;
} else { return boost::indeterminate;
return boost::indeterminate; }
} case header_name:
case space_before_header_value: if (input == ':') {
if (input == ' ') { state_ = space_before_header_value;
state_ = header_value; return boost::indeterminate;
return boost::indeterminate; } else if (!isChar(input) || isCTL(input) || isTSpecial(input)) {
} else { return false;
return false; } else {
} header.name.push_back(input);
case header_value: return boost::indeterminate;
if (input == '\r') { }
state_ = expecting_newline_2; case space_before_header_value:
return boost::indeterminate; if (input == ' ') {
} else if (isCTL(input)) { state_ = header_value;
return false; return boost::indeterminate;
} else { } else {
return boost::indeterminate; return false;
} }
case expecting_newline_2: case header_value:
if (input == '\n') { if (input == '\r') {
state_ = header_line_start; state_ = expecting_newline_2;
return boost::indeterminate; return boost::indeterminate;
} else { } else if (isCTL(input)) {
return false; return false;
} } else {
case expecting_newline_3: header.value.push_back(input);
return (input == '\n'); return boost::indeterminate;
default: }
return false; case expecting_newline_2:
} if (input == '\n') {
} state_ = header_line_start;
return boost::indeterminate;
} else {
return false;
}
case expecting_newline_3:
return (input == '\n');
default:
return false;
}
}
inline bool isChar(int c) { inline bool isChar(int c) {
return c >= 0 && c <= 127; return c >= 0 && c <= 127;
} }
inline bool isCTL(int c) { inline bool isCTL(int c) {
return (c >= 0 && c <= 31) || (c == 127); return (c >= 0 && c <= 31) || (c == 127);
} }
inline bool isTSpecial(int c) { inline bool isTSpecial(int c) {
switch (c) { switch (c) {
case '(': case ')': case '<': case '>': case '@': case '(': case ')': case '<': case '>': case '@':
case ',': case ';': case ':': case '\\': case '"': case ',': case ';': case ':': case '\\': case '"':
case '/': case '[': case ']': case '?': case '=': case '/': case '[': case ']': case '?': case '=':
case '{': case '}': case ' ': case '\t': case '{': case '}': case ' ': case '\t':
return true; return true;
default: default:
return false; return false;
} }
} }
inline bool isDigit(int c) { inline bool isDigit(int c) {
return c >= '0' && c <= '9'; return c >= '0' && c <= '9';
} }
enum state { enum state {
method_start, method_start,
method, method,
uri_start, uri_start,
uri, uri,
http_version_h, http_version_h,
http_version_t_1, http_version_t_1,
http_version_t_2, http_version_t_2,
http_version_p, http_version_p,
http_version_slash, http_version_slash,
http_version_major_start, http_version_major_start,
http_version_major, http_version_major,
http_version_minor_start, http_version_minor_start,
http_version_minor, http_version_minor,
expecting_newline_1, expecting_newline_1,
header_line_start, header_line_start,
header_lws, header_lws,
header_name, header_name,
space_before_header_value, space_before_header_value,
header_value, header_value,
expecting_newline_2, expecting_newline_2,
expecting_newline_3 expecting_newline_3
} state_; } state_;
Header header;
}; };
} // namespace http } // namespace http