Open-Harbor/osrm-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

replaced contigouos output data with collection of small'ish sub blocks. saves (re-)allocations. also removed a remaining stringstream

Browse Source
This commit is contained in:
Dennis Luxen
2013-11-14 17:16:26 -05:00
parent 8b6fe691ed
commit cabaad4b17
17 changed files with 391 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Server/Connection.h
+99 -68
View File
@@ -37,11 +37,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <boost/assert.hpp>
#include <boost/bind.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <zlib.h>
// #include <zlib.h>
#include <string>
#include <vector>
namespace http {
@@ -92,7 +96,7 @@ private:
request_handler.handle_request(request, reply);
Header compression_header;
std::vector<unsigned char> compressed_output;
std::vector<char> compressed_output;
std::vector<boost::asio::const_buffer> output_buffer;
switch(compression_type) {
case deflateRFC1951:
@@ -102,11 +106,10 @@ private:
reply.headers.begin(),
compression_header
);
compressCharArray(
reply.content.c_str(),
reply.content.length(),
compressed_output,
compression_type
compressBufferCollection(
reply.content,
compression_type,
compressed_output
);
reply.setSize(compressed_output.size());
output_buffer = reply.HeaderstoBuffers();
@@ -132,11 +135,10 @@ private:
reply.headers.begin(),
compression_header
);
compressCharArray(
reply.content.c_str(),
reply.content.length(),
compressed_output,
compression_type
compressBufferCollection(
reply.content,
compression_type,
compressed_output
);
reply.setSize(compressed_output.size());
output_buffer = reply.HeaderstoBuffers();
@@ -210,71 +212,100 @@ private:
}
}
void compressBufferCollection(
std::vector<std::string> uncompressed_data,
CompressionType compression_type,
std::vector<char> & compressed_data
) {
boost::iostreams::gzip_params compression_parameters;
compression_parameters.level = boost::iostreams::zlib::best_speed;
if ( deflateRFC1951 == compression_type ) {
compression_parameters.noheader = true;
}
BOOST_ASSERT( compressed_data.empty() );
boost::iostreams::filtering_ostream compressing_stream;
compressing_stream.push(
boost::iostreams::gzip_compressor(compression_parameters)
);
compressing_stream.push(
boost::iostreams::back_inserter(compressed_data)
);
BOOST_FOREACH( const std::string & line, uncompressed_data) {
compressing_stream << line;
}
compressing_stream.reset();
}
// Big thanks to deusty who explains how to use gzip compression by
// the right call to deflateInit2():
// http://deusty.blogspot.com/2007/07/gzip-compressiondecompression.html
void compressCharArray(
const char * in_data,
size_t in_data_size,
std::vector<unsigned char> & buffer,
CompressionType type
) {
const size_t BUFSIZE = 128 * 1024;
unsigned char temp_buffer[BUFSIZE];
// void compressCharArray(
// const char * in_data,
// size_t in_data_size,
// std::vector<unsigned char> & buffer,
// CompressionType type
// ) {
// const size_t BUFSIZE = 128 * 1024;
// unsigned char temp_buffer[BUFSIZE];
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.total_out = 0;
strm.next_in = (unsigned char *)(in_data);
strm.avail_in = in_data_size;
strm.next_out = temp_buffer;
strm.avail_out = BUFSIZE;
strm.data_type = Z_ASCII;
// z_stream strm;
// strm.zalloc = Z_NULL;
// strm.zfree = Z_NULL;
// strm.opaque = Z_NULL;
// strm.total_out = 0;
// strm.next_in = (unsigned char *)(in_data);
// strm.avail_in = in_data_size;
// strm.next_out = temp_buffer;
// strm.avail_out = BUFSIZE;
// strm.data_type = Z_ASCII;
switch(type){
case deflateRFC1951:
deflateInit(&strm, Z_BEST_SPEED);
break;
case gzipRFC1952:
deflateInit2(
&strm,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
(15+16),
9,
Z_DEFAULT_STRATEGY
);
break;
default:
BOOST_ASSERT_MSG(false, "should not happen");
break;
}
// switch(type){
// case deflateRFC1951:
// deflateInit(&strm, Z_BEST_SPEED);
// break;
// case gzipRFC1952:
// deflateInit2(
// &strm,
// Z_DEFAULT_COMPRESSION,
// Z_DEFLATED,
// (15+16),
// 9,
// Z_DEFAULT_STRATEGY
// );
// break;
// default:
// BOOST_ASSERT_MSG(false, "should not happen");
// break;
// }
int deflate_res = Z_OK;
do {
if ( 0 == strm.avail_out ) {
buffer.insert(buffer.end(), temp_buffer, temp_buffer + BUFSIZE);
strm.next_out = temp_buffer;
strm.avail_out = BUFSIZE;
}
deflate_res = deflate(&strm, Z_FINISH);
// int deflate_res = Z_OK;
// do {
// if ( 0 == strm.avail_out ) {
// buffer.insert(buffer.end(), temp_buffer, temp_buffer + BUFSIZE);
// strm.next_out = temp_buffer;
// strm.avail_out = BUFSIZE;
// }
// deflate_res = deflate(&strm, Z_FINISH);
} while (deflate_res == Z_OK);
// } while (deflate_res == Z_OK);
BOOST_ASSERT_MSG(
deflate_res == Z_STREAM_END,
"compression not properly finished"
);
// BOOST_ASSERT_MSG(
// deflate_res == Z_STREAM_END,
// "compression not properly finished"
// );
buffer.insert(
buffer.end(),
temp_buffer,
temp_buffer + BUFSIZE - strm.avail_out
);
deflateEnd(&strm);
}
// buffer.insert(
// buffer.end(),
// temp_buffer,
// temp_buffer + BUFSIZE - strm.avail_out
// );
// deflateEnd(&strm);
// }
boost::asio::io_service::strand strand;
boost::asio::ip::tcp::socket TCP_socket;
Server/Http/Reply.cpp
+7 -6
View File
@@ -41,15 +41,16 @@ void Reply::setSize(const unsigned size) {
std::vector<boost::asio::const_buffer> Reply::toBuffers(){
std::vector<boost::asio::const_buffer> buffers;
buffers.push_back(ToBuffer(status));
for (std::size_t i = 0; i < headers.size(); ++i) {
Header& h = headers[i];
BOOST_FOREACH(const Header & h, headers) {
buffers.push_back(boost::asio::buffer(h.name));
buffers.push_back(boost::asio::buffer(seperators));
buffers.push_back(boost::asio::buffer(h.value));
buffers.push_back(boost::asio::buffer(crlf));
}
buffers.push_back(boost::asio::buffer(crlf));
buffers.push_back(boost::asio::buffer(content));
BOOST_FOREACH(const std::string & line, content) {
buffers.push_back(boost::asio::buffer(line));
}
return buffers;
}
@@ -70,7 +71,8 @@ std::vector<boost::asio::const_buffer> Reply::HeaderstoBuffers(){
Reply Reply::StockReply(Reply::status_type status) {
Reply rep;
rep.status = status;
rep.content = ToString(status);
rep.content.clear();
rep.content.push_back( ToString(status) );
rep.headers.resize(3);
rep.headers[0].name = "Access-Control-Allow-Origin";
rep.headers[0].value = "*";
@@ -85,7 +87,6 @@ Reply Reply::StockReply(Reply::status_type status) {
return rep;
}
std::string Reply::ToString(Reply::status_type status) {
switch (status) {
case Reply::ok:
@@ -110,7 +111,7 @@ boost::asio::const_buffer Reply::ToBuffer(Reply::status_type status) {
Reply::Reply() : status(ok) {
content.reserve(2 << 20);
}
}
Server/Http/Reply.h
+1 -1
View File
@@ -59,7 +59,7 @@ const std::string internalServerErrorString = "HTTP/1.0 500 Internal Server Erro
std::vector<Header> headers;
std::vector<boost::asio::const_buffer> toBuffers();
std::vector<boost::asio::const_buffer> HeaderstoBuffers();
std::string content;
std::vector<std::string> content;
static Reply StockReply(status_type status);
void setSize(const unsigned size);
Reply();
Server/RequestHandler.h
+9 -7
View File
@@ -85,16 +85,18 @@ public:
const int position = std::distance(request.begin(), it);
std::string tmp_position_string;
intToString(position, tmp_position_string);
rep.content += "Input seems to be malformed close to position ";
rep.content += "<br><pre>";
rep.content += request;
rep.content += tmp_position_string;
rep.content += "<br>";
rep.content.push_back(
"Input seems to be malformed close to position "
"<br><pre>"
);
rep.content.push_back( request );
rep.content.push_back(tmp_position_string);
rep.content.push_back("<br>");
const unsigned end = std::distance(request.begin(), it);
for(unsigned i = 0; i < end; ++i) {
rep.content += "&nbsp;";
rep.content.push_back("&nbsp;");
}
rep.content += "^<br></pre>";
rep.content.push_back("^<br></pre>");
} else {
//parsing done, lets call the right plugin to handle the request
BOOST_ASSERT_MSG(
Server/RequestParser.h
+11 -2
View File
@@ -42,10 +42,19 @@ public:
RequestParser();
void Reset();
boost::tuple<boost::tribool, char*> Parse(Request& req, char* begin, char* end, CompressionType * compressionType);
boost::tuple<boost::tribool, char*> Parse(
Request& req,
char* begin,
char* end,
CompressionType * compressionType
);
private:
boost::tribool consume(Request& req, char input, CompressionType * compressionType);
boost::tribool consume(
Request& req,
char input,
CompressionType * compressionType
);
inline bool isChar(int c);