diff --git a/Makefile b/Makefile index 5652d59be..02c929105 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ # USE_FUTEX : enable use of futex on kernel 2.6. Automatic. # USE_ACCEPT4 : enable use of accept4() on linux. Automatic. # USE_MY_ACCEPT4 : use own implemention of accept4() if glibc < 2.10. +# USE_ZLIB : enable zlib library support. # # Options can be forced by specifying "USE_xxx=1" or can be disabled by using # "USE_xxx=" (empty string). @@ -402,6 +403,12 @@ OPTIONS_CFLAGS += -DUSE_GETADDRINFO BUILD_OPTIONS += $(call ignore_implicit,USE_GETADDRINFO) endif +ifneq ($(USE_ZLIB),) +OPTIONS_CFLAGS += -DUSE_ZLIB +BUILD_OPTIONS += $(call ignore_implicit,USE_ZLIB) +OPTIONS_LDFLAGS += -lz +endif + ifneq ($(USE_POLL),) OPTIONS_CFLAGS += -DENABLE_POLL OPTIONS_OBJS += src/ev_poll.o @@ -594,6 +601,7 @@ OBJS = src/haproxy.o src/sessionhash.o src/base64.o src/protocol.o \ src/stream_interface.o src/dumpstats.o src/proto_tcp.o \ src/session.o src/hdr_idx.o src/ev_select.o src/signal.o \ src/acl.o src/sample.o src/memory.o src/freq_ctr.o src/auth.o \ + src/compression.o EBTREE_OBJS = $(EBTREE_DIR)/ebtree.o \ $(EBTREE_DIR)/eb32tree.o $(EBTREE_DIR)/eb64tree.o \ diff --git a/README b/README index b886eee1f..d40371105 100644 --- a/README +++ b/README @@ -89,6 +89,10 @@ will automatically be linked with haproxy. Some systems also require libz, so if the build fails due to missing symbols such as deflateInit(), then try again with "ADDLIB=-lz". +It is also possible to include native support for ZLIB to benefit from HTTP +compression. For this, pass "USE_ZLIB=1" on the "make" command line and ensure +that zlib is present on the system. + By default, the DEBUG variable is set to '-g' to enable debug symbols. It is not wise to disable it on uncommon systems, because it's often the only way to get a complete core when you need one. Otherwise, you can set DEBUG to '-s' to @@ -102,11 +106,12 @@ And I build it this way on OpenBSD or FreeBSD : $ make -f Makefile.bsd REGEX=pcre DEBUG= COPTS.generic="-Os -fomit-frame-pointer -mgnu" -And on a recent Linux with SSL support : +And on a recent Linux with SSL and ZLIB support : - $ make TARGET=linux2628 CPU=native USE_PCRE=1 USE_OPENSSL=1 + $ make TARGET=linux2628 CPU=native USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 -In order to build a 32-bit binary on an x86_64 Linux system with SSL support : +In order to build a 32-bit binary on an x86_64 Linux system with SSL support +without support for compression but when OpenSSL requires ZLIB anyway : $ make TARGET=linux26 ARCH=i386 USE_OPENSSL=1 ADDLIB=-lz diff --git a/doc/configuration.txt b/doc/configuration.txt index 69f459172..9270e169c 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -994,6 +994,7 @@ capture cookie - X X - capture request header - X X - capture response header - X X - clitimeout (deprecated) X X X - +compression X X X X contimeout (deprecated) X - X X cookie X - X X default-server X - X X @@ -1777,6 +1778,18 @@ clitimeout (deprecated) See also : "timeout client", "timeout http-request", "timeout server", and "srvtimeout". +compression algo [ gzip ] ... +compression type ... + Enable HTTP compression. + May be used in sections : defaults | frontend | listen | backend + yes | yes | yes | yes + Arguments : + algo is followed by the list of supported compression algorithms. + type is followed by the list of MIME types that will be compressed. + + Examples : + compression algo gzip + compression type text/html text/plain contimeout (deprecated) Set the maximum time to wait for a connection attempt to a server to succeed. diff --git a/include/proto/compression.h b/include/proto/compression.h new file mode 100644 index 000000000..6f1a26a0b --- /dev/null +++ b/include/proto/compression.h @@ -0,0 +1,66 @@ +/* + * include/proto/compression.h + * This file defines function prototypes for compression. + * + * Copyright 2012 (C) Exceliance, David Du Colombier + * William Lallemand + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, version 2.1 + * exclusively. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _PROTO_COMP_H +#define _PROTO_COMP_H + +#include + +int comp_append_type(struct comp *comp, const char *type); +int comp_append_algo(struct comp *comp, const char *algo); + +int http_emit_chunk_size(char *out, unsigned int chksz, int add_crlf); +int http_compression_buffer_init(struct session *s, struct buffer *in, struct buffer *out); +int http_compression_buffer_add_data(struct session *s, struct buffer *in, struct buffer *out); +int http_compression_buffer_end(struct session *s, struct buffer **in, struct buffer **out, int end); + +int identity_init(void *v, int level); +int identity_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len); +int identity_flush(void *comp_ctx, struct buffer *out, int flag); +int identity_reset(void *comp_ctx); +int identity_end(void *comp_ctx); + + +#ifdef USE_ZLIB + +int deflate_init(void *comp_ctx, int level); +int deflate_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len); +int deflate_flush(void *comp_ctx, struct buffer *out, int flag); +int deflate_reset(void *comp_ctx); +int deflate_end(void *comp_ctx); + +int gzip_init(void *comp_ctx, int level); +int gzip_add_data(void *v, const char *in_data, int in_len, char *out_data, int out_len); +int gzip_flush(void *comp_ctx, struct buffer *out, int flag); +int gzip_reset(void *comp_ctx); +int gzip_end(void *comp_ctx); + +#endif /* USE_ZLIB */ + +#endif /* _PROTO_COMP_H */ + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */ diff --git a/include/types/compression.h b/include/types/compression.h new file mode 100644 index 000000000..96dd10772 --- /dev/null +++ b/include/types/compression.h @@ -0,0 +1,63 @@ +/* + * include/types/compression.h + * This file defines everything related to compression. + * + * Copyright 2012 Exceliance, David Du Colombier + William Lallemand + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, version 2.1 + * exclusively. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _TYPES_COMP_H +#define _TYPES_COMP_H + +#include + +struct comp { + struct comp_algo *algos; + struct comp_type *types; +}; + +struct comp_algo { + char *name; + int name_len; + int (*init)(void *, int); + int (*add_data)(void *v, const char *in_data, int in_len, char *out_data, int out_len); + int (*flush)(void *v, struct buffer *out, int flag); + int (*reset)(void *v); + int (*end)(void *v); + struct comp_algo *next; +}; + +union comp_ctx { + z_stream strm; /* zlib */ +}; + +struct comp_type { + char *name; + int name_len; + struct comp_type *next; +}; + + +#endif /* _TYPES_COMP_H */ + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */ + diff --git a/include/types/proxy.h b/include/types/proxy.h index 9bfa68def..343d4ad04 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -363,6 +363,7 @@ struct proxy { struct list listeners; /* list of listeners belonging to this frontend */ } conf; /* config information */ void *parent; /* parent of the proxy when applicable */ + struct comp *comp; /* http compression */ }; struct switching_rule { diff --git a/include/types/session.h b/include/types/session.h index e3ccde8ed..4726a19a2 100644 --- a/include/types/session.h +++ b/include/types/session.h @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -155,6 +156,8 @@ struct session { void (*srv_error)(struct session *s, /* the function to call upon unrecoverable server errors (or NULL) */ struct stream_interface *si); unsigned int uniq_id; /* unique ID used for the traces */ + struct comp_algo *comp_algo; /* HTTP compression algorithm if not NULL */ + union comp_ctx comp_ctx; /* HTTP compression context */ char *unique_id; /* custom unique ID */ }; diff --git a/src/cfgparse.c b/src/cfgparse.c index 3f43f581d..c6138fd72 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -33,6 +33,7 @@ #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -1671,6 +1673,13 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm) if (defproxy.header_unique_id) curproxy->header_unique_id = strdup(defproxy.header_unique_id); + /* default compression options */ + if (defproxy.comp != NULL) { + curproxy->comp = calloc(1, sizeof(struct comp)); + curproxy->comp->algos = defproxy.comp->algos; + curproxy->comp->types = defproxy.comp->types; + } + curproxy->grace = defproxy.grace; curproxy->conf.used_listener_id = EB_ROOT; curproxy->conf.used_server_id = EB_ROOT; @@ -5236,6 +5245,57 @@ stats_error_parsing: free(err); } } + else if (!strcmp(args[0], "compression")) { + struct comp *comp; + if (curproxy->comp == NULL) { + comp = calloc(1, sizeof(struct comp)); + curproxy->comp = comp; + } else { + comp = curproxy->comp; + } + + if (!strcmp(args[1], "algo")) { + int cur_arg; + cur_arg = 2; + if (!*args[cur_arg]) { + Alert("parsing [%s:%d] : '%s' expects \n", + file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + while (*(args[cur_arg])) { + if (comp_append_algo(comp, args[cur_arg]) < 0) { + Alert("parsing [%s:%d] : '%s' : '%s' is not a supported algorithm.\n", + file, linenum, args[0], args[cur_arg]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + cur_arg ++; + continue; + } + } + else if (!strcmp(args[1], "type")) { + int cur_arg; + cur_arg = 2; + if (!*args[cur_arg]) { + Alert("parsing [%s:%d] : '%s' expects \n", + file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + while (*(args[cur_arg])) { + comp_append_type(comp, args[cur_arg]); + cur_arg ++; + continue; + } + } + else { + Alert("parsing [%s:%d] : '%s' expects algo or type\n", + file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + } else { struct cfg_kw_list *kwl; int index; @@ -5263,7 +5323,7 @@ stats_error_parsing: } } } - + Alert("parsing [%s:%d] : unknown keyword '%s' in '%s' section\n", file, linenum, args[0], cursection); err_code |= ERR_ALERT | ERR_FATAL; goto out; diff --git a/src/compression.c b/src/compression.c new file mode 100644 index 000000000..87449acf1 --- /dev/null +++ b/src/compression.c @@ -0,0 +1,415 @@ +/* + * HTTP compression. + * + * Copyright 2012 Exceliance, David Du Colombier + * William Lallemand + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + */ + +#include +#include + +#include + +#include +#include + +#include +#include + +static const struct comp_algo comp_algos[] = +{ + { "identity", 8, identity_init, identity_add_data, identity_flush, identity_reset, identity_end }, +#ifdef USE_ZLIB + { "deflate", 7, deflate_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end }, + { "gzip", 4, gzip_init, deflate_add_data, deflate_flush, deflate_reset, deflate_end }, +#endif /* USE_ZLIB */ + { NULL, 0, NULL , NULL, NULL, NULL, NULL } +}; + +/* + * Add a content-type in the configuration + */ +int comp_append_type(struct comp *comp, const char *type) +{ + struct comp_type *comp_type; + + comp_type = calloc(1, sizeof(struct comp_type)); + comp_type->name_len = strlen(type); + comp_type->name = strdup(type); + comp_type->next = comp->types; + comp->types = comp_type; + return 0; +} + +/* + * Add an algorithm in the configuration + */ +int comp_append_algo(struct comp *comp, const char *algo) +{ + struct comp_algo *comp_algo; + int i; + + for (i = 0; comp_algos[i].name; i++) { + if (!strcmp(algo, comp_algos[i].name)) { + comp_algo = calloc(1, sizeof(struct comp_algo)); + memmove(comp_algo, &comp_algos[i], sizeof(struct comp_algo)); + comp_algo->next = comp->algos; + comp->algos = comp_algo; + return 0; + } + } + return -1; +} + +/* emit the chunksize followed by a CRLF on the output and return the number of + * bytes written. Appends additional CRLF after the first one. Chunk + * sizes are truncated to 6 hex digits (16 MB) and padded left. The caller is + * responsible for ensuring there is enough room left in the output buffer for + * the string (8 bytes * add_crlf*2). + */ +int http_emit_chunk_size(char *out, unsigned int chksz, int add_crlf) +{ + int shift; + int pos = 0; + + for (shift = 20; shift >= 0; shift -= 4) + out[pos++] = hextab[(chksz >> shift) & 0xF]; + + do { + out[pos++] = '\r'; + out[pos++] = '\n'; + } while (--add_crlf >= 0); + + return pos; +} + +/* + * Init HTTP compression + */ +int http_compression_buffer_init(struct session *s, struct buffer *in, struct buffer *out) +{ + struct http_msg *msg = &s->txn.rsp; + int left; + + /* not enough space */ + if (in->size - buffer_len(in) < 40) + return -1; + + /* + * Skip data, we don't need them in the new buffer. They are results + * of CHUNK_CRLF and CHUNK_SIZE parsing. + */ + b_adv(in, msg->next); + msg->next = 0; + msg->sov = 0; + msg->sol = 0; + + out->size = global.tune.bufsize; + out->i = 0; + out->o = 0; + out->p = out->data; + /* copy output data */ + if (in->o > 0) { + left = in->o - bo_contig_data(in); + memcpy(out->data, bo_ptr(in), bo_contig_data(in)); + out->p += bo_contig_data(in); + if (left > 0) { /* second part of the buffer */ + memcpy(out->p, in->data, left); + out->p += left; + } + out->o = in->o; + } + out->i += http_emit_chunk_size(out->p, 0, 0); + + return 0; +} + +/* + * Add data to compress + */ +int http_compression_buffer_add_data(struct session *s, struct buffer *in, struct buffer *out) +{ + struct http_msg *msg = &s->txn.rsp; + int data_process_len; + int left; + int ret; + + /* + * Skip data, we don't need them in the new buffer. They are results + * of CHUNK_CRLF and CHUNK_SIZE parsing. + */ + b_adv(in, msg->next); + msg->next = 0; + msg->sov = 0; + msg->sol = 0; + + /* + * select the smallest size between the announced chunk size, the input + * data, and the available output buffer size + */ + data_process_len = MIN(in->i, msg->chunk_len); + data_process_len = MIN(out->size - buffer_len(out), data_process_len); + + left = data_process_len - bi_contig_data(in); + if (left <= 0) { + ret = s->comp_algo->add_data(&s->comp_ctx.strm, bi_ptr(in), + data_process_len, bi_end(out), + out->size - buffer_len(out)); + if (ret < 0) + return -1; + out->i += ret; + + } else { + ret = s->comp_algo->add_data(&s->comp_ctx.strm, bi_ptr(in), bi_contig_data(in), bi_end(out), out->size - buffer_len(out)); + if (ret < 0) + return -1; + out->i += ret; + ret = s->comp_algo->add_data(&s->comp_ctx.strm, in->data, left, bi_end(out), out->size - buffer_len(out)); + if (ret < 0) + return -1; + out->i += ret; + } + + b_adv(in, data_process_len); + msg->chunk_len -= data_process_len; + + return 0; +} + +/* + * Flush data in process, and write the header and footer of the chunk. Upon + * success, in and out buffers are swapped to avoid a copy. + */ +int http_compression_buffer_end(struct session *s, struct buffer **in, struct buffer **out, int end) +{ + int to_forward; + int left; + struct http_msg *msg = &s->txn.rsp; + struct buffer *ib = *in, *ob = *out; + int ret; + + /* flush data here */ + + if (end) + ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_FINISH); /* end of data */ + else + ret = s->comp_algo->flush(&s->comp_ctx, ob, Z_SYNC_FLUSH); /* end of buffer */ + + if (ret < 0) + return -1; /* flush failed */ + + if (ob->i > 8) { + /* more than a chunk size => some data were emitted */ + char *tail = ob->p + ob->i; + + /* write real size at the begining of the chunk, no need of wrapping */ + http_emit_chunk_size(ob->p, ob->i - 8, 0); + + /* chunked encoding requires CRLF after data */ + *tail++ = '\r'; + *tail++ = '\n'; + + if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->chunk_len == 0) { + /* End of data, 0 is needed but we're not + * in chunked mode on input so we must add it ourselves. + */ + memcpy(tail, "0\r\n\r\n", 5); + tail += 5; + } + ob->i = tail - ob->p; + } else { + /* no data were sent, cancel the chunk size */ + ob->i = 0; + } + + to_forward = ob->i; + + /* copy the remaining data in the tmp buffer. */ + if (ib->i > 0) { + left = ib->i - bi_contig_data(ib); + memcpy(bi_end(ob), bi_ptr(ib), bi_contig_data(ib)); + ob->i += bi_contig_data(ib); + if (left > 0) { + memcpy(bi_end(ob), ib->data, left); + ob->i += left; + } + } + + /* swap the buffers */ + *in = ob; + *out = ib; + + /* forward the new chunk without remaining data */ + b_adv(ob, to_forward); + + /* if there are data between p and next, there are trailers, must forward them */ + b_adv(ob, msg->next); + msg->next = 0; + + return to_forward; +} + + +/**************************** + **** Identity algorithm **** + ****************************/ + +/* + * Init the identity algorithm + */ +int identity_init(void *v, int level) +{ + return 0; +} + +/* + * Process data + * Return size of processed data or -1 on error + */ +int identity_add_data(void *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len) +{ + if (out_len < in_len) + return -1; + + memcpy(out_data, in_data, in_len); + + return in_len; +} + +int identity_flush(void *comp_ctx, struct buffer *out, int flag) +{ + return 0; +} + + +int identity_reset(void *comp_ctx) +{ + return 0; +} + +/* + * Deinit the algorithm + */ +int identity_end(void *comp_ctx) +{ + return 0; +} + + +#ifdef USE_ZLIB + +/************************** +**** gzip algorithm **** +***************************/ +int gzip_init(void *v, int level) +{ + z_stream *strm; + + strm = v; + + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + + if (deflateInit2(strm, level, Z_DEFLATED, MAX_WBITS + 16, 9, Z_DEFAULT_STRATEGY) != Z_OK) + return -1; + + return 0; +} +/************************** +**** Deflate algorithm **** +***************************/ + +int deflate_init(void *comp_ctx, int level) +{ + z_stream *strm; + + strm = comp_ctx; + + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + + if (deflateInit(strm, level) != Z_OK) + return -1; + + return 0; +} + +int deflate_add_data(void *comp_ctx, const char *in_data, int in_len, char *out_data, int out_len) +{ + z_stream *strm; + int ret; + + if (in_len <= 0) + return 0; + + + if (out_len <= 0) + return -1; + + strm = comp_ctx; + + strm->next_in = (unsigned char *)in_data; + strm->avail_in = in_len; + strm->next_out = (unsigned char *)out_data; + strm->avail_out = out_len; + + ret = deflate(strm, Z_NO_FLUSH); + if (ret != Z_OK) + return -1; + + /* deflate update the available data out */ + + return out_len - strm->avail_out; +} + +int deflate_flush(void *comp_ctx, struct buffer *out, int flag) +{ + int ret; + z_stream *strm; + int out_len = 0; + + strm = comp_ctx; + strm->next_out = (unsigned char *)bi_end(out); + strm->avail_out = out->size - buffer_len(out); + + ret = deflate(strm, flag); + if (ret != Z_OK && ret != Z_STREAM_END) + return -1; + + out_len = (out->size - buffer_len(out)) - strm->avail_out; + out->i += out_len; + + return out_len; +} + +int deflate_reset(void *comp_ctx) +{ + z_stream *strm; + + strm = comp_ctx; + if (deflateReset(strm) == Z_OK) + return 0; + return -1; +} + +int deflate_end(void *comp_ctx) +{ + z_stream *strm; + + strm = comp_ctx; + if (deflateEnd(strm) == Z_OK) + return 0; + + return -1; +} + +#endif /* USE_ZLIB */ + diff --git a/src/proto_http.c b/src/proto_http.c index 4377573d9..720d66bf3 100644 --- a/src/proto_http.c +++ b/src/proto_http.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -1966,6 +1967,143 @@ static inline int http_skip_chunk_crlf(struct http_msg *msg) return 1; } + +/* + * Selects a compression algorithm depending on the client request. +*/ + +int select_compression_request_header(struct session *s, struct buffer *req) +{ + struct http_txn *txn = &s->txn; + struct hdr_ctx ctx; + struct comp_algo *comp_algo = NULL; + + ctx.idx = 0; + /* no compression when Cache-Control: no-transform found */ + while (http_find_header2("Cache-Control", 13, req->p, &txn->hdr_idx, &ctx)) { + if (word_match(ctx.line + ctx.val, ctx.vlen, "no-transform", 12)) { + s->comp_algo = NULL; + return 0; + } + } + + /* search for the algo in the backend in priority or the frontend */ + if ((s->be->comp && (comp_algo = s->be->comp->algos)) || (s->fe->comp && (comp_algo = s->fe->comp->algos))) { + ctx.idx = 0; + while (http_find_header2("Accept-Encoding", 15, req->p, &txn->hdr_idx, &ctx)) { + for (; comp_algo; comp_algo = comp_algo->next) { + if (word_match(ctx.line + ctx.val, ctx.vlen, comp_algo->name, comp_algo->name_len)) { + s->comp_algo = comp_algo; + return 1; + } + } + } + } + + /* identity is implicit does not require headers */ + if ((s->be->comp && (comp_algo = s->be->comp->algos)) || (s->fe->comp && (comp_algo = s->fe->comp->algos))) { + for (; comp_algo; comp_algo = comp_algo->next) { + if (comp_algo->add_data == identity_add_data) { + s->comp_algo = comp_algo; + return 1; + } + } + } + + s->comp_algo = NULL; + + return 0; +} + +/* + * Selects a comression algorithm depending of the server response. + */ +int select_compression_response_header(struct session *s, struct buffer *res) +{ + struct http_txn *txn = &s->txn; + struct http_msg *msg = &txn->rsp; + struct hdr_ctx ctx; + struct comp_type *comp_type; + char *hdr_val; + int hdr_len; + + /* no common compression algorithm was found in request header */ + if (s->comp_algo == NULL) + goto fail; + + /* HTTP < 1.1 should not be compressed */ + if (!(msg->flags & HTTP_MSGF_VER_11)) + goto fail; + + hdr_val = trash; + ctx.idx = 0; + + /* Content-Length is null */ + if (!(msg->flags & HTTP_MSGF_TE_CHNK) && msg->body_len == 0) + goto fail; + + /* content is already compressed */ + if (http_find_header2("Content-Encoding", 16, res->p, &txn->hdr_idx, &ctx)) + goto fail; + + comp_type = NULL; + + /* if there was a compression content-type option in the backend or the frontend + * The backend have priority. + */ + if ((s->be->comp && (comp_type = s->be->comp->types)) || (s->fe->comp && (comp_type = s->fe->comp->types))) { + if (http_find_header2("Content-Type", 12, res->p, &txn->hdr_idx, &ctx)) { + for (; comp_type; comp_type = comp_type->next) { + if (strncmp(ctx.line+ctx.val, comp_type->name, comp_type->name_len) == 0) + /* this Content-Type should be compressed */ + break; + } + } + /* this Content-Type should not be compressed */ + if (comp_type == NULL) + goto fail; + } + + ctx.idx = 0; + + /* remove Content-Length header */ + if ((msg->flags & HTTP_MSGF_CNT_LEN) && http_find_header2("Content-Length", 14, res->p, &txn->hdr_idx, &ctx)) + http_remove_header2(msg, &txn->hdr_idx, &ctx); + + /* add Transfer-Encoding header */ + if (!(msg->flags & HTTP_MSGF_TE_CHNK)) + http_header_add_tail2(&txn->rsp, &txn->hdr_idx, "Transfer-Encoding: chunked", 26); + + /* + * Add Content-Encoding header when it's not identity encoding. + * RFC 2616 : Identity encoding: This content-coding is used only in the + * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding + * header. + */ + if (s->comp_algo->add_data != identity_add_data) { + hdr_len = 18; + memcpy(hdr_val, "Content-Encoding: ", hdr_len); + memcpy(hdr_val + hdr_len, s->comp_algo->name, s->comp_algo->name_len); + hdr_len += s->comp_algo->name_len; + hdr_val[hdr_len] = '\0'; + http_header_add_tail2(&txn->rsp, &txn->hdr_idx, hdr_val, hdr_len); + } + + /* initialize compression */ + if (s->comp_algo->init(&s->comp_ctx.strm, 1) < 0) + goto fail; + + return 1; + +fail: + if (s->comp_algo) { + s->comp_algo->end(&s->comp_ctx.strm); + s->comp_algo = NULL; + } + return 0; +} + + /* This stream analyser waits for a complete HTTP request. It returns 1 if the * processing can continue on next analysers, or zero if it either needs more * data or wants to immediately abort the request (eg: timeout, error, ...). It @@ -3328,6 +3466,9 @@ int http_process_request(struct session *s, struct channel *req, int an_bit) req->buf->i, req->analysers); + if (s->fe->comp || s->be->comp) + select_compression_request_header(s, req->buf); + /* * Right now, we know that we have processed the entire headers * and that unwanted requests have been filtered out. We can do @@ -4956,6 +5097,9 @@ int http_wait_for_response(struct session *s, struct channel *rep, int an_bit) msg->body_len = msg->chunk_len = cl; } + if (s->fe->comp || s->be->comp) + select_compression_response_header(s, rep->buf); + /* FIXME: we should also implement the multipart/byterange method. * For now on, we resort to close mode in this case (unknown length). */ @@ -5350,6 +5494,8 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi struct http_txn *txn = &s->txn; struct http_msg *msg = &s->txn.rsp; unsigned int bytes; + static struct buffer *tmpbuf = NULL; + int compressing = 0; if (unlikely(msg->msg_state < HTTP_MSG_BODY)) return 0; @@ -5368,13 +5514,24 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi /* in most states, we should abort in case of early close */ channel_auto_close(res); + /* no data */ + if (res->buf->i == 0) + return 0; + + /* this is the first time we need the compression buffer */ + if (s->comp_algo != NULL && tmpbuf == NULL) { + if ((tmpbuf = pool_alloc2(pool2_buffer)) == NULL) + goto aborted_xfer; /* no memory */ + } + if (msg->msg_state < HTTP_MSG_CHUNK_SIZE) { /* we have msg->sov which points to the first byte of message body. - * rep->buf->p still points to the beginning of the message and msg->sol - * is still null. We must save the body in msg->next because it - * survives buffer re-alignments. + * rep->buf.p still points to the beginning of the message and msg->sol + * is still null. We forward the headers, we don't need them. */ - msg->next = msg->sov; + channel_forward(res, msg->sov); + msg->next = 0; + msg->sov = 0; if (msg->flags & HTTP_MSGF_TE_CHNK) msg->msg_state = HTTP_MSG_CHUNK_SIZE; @@ -5382,20 +5539,32 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi msg->msg_state = HTTP_MSG_DATA; } + if (s->comp_algo != NULL) { + int ret = http_compression_buffer_init(s, res->buf, tmpbuf); /* init a buffer with headers */ + if (ret < 0) + goto missing_data; /* not enough spaces in buffers */ + compressing = 1; + } + while (1) { http_silent_debug(__LINE__, s); /* we may have some data pending between sol and sov */ - bytes = msg->sov - msg->sol; - if (msg->chunk_len || bytes) { - msg->sol = msg->sov; - msg->next -= bytes; /* will be forwarded */ - msg->chunk_len += bytes; - msg->chunk_len -= channel_forward(res, msg->chunk_len); + if (s->comp_algo == NULL) { + bytes = msg->sov - msg->sol; + if (msg->chunk_len || bytes) { + msg->sol = msg->sov; + msg->next -= bytes; /* will be forwarded */ + msg->chunk_len += bytes; + msg->chunk_len -= channel_forward(res, msg->chunk_len); + } } if (msg->msg_state == HTTP_MSG_DATA) { /* must still forward */ - if (res->to_forward) + if (compressing) + http_compression_buffer_add_data(s, res->buf, tmpbuf); + + if (res->to_forward || msg->chunk_len) goto missing_data; /* nothing left to forward */ @@ -5418,6 +5587,13 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_CHUNK_SIZE, s->fe); goto return_bad_res; } + /* skipping data if we are in compression mode */ + if (compressing && msg->chunk_len > 0) { + b_adv(res->buf, msg->next); + msg->next = 0; + msg->sov = 0; + msg->sol = 0; + } /* otherwise we're in HTTP_MSG_DATA or HTTP_MSG_TRAILERS state */ } else if (msg->msg_state == HTTP_MSG_CHUNK_CRLF) { @@ -5431,6 +5607,13 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_CHUNK_CRLF, s->fe); goto return_bad_res; } + /* skipping data in buffer for compression */ + if (compressing) { + b_adv(res->buf, msg->next); + msg->next = 0; + msg->sov = 0; + msg->sol = 0; + } /* we're in MSG_CHUNK_SIZE now */ } else if (msg->msg_state == HTTP_MSG_TRAILERS) { @@ -5443,11 +5626,20 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi http_capture_bad_message(&s->be->invalid_rep, s, msg, HTTP_MSG_TRAILERS, s->fe); goto return_bad_res; } + if (compressing) { + http_compression_buffer_end(s, &res->buf, &tmpbuf, 1); + compressing = 0; + } /* we're in HTTP_MSG_DONE now */ } else { int old_state = msg->msg_state; + if (compressing) { + http_compression_buffer_end(s, &res->buf, &tmpbuf, 1); + compressing = 0; + } + /* other states, DONE...TUNNEL */ /* for keep-alive we don't want to forward closes on DONE */ if ((txn->flags & TX_CON_WANT_MSK) == TX_CON_WANT_KAL || @@ -5476,6 +5668,10 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi } missing_data: + if (compressing) { + http_compression_buffer_end(s, &res->buf, &tmpbuf, 0); + compressing = 0; + } /* stop waiting for data if the input is closed before the end */ if (res->flags & CF_SHUTR) { if (!(s->flags & SN_ERR_MASK)) @@ -5494,12 +5690,14 @@ int http_response_forward_body(struct session *s, struct channel *res, int an_bi goto return_bad_res; /* forward any data pending between sol and sov */ - bytes = msg->sov - msg->sol; - if (msg->chunk_len || bytes) { - msg->sol = msg->sov; - msg->next -= bytes; /* will be forwarded */ - msg->chunk_len += bytes; - msg->chunk_len -= channel_forward(res, msg->chunk_len); + if (s->comp_algo == NULL) { + bytes = msg->sov - msg->sol; + if (msg->chunk_len || bytes) { + msg->sol = msg->sov; + msg->next -= bytes; /* will be forwarded */ + msg->chunk_len += bytes; + msg->chunk_len -= channel_forward(res, msg->chunk_len); + } } /* When TE: chunked is used, we need to get there again to parse remaining diff --git a/src/session.c b/src/session.c index 77a214de8..f148cb8c2 100644 --- a/src/session.c +++ b/src/session.c @@ -347,6 +347,7 @@ int session_complete(struct session *s) */ s->be = s->fe; s->req = s->rep = NULL; /* will be allocated later */ + s->comp_algo = NULL; /* Let's count a session now */ proxy_inc_fe_sess_ctr(l, p); @@ -551,6 +552,11 @@ static void session_free(struct session *s) sess_change_server(s, NULL); } + if (s->comp_algo) { + s->comp_algo->end(&s->comp_ctx.strm); + s->comp_algo = NULL; + } + if (s->req->pipe) put_pipe(s->req->pipe);