Index: subversion/include/svn_io.h =================================================================== --- subversion/include/svn_io.h (revision 5509) +++ subversion/include/svn_io.h (working copy) @@ -397,6 +397,19 @@ svn_stream_t *svn_stream_from_stringbuf (svn_stringbuf_t *str, apr_pool_t *pool); +/** Wrap compression functions around the stream @a stream. + * + * Return a stream that decompresses all data read and compresses all + * data written. The stream @a stream is used to read and write all + * compressed data. All compression data structures are allocated on + * @a pool. If compression support is not compiled in then @c + * svn_stream_compressed() returns @a stream unmodified. Make sure you + * call @c svn_stream_close() on the stream returned by this function, + * so that all data is flushed and cleaned up. + */ +svn_stream_t *svn_stream_compressed (svn_stream_t *stream, + apr_pool_t *pool); + /** Read from a generic stream. */ svn_error_t *svn_stream_read (svn_stream_t *stream, char *buffer, apr_size_t *len); Index: subversion/libsvn_subr/stream.c =================================================================== --- subversion/libsvn_subr/stream.c (revision 5509) +++ subversion/libsvn_subr/stream.c (working copy) @@ -16,6 +16,8 @@ * ==================================================================== */ +#include "svn_private_config.h" + #include #include @@ -25,6 +27,11 @@ #include #include +#ifdef SVN_HAVE_ZLIB +#include +#endif + +#include "svn_pools.h" #include "svn_io.h" #include "svn_error.h" #include "svn_string.h" @@ -250,7 +257,293 @@ return stream; } + +/* Compressed stream support */ +#ifdef SVN_HAVE_ZLIB + +#define ZBUFFER_SIZE 4096 /* The size of the buffer the + compressed stream uses to read from + the substream. Basically an + arbitrary value, picked to be about + page-sized. */ + +struct zbaton { + z_stream *in; /* compressed stream for reading */ + z_stream *out; /* compressed stream for writing */ + svn_read_fn_t read; /* substream's read function */ + svn_write_fn_t write; /* substream's write function */ + svn_close_fn_t close; /* substream's close function */ + void *read_buffer; /* buffer used for reading from + substream */ + int read_flush; /* what flush mode to use while + reading */ + apr_pool_t *pool; /* The pool this baton is allocated + on */ + void *subbaton; /* The substream's baton */ +}; + +/* zlib alloc function. opaque is the pool we need. */ +static voidpf +zalloc(voidpf opaque, uInt items, uInt size) +{ + apr_pool_t *pool = opaque; + + return apr_palloc(pool, items * size); +} + +/* zlib free function */ +static void +zfree(voidpf opaque, voidpf address) +{ + /* Empty, since we allocate on the pool */ +} + +/* Converts a zlib error to an svn_error_t. zerr is the error code, + function is the function name, and stream is the z_stream we are + using. */ +static svn_error_t * +zerr_to_svn_error (int zerr, const char *function, z_stream *stream) +{ + apr_status_t status; + char *message; + + if (zerr == Z_OK) + return SVN_NO_ERROR; + + switch (zerr) + { + case Z_STREAM_ERROR: + status = SVN_ERR_STREAM_MALFORMED_DATA; + message = "stream error"; + break; + + case Z_MEM_ERROR: + status = APR_ENOMEM; + message = "out of memory"; + break; + + case Z_BUF_ERROR: + status = APR_ENOMEM; + message = "buffer error"; + break; + + case Z_VERSION_ERROR: + status = SVN_ERR_STREAM_UNRECOGNIZED_DATA; + message = "version error"; + break; + + case Z_DATA_ERROR: + status = SVN_ERR_STREAM_MALFORMED_DATA; + message = "corrupted data"; + break; + + default: + status = SVN_ERR_STREAM_UNRECOGNIZED_DATA; + message = "error"; + break; + } + + if (stream->msg != NULL) + return svn_error_createf (status, NULL, "zlib (%s): %s: %s", function, + message, stream->msg); + else + return svn_error_createf (status, NULL, "zlib (%s): %s", function, + message); +} + +/* Helper function to figure out the sync mode */ +static svn_error_t * +read_helper_gz (svn_read_fn_t read, void *baton, char *buffer, + apr_size_t *len, int *zflush) +{ + apr_size_t orig_len = *len; + + SVN_ERR ((*read) (baton, buffer, len)); + + /* I wanted to use Z_FINISH here, but we need to know our buffer is + big enough */ + *zflush = (*len) < orig_len ? Z_SYNC_FLUSH : Z_SYNC_FLUSH; + + return SVN_NO_ERROR; +} + +/* Handle reading from a compressed stream */ +static svn_error_t * +read_handler_gz (void *baton, char *buffer, apr_size_t *len) +{ + struct zbaton *btn = baton; + int zerr; + + if (btn->in == NULL) + { + btn->in = apr_palloc (btn->pool, sizeof (z_stream)); + btn->in->zalloc = zalloc; + btn->in->zfree = zfree; + btn->in->opaque = btn->pool; + btn->read_buffer = apr_palloc(btn->pool, ZBUFFER_SIZE); + btn->in->next_in = btn->read_buffer; + btn->in->avail_in = ZBUFFER_SIZE; + + SVN_ERR (read_helper_gz (btn->read, btn->subbaton, btn->read_buffer, + &btn->in->avail_in, &btn->read_flush)); + + zerr = inflateInit (btn->in); + SVN_ERR (zerr_to_svn_error (zerr, "inflateInit", btn->in)); + } + + btn->in->next_out = buffer; + btn->in->avail_out = *len; + + while (btn->in->avail_out > 0) + { + if (btn->in->avail_in <= 0) + { + btn->in->avail_in = ZBUFFER_SIZE; + btn->in->next_in = btn->read_buffer; + SVN_ERR (read_helper_gz (btn->read, btn->subbaton, btn->read_buffer, + &btn->in->avail_in, &btn->read_flush)); + } + + zerr = inflate (btn->in, btn->read_flush); + if (zerr == Z_STREAM_END) + break; + else if (zerr != Z_OK) + return zerr_to_svn_error(zerr, "inflate", btn->in); + } + + *len -= btn->in->avail_out; + return SVN_NO_ERROR; +} + +/* Compress data and write it to the substream */ +static svn_error_t * +write_handler_gz (void *baton, const char *buffer, apr_size_t *len) +{ + struct zbaton *btn = baton; + apr_pool_t *subpool; + void *write_buf; + apr_size_t buf_size, write_len; + int zerr; + + if (btn->out == NULL) + { + btn->out = apr_palloc (btn->pool, sizeof (z_stream)); + btn->out->zalloc = zalloc; + btn->out->zfree = zfree; + btn->out->opaque = btn->pool; + + zerr = deflateInit (btn->out, Z_DEFAULT_COMPRESSION); + SVN_ERR (zerr_to_svn_error (zerr, "deflateInit", btn->out)); + } + + /* The largest buffer we should need is 0.1% larger than the + compressed data, + 12 bytes. This info comes from zlib.h. */ + buf_size = *len + (*len / 1000) + 13; + subpool = svn_pool_create (btn->pool); + write_buf = apr_palloc (subpool, buf_size); + + btn->out->next_in = (char *) buffer; + btn->out->avail_in = *len; + + while (btn->out->avail_in > 0) + { + btn->out->next_out = write_buf; + btn->out->avail_out = buf_size; + + zerr = deflate (btn->out, Z_NO_FLUSH); + SVN_ERR (zerr_to_svn_error (zerr, "deflate", btn->out)); + write_len = buf_size - btn->out->avail_out; + if (write_len > 0) + SVN_ERR (btn->write (btn->subbaton, write_buf, &write_len)); + } + + svn_pool_destroy (subpool); + + return SVN_NO_ERROR; +} + +/* Handle flushing and closing the stream */ +static svn_error_t * +close_handler_gz (void *baton) +{ + struct zbaton *btn = baton; + int zerr; + + if (btn->in != NULL) + { + zerr = inflateEnd(btn->in); + SVN_ERR (zerr_to_svn_error (zerr, "inflateEnd", btn->in)); + } + + if (btn->out != NULL) + { + void *buf; + apr_size_t write_len; + + buf = apr_palloc (btn->pool, ZBUFFER_SIZE); + + while (TRUE) + { + btn->out->next_out = buf; + btn->out->avail_out = ZBUFFER_SIZE; + + zerr = deflate (btn->out, Z_FINISH); + if (zerr != Z_STREAM_END && zerr != Z_OK) + return zerr_to_svn_error (zerr, "deflate", btn->out); + write_len = ZBUFFER_SIZE - btn->out->avail_out; + if (write_len > 0) + SVN_ERR (btn->write (btn->subbaton, buf, &write_len)); + if (zerr == Z_STREAM_END) + break; + } + + zerr = deflateEnd(btn->out); + SVN_ERR (zerr_to_svn_error (zerr, "deflateEnd", btn->out)); + } + + if (btn->close != NULL) + return btn->close (btn->subbaton); + else + return SVN_NO_ERROR; +} + +#endif /* SVN_HAVE_ZLIB */ + +svn_stream_t * +svn_stream_compressed (svn_stream_t *stream, apr_pool_t *pool) +{ +#ifdef SVN_HAVE_ZLIB + + struct svn_stream_t *zstream; + struct zbaton *baton; + + assert(stream != NULL); + + baton = apr_palloc (pool, sizeof (*baton)); + baton->in = baton->out = NULL; + baton->read = stream->read_fn; + baton->write = stream->write_fn; + baton->close = stream->close_fn; + baton->subbaton = stream->baton; + baton->pool = pool; + baton->read_buffer = NULL; + baton->read_flush = Z_SYNC_FLUSH; + + zstream = svn_stream_create(baton, pool); + svn_stream_set_read(zstream, read_handler_gz); + svn_stream_set_write(zstream, write_handler_gz); + svn_stream_set_close(zstream, close_handler_gz); + + return zstream; + +#else + + return stream; + +#endif /* SVN_HAVE_ZLIB */ +} + /* Miscellaneous stream functions. */ struct string_stream_baton Index: subversion/tests/libsvn_subr/stream-test.c =================================================================== --- subversion/tests/libsvn_subr/stream-test.c (revision 5509) +++ subversion/tests/libsvn_subr/stream-test.c (working copy) @@ -118,6 +118,120 @@ return SVN_NO_ERROR; } +/* generate some poorly compressable data */ +static svn_stringbuf_t * +generate_test_bytes(int num_bytes, apr_pool_t *pool) +{ + svn_stringbuf_t *buffer = svn_stringbuf_create("", pool); + int total, repeat, repeat_iter; + unsigned char c; + + for (total = 0, repeat = 1, c = 0; total < num_bytes; total++) + { + svn_stringbuf_appendbytes(buffer, &c, 1); + + repeat_iter--; + if (repeat_iter == 0) + { + if (c == 255) + repeat++; + c = (c + 1) % 255; + repeat_iter = repeat; + } + } + + return buffer; +} + + +static svn_error_t * +test_stream_compressed (const char **msg, + svn_boolean_t msg_only, + apr_pool_t *pool) +{ +#define NUM_TEST_STRINGS 5 +#define TEST_BUF_SIZE 10 +#define GENERATED_SIZE 20000 + + int i; + svn_stringbuf_t *bufs[NUM_TEST_STRINGS]; + apr_pool_t *subpool = svn_pool_create (pool); + + static const char * const strings[NUM_TEST_STRINGS - 1] = { + /* 0 */ + "", + /* 1 */ + "This is a string.", + /* 2 */ + "This is, by comparison to the previous string, a much longer string.", + /* 3 */ + "And if you thought that last string was long, you just wait until " + "I'm finished here. I mean, how can a string really claim to be long " + "when it fits on a single line of 80-columns? Give me a break. " + "Now, I'm not saying that I'm the longest string out there--far from " + "it--but I feel that it is safe to assume that I'm far longer than my " + "peers. And that demands some amount of respect, wouldn't you say?" + }; + + + *msg = "test compressed streams"; + + if (msg_only) + return SVN_NO_ERROR; + + for (i = 0; i < (NUM_TEST_STRINGS - 1); i++) + bufs[i] = svn_stringbuf_create (strings[i], pool); + + /* the last buffer is for the generated data */ + bufs[NUM_TEST_STRINGS - 1] = generate_test_bytes(GENERATED_SIZE, pool); + + for (i = 0; i < NUM_TEST_STRINGS; i++) + { + svn_stream_t *stream; + svn_stringbuf_t *origbuf, *inbuf, *outbuf; + char buf[TEST_BUF_SIZE]; + apr_size_t len; + + origbuf = bufs[i]; + inbuf = svn_stringbuf_create ("", subpool); + outbuf = svn_stringbuf_create ("", subpool); + + stream = svn_stream_compressed (svn_stream_from_stringbuf (outbuf, + subpool), + subpool); + len = origbuf->len; + SVN_ERR (svn_stream_write (stream, origbuf->data, &len)); + SVN_ERR (svn_stream_close (stream)); + + stream = svn_stream_compressed (svn_stream_from_stringbuf (outbuf, + subpool), + subpool); + len = TEST_BUF_SIZE; + while (len >= TEST_BUF_SIZE) + { + len = TEST_BUF_SIZE; + SVN_ERR (svn_stream_read (stream, buf, &len)); + if (len > 0) + svn_stringbuf_appendbytes (inbuf, buf, len); + } + + if (! svn_stringbuf_compare (inbuf, origbuf)) + return svn_error_create (SVN_ERR_TEST_FAILED, NULL, + "Got unexpected result."); + + SVN_ERR (svn_stream_close (stream)); + + svn_pool_clear (subpool); + } + +#undef NUM_TEST_STRINGS +#undef TEST_BUF_SIZE +#undef GENEREATED_SIZE + + svn_pool_destroy (subpool); + return SVN_NO_ERROR; +} + /* The test table. */ @@ -125,5 +239,6 @@ { SVN_TEST_NULL, SVN_TEST_PASS (test_stream_from_string), + SVN_TEST_PASS (test_stream_compressed), SVN_TEST_NULL }; Index: configure.in =================================================================== --- configure.in (revision 5509) +++ configure.in (working copy) @@ -330,6 +330,21 @@ fi ]) + +AC_ARG_WITH(zlib, +AC_HELP_STRING([--with-zlib], [enable zlib support]), +[ + + if test "$withval" = "yes" ; then + AC_CHECK_HEADER(zlib.h, [ + AC_CHECK_LIB(z, inflate, [ + AC_DEFINE([SVN_HAVE_ZLIB], [1], [Is zlib support enabled?]) + LIBS="$LIBS -lz" + ]) + ]) + fi + +]) MOD_ACTIVATION="-a" AC_ARG_ENABLE(mod-activation,