On Mon, Dec 1, 2008 at 8:59 AM, <hwright_at_tigris.org> wrote:
> Author: hwright
> Date: Mon Dec 1 08:59:39 2008
> New Revision: 34502
>
> Log:
> Merge the fsfs-pack branch to trunk.
>
> For a complete log, please see log messages on the branch.
>
> Added:
> trunk/subversion/tests/libsvn_fs_fs/ (props changed)
> - copied from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/
> Replaced:
> trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c
> - copied unchanged from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c
> Modified:
> trunk/ (props changed)
> trunk/build.conf
> trunk/subversion/include/svn_fs.h
> trunk/subversion/include/svn_repos.h
> trunk/subversion/libsvn_fs/fs-loader.c
> trunk/subversion/libsvn_fs/fs-loader.h
> trunk/subversion/libsvn_fs_base/fs.c
> trunk/subversion/libsvn_fs_fs/caching.c
> trunk/subversion/libsvn_fs_fs/fs.c
> trunk/subversion/libsvn_fs_fs/fs.h
> trunk/subversion/libsvn_fs_fs/fs_fs.c
> trunk/subversion/libsvn_fs_fs/fs_fs.h
> trunk/subversion/libsvn_fs_fs/structure
> trunk/subversion/libsvn_repos/fs-wrap.c
> trunk/subversion/svnadmin/main.c
>
> Merged:
> /branches/fsfs-pack:r33643-34501
>
> Modified: trunk/build.conf
> URL: http://svn.collab.net/viewvc/svn/trunk/build.conf?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/build.conf Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/build.conf Mon Dec 1 08:59:39 2008 (r34502)
> @@ -3,7 +3,7 @@
> #
> ######################################################################
> #
> -# Copyright (c) 2000-2006 CollabNet. All rights reserved.
> +# Copyright (c) 2000-2008 CollabNet. All rights reserved.
> #
> # This software is licensed as described in the file COPYING, which
> # you should have received as part of this distribution. The terms
> @@ -611,6 +611,17 @@ libs = libsvn_test libsvn_fs libsvn_fs_b
> libsvn_subr apriconv apr
>
> # ----------------------------------------------------------------------------
> +# Tests for libsvn_fs_fs
> +[fs-pack-test]
> +description = Test fsfs packing in libsvn_fs_fs
> +type = exe
> +path = subversion/tests/libsvn_fs_fs
> +sources = fs-pack-test.c
> +install = test
> +libs = libsvn_test libsvn_fs libsvn_fs_fs libsvn_delta
> + libsvn_subr apriconv apr
> +
> +# ----------------------------------------------------------------------------
> # Tests for libsvn_fs
>
> [locks-test]
> @@ -963,7 +974,7 @@ libs = svn svnserve svnadmin svnlook svn
> type = project
> path = build/win32
> libs = __ALL__
> - fs-test fs-base-test skel-test key-test strings-reps-test changes-test locks-test
> + fs-test fs-base-test fs-fsfs-test skel-test key-test strings-reps-test changes-test locks-test
> repos-test
> checksum-test compat-test config-test hashdump-test mergeinfo-test opt-test path-test stream-test
> string-test time-test utf-test target-test error-test cache-test
>
> Modified: trunk/subversion/include/svn_fs.h
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/include/svn_fs.h?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/include/svn_fs.h Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/include/svn_fs.h Mon Dec 1 08:59:39 2008 (r34502)
> @@ -2024,6 +2024,20 @@ svn_error_t *
> svn_fs_print_modules(svn_stringbuf_t *output,
> apr_pool_t *pool);
>
> +
> +/**
> + * Possibly update the filesystem located in the directory @a path
> + * to use disk space more efficiently.
> + *
> + * @since New in 1.6.
> + */
> +svn_error_t *
> +svn_fs_pack(const char *db_path,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool);
> +
> +
> /** @} */
>
> #ifdef __cplusplus
>
> Modified: trunk/subversion/include/svn_repos.h
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/include/svn_repos.h?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/include/svn_repos.h Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/include/svn_repos.h Mon Dec 1 08:59:39 2008 (r34502)
> @@ -306,6 +306,20 @@ svn_repos_hotcopy(const char *src_path,
> svn_boolean_t clean_logs,
> apr_pool_t *pool);
>
> +
> +/**
> + * Possibly update the repository, @a repos, to use a more efficient
> + * filesystem representation. Use @a pool for allocations.
> + *
> + * @since New in 1.6.
> + */
> +svn_error_t *
> +svn_repos_fs_pack(svn_repos_t *repos,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool);
> +
> +
> /**
> * Run database recovery procedures on the repository at @a path,
> * returning the database to a consistent state. Use @a pool for all
>
> Modified: trunk/subversion/libsvn_fs/fs-loader.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs/fs-loader.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs/fs-loader.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs/fs-loader.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -488,6 +488,18 @@ svn_fs_hotcopy(const char *src_path, con
> }
>
> svn_error_t *
> +svn_fs_pack(const char *path,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool)
> +{
> + fs_library_vtable_t *vtable;
> +
> + SVN_ERR(fs_library_vtable(&vtable, path, pool));
> + return vtable->pack(path, cancel_func, cancel_baton, pool);
> +}
> +
> +svn_error_t *
> svn_fs_recover(const char *path,
> svn_cancel_func_t cancel_func, void *cancel_baton,
> apr_pool_t *pool)
>
> Modified: trunk/subversion/libsvn_fs/fs-loader.h
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs/fs-loader.h?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs/fs-loader.h Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs/fs-loader.h Mon Dec 1 08:59:39 2008 (r34502)
> @@ -88,6 +88,8 @@ typedef struct fs_library_vtable_t
> svn_error_t *(*recover)(svn_fs_t *fs,
> svn_cancel_func_t cancel_func, void *cancel_baton,
> apr_pool_t *pool);
> + svn_error_t *(*pack)(const char *path, svn_cancel_func_t cancel_func,
> + void *cancel_baton, apr_pool_t *pool);
>
> /* Provider-specific functions should go here, even if they could go
> in an object vtable, so that they are all kept together. */
>
> Modified: trunk/subversion/libsvn_fs_base/fs.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_base/fs.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_base/fs.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_base/fs.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -877,6 +877,16 @@ base_bdb_recover(svn_fs_t *fs,
> return bdb_recover(fs->path, FALSE, pool);
> }
>
> +static svn_error_t *
> +base_bdb_pack(const char *path,
> + svn_cancel_func_t cancel,
> + void *cancel_baton,
> + apr_pool_t *pool)
> +{
> + /* Packing is currently a no op for BDB. */
> + return SVN_NO_ERROR;
> +}
> +
>
>
> /* Running the 'archive' command on a Berkeley DB-based filesystem. */
> @@ -1327,6 +1337,7 @@ static fs_library_vtable_t library_vtabl
> base_hotcopy,
> base_get_description,
> base_bdb_recover,
> + base_bdb_pack,
> base_bdb_logfiles,
> svn_fs_base__id_parse
> };
>
> Modified: trunk/subversion/libsvn_fs_fs/caching.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/caching.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/caching.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/caching.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -104,6 +104,48 @@ dup_dir_listing(void **out,
> return SVN_NO_ERROR;
> }
>
> +
> +/** Caching packed rev offsets. **/
> +/* Implements svn_cache__serialize_func_t */
> +static svn_error_t *
> +offset_serialize(char **data,
> + apr_size_t *data_len,
> + void *in,
> + apr_pool_t *pool)
> +{
> + *data = apr_off_t_toa(pool, *((apr_off_t *) in));
> + *data_len = strlen(*data);
> + return SVN_NO_ERROR;
> +}
> +
> +/* Implements svn_cache__deserialize_func_t */
> +static svn_error_t *
> +offset_deserialize(void **out,
> + const char *data,
> + apr_size_t data_len,
> + apr_pool_t *pool)
> +{
> + *out = apr_palloc(pool, sizeof (apr_off_t));
> + apr_strtoff((apr_off_t *)*out, data, NULL, 0);
> +
> + return SVN_NO_ERROR;
> +}
> +
> +/* Implements svn_cache__dup_func_t */
> +static svn_error_t *
> +dup_pack_offset(void **out,
> + void *in,
> + apr_pool_t *pool)
> +{
> + apr_off_t *offset = apr_palloc(pool, sizeof(*offset));
> +
> + *offset = *((apr_off_t *) in);
> +
> + *out = offset;
> + return SVN_NO_ERROR;
> +}
> +
> +
> /* Return a memcache in *MEMCACHE_P for FS if it's configured to use
> memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean
> indicating whether cache errors should be returned to the caller or
> @@ -218,6 +260,25 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
> SVN_ERR(svn_cache__set_error_handler(ffd->dir_cache,
> warn_on_cache_errors, fs, pool));
>
> + /* Only 16 bytes per entry (a revision number + the corresponding offset). */
> + if (memcache)
> + SVN_ERR(svn_cache__create_memcache(&(ffd->packed_offset_cache),
> + memcache,
> + offset_serialize,
> + offset_deserialize,
> + sizeof(svn_revnum_t),
> + apr_pstrcat(pool, prefix, "PACK-OFFSET",
> + NULL),
> + fs->pool));
> + else
> + SVN_ERR(svn_cache__create_inprocess(&(ffd->packed_offset_cache),
> + dup_pack_offset, sizeof(svn_revnum_t),
> + 16, 1024, FALSE, fs->pool));
It doesn't matter too much, but the idea behind the size estimation is
to try to get each page to be about 8K (ie, the minimum pool
allocation), so you probably want like 500 per page here.
> +
> + if (! no_handler)
> + SVN_ERR(svn_cache__set_error_handler(ffd->packed_offset_cache,
> + warn_on_cache_errors, fs, pool));
> +
> if (memcache)
> {
> SVN_ERR(svn_cache__create_memcache(&(ffd->fulltext_cache),
>
> Modified: trunk/subversion/libsvn_fs_fs/fs.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/fs.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/fs.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -321,6 +321,7 @@ static fs_library_vtable_t library_vtabl
> fs_hotcopy,
> fs_get_description,
> svn_fs_fs__recover,
> + svn_fs_fs__pack,
> fs_logfiles
> };
>
>
> Modified: trunk/subversion/libsvn_fs_fs/fs.h
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs.h?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/fs.h Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/fs.h Mon Dec 1 08:59:39 2008 (r34502)
> @@ -52,6 +52,8 @@ extern "C" {
> #define PATH_TXN_CURRENT "txn-current" /* File with next txn key */
> #define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */
> #define PATH_LOCKS_DIR "locks" /* Directory of locks */
> +#define PATH_MAX_PACKED_REV "max-packed-rev" /* Youngest revision which
> + has been packed. */
> /* If you change this, look at tests/svn_test_fs.c(maybe_install_fsfs_conf) */
> #define PATH_CONFIG "fsfs.conf" /* Configuration */
>
> @@ -104,6 +106,9 @@ extern "C" {
> /* The minimum format number that allows rep sharing. */
> #define SVN_FS_FS__MIN_REP_SHARING_FORMAT 4
>
> +/* The minimum format number that supports packed shards. */
> +#define SVN_FS_FS__MIN_PACKED_FORMAT 4
> +
> /* Private FSFS-specific data shared between all svn_txn_t objects that
> relate to a particular transaction in a filesystem (as identified
> by transaction id and filesystem UUID). Objects of this type are
> @@ -218,11 +223,18 @@ typedef struct
> rep key to svn_string_t. */
> svn_cache__t *fulltext_cache;
>
> + /* Pack manifest cache; maps revision numbers to offsets in their respective
> + pack files. */
> + svn_cache__t *packed_offset_cache;
> +
> /* Data shared between all svn_fs_t objects for a given filesystem. */
> fs_fs_shared_data_t *shared;
>
> /* The sqlite database used for rep caching. */
> struct rep_cache_t rep_cache;
> +
> + /* The youngest revision in a pack file. */
> + svn_revnum_t max_packed_rev;
> } fs_fs_data_t;
>
>
>
> Modified: trunk/subversion/libsvn_fs_fs/fs_fs.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs_fs.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -103,6 +103,10 @@
> #define REP_PLAIN "PLAIN"
> #define REP_DELTA "DELTA"
>
> +/* The size of an entry in the pack file manifest, not including terminating
> + characters. */
> +#define PACK_MANIFEST_ENTRY_LEN 19
> +
> /* Notes:
>
> To avoid opening and closing the rev-files all the time, it would
> @@ -165,6 +169,19 @@ path_lock(svn_fs_t *fs, apr_pool_t *pool
> }
>
> static const char *
> +path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind,
> + apr_pool_t *pool)
> +{
> + fs_fs_data_t *ffd = fs->fsap_data;
> +
> + assert(ffd->max_files_per_dir);
> + return svn_path_join_many(pool, fs->path, PATH_REVS_DIR,
> + apr_psprintf(pool, "%ld.%s",
> + rev / ffd->max_files_per_dir, kind),
> + NULL);
> +}
> +
> +static const char *
> path_rev_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
> {
> fs_fs_data_t *ffd = fs->fsap_data;
> @@ -247,6 +264,12 @@ path_txn_next_ids(svn_fs_t *fs, const ch
> }
>
> static APR_INLINE const char *
> +path_max_packed_rev(svn_fs_t *fs, apr_pool_t *pool)
> +{
> + return svn_path_join(fs->path, PATH_MAX_PACKED_REV, pool);
> +}
> +
> +static APR_INLINE const char *
> path_txn_proto_rev(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
> {
> fs_fs_data_t *ffd = fs->fsap_data;
> @@ -1020,6 +1043,25 @@ write_config(svn_fs_t *fs,
> }
>
> static svn_error_t *
> +read_max_packed_rev(svn_revnum_t *max_packed_rev,
> + const char *path,
> + apr_pool_t *pool)
> +{
> + char buf[80];
> + apr_file_t *file;
> + apr_size_t len;
> +
> + SVN_ERR(svn_io_file_open(&file, path, APR_READ | APR_BUFFERED,
> + APR_OS_DEFAULT, pool));
> + len = sizeof(buf);
> + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
> + SVN_ERR(svn_io_file_close(file, pool));
> +
> + *max_packed_rev = SVN_STR_TO_REV(buf);
> + return SVN_NO_ERROR;
> +}
> +
> +static svn_error_t *
> get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool);
>
> svn_error_t *
> @@ -1053,6 +1095,11 @@ svn_fs_fs__open(svn_fs_t *fs, const char
>
> SVN_ERR(svn_io_file_close(uuid_file, pool));
>
> + /* Read the max packed revision. */
> + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
> + SVN_ERR(read_max_packed_rev(&ffd->max_packed_rev,
> + path_max_packed_rev(fs, pool), pool));
> +
> /* Read the configuration file. */
> SVN_ERR(svn_config_read(&ffd->config,
> svn_path_join(fs->path, PATH_CONFIG, pool),
> @@ -1118,6 +1165,10 @@ upgrade_body(void *baton, apr_pool_t *po
> (svn_path_join(fs->path, PATH_TXN_PROTOS_DIR, pool), pool));
> }
>
> + /* If our filesystem is new enough, write the max packed rev file. */
> + if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
> + SVN_ERR(svn_io_file_create(path_max_packed_rev(fs, pool), "0\n", pool));
> +
> /* Bump the format file. */
> return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir,
> TRUE, pool);
> @@ -1294,6 +1345,9 @@ svn_fs_fs__hotcopy(const char *src_path,
> /* Copy the uuid. */
> SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_UUID, pool));
>
> + /* Copy the max packed rev. */
> + SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_MAX_PACKED_REV, pool));
> +
Doesn't this need to be conditional on it existing, or does
svn_io_dir_file_copy ignore file-not-found errors?
> /* Find the youngest revision from this current file. */
> SVN_ERR(get_youngest(&youngest, dst_path, pool));
>
> @@ -1514,6 +1568,80 @@ ensure_revision_exists(svn_fs_t *fs,
> _("No such revision %ld"), rev);
> }
>
> +/* Open the correct revision file for REV. If the filesystem FS has
> + been packed, *FILE will be set to the packed file; otherwise, set *FILE
> + to the revision file for REV. Use POOL for allocations. */
> +static svn_error_t *
> +open_pack_or_rev_file(apr_file_t **file,
> + svn_fs_t *fs,
> + svn_revnum_t rev,
> + apr_pool_t *pool)
> +{
> + fs_fs_data_t *ffd = fs->fsap_data;
> + svn_error_t *err;
> +
> + err = svn_io_file_open(file,
> + rev < ffd->max_packed_rev
If it is the max *packed* rev, then shouldn't this be <=? (This falls
into the category of "I'm not sure how it worked with this code", if
I'm correct...)
> + ? path_rev_packed(fs, rev, "pack", pool)
> + : svn_fs_fs__path_rev(fs, rev, pool),
> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
> +
> + if (err && APR_STATUS_IS_ENOENT(err->apr_err))
> + {
> + svn_error_clear(err);
> + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
> + _("No such revision %ld"), rev);
> + }
> +
> + return err;
> +}
> +
> +/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
> + Use POOL for temporary allocations. */
> +static svn_error_t *
> +get_packed_offset(apr_off_t *rev_offset,
> + svn_fs_t *fs,
> + svn_revnum_t rev,
> + apr_pool_t *pool)
> +{
> + fs_fs_data_t *ffd = fs->fsap_data;
> + apr_file_t *manifest_file;
> + char buf[PACK_MANIFEST_ENTRY_LEN + 1];
> + apr_off_t rev_index_offset;
> + svn_boolean_t is_cached;
> + apr_off_t *cached_rev_offset;
> + apr_size_t len;
> +
> + SVN_ERR(svn_cache__get((void **) &cached_rev_offset, &is_cached,
> + ffd->packed_offset_cache, &rev, pool));
> +
> + if (is_cached)
> + {
> + *rev_offset = *cached_rev_offset;
> + return SVN_NO_ERROR;
> + }
> +
> + /* Open the manifest file. */
> + SVN_ERR(svn_io_file_open(&manifest_file, path_rev_packed(fs, rev, "manifest",
> + pool),
> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
> +
> + /* Seek to the correct offset. */
Hmm. These files aren't actually that big, right? I would just read
the whole manifest into the cache instead of seeking and reading one
entry (which is likely to be the same amount of IO). Just make sure
that the cache is a reasonable size...
> + rev_index_offset = (rev % ffd->max_files_per_dir) *
> + (PACK_MANIFEST_ENTRY_LEN + 1);
> + SVN_ERR(svn_io_file_seek(manifest_file, APR_SET, &rev_index_offset, pool));
> +
> + /* Read the revision offset and close the file. */
> + len = PACK_MANIFEST_ENTRY_LEN;
> + SVN_ERR(svn_io_file_read(manifest_file, buf, &len, pool));
Is svn_io_file_read guaranteed to read the entire length, or do you
need to check len? (Yeah, it's short enough that a partial read seems
unlikely.)
> + buf[PACK_MANIFEST_ENTRY_LEN] = 0;
> + SVN_ERR(svn_io_file_close(manifest_file, pool));
> +
> + *rev_offset = apr_atoi64(buf);
> +
> + return svn_cache__set(ffd->packed_offset_cache, &rev, rev_offset, pool);
> +}
> +
> /* Open the revision file for revision REV in filesystem FS and store
> the newly opened file in FILE. Seek to location OFFSET before
> returning. Perform temporary allocations in POOL. */
> @@ -1524,12 +1652,20 @@ open_and_seek_revision(apr_file_t **file
> apr_off_t offset,
> apr_pool_t *pool)
> {
> + fs_fs_data_t *ffd = fs->fsap_data;
> apr_file_t *rev_file;
>
> SVN_ERR(ensure_revision_exists(fs, rev, pool));
>
> - SVN_ERR(svn_io_file_open(&rev_file, svn_fs_fs__path_rev(fs, rev, pool),
> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
> + SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, pool));
> +
> + if (rev < ffd->max_packed_rev)
ditto.
> + {
> + apr_off_t rev_offset;
> +
> + SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
> + offset += rev_offset;
> + }
>
> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
>
> @@ -2072,19 +2208,30 @@ read_rep_line(struct rep_args **rep_args
> _("Malformed representation header"));
> }
>
> -/* Given a revision file REV_FILE, find the Node-ID of the header
> - located at OFFSET and store it in *ID_P. Allocate temporary
> - variables from POOL. */
> +/* Given a revision file REV_FILE, opened to REV in FS, find the Node-ID
> + of the header located at OFFSET and store it in *ID_P. Allocate
> + temporary variables from POOL. */
> static svn_error_t *
> get_fs_id_at_offset(svn_fs_id_t **id_p,
> apr_file_t *rev_file,
> + svn_fs_t *fs,
> + svn_revnum_t rev,
> apr_off_t offset,
> apr_pool_t *pool)
> {
> + fs_fs_data_t *ffd = fs->fsap_data;
> svn_fs_id_t *id;
> apr_hash_t *headers;
> const char *node_id_str;
>
> + if (rev < ffd->max_packed_rev)
ditto.
> + {
> + apr_off_t rev_offset;
> +
> + SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
> + offset += rev_offset;
> + }
> +
> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
>
> SVN_ERR(read_header_block(&headers,
> @@ -2109,27 +2256,50 @@ get_fs_id_at_offset(svn_fs_id_t **id_p,
> }
>
>
> -/* Given an open revision file REV_FILE, locate the trailer that
> +/* Given an open revision file REV_FILE in FS for REV, locate the trailer that
> specifies the offset to the root node-id and to the changed path
> information. Store the root node offset in *ROOT_OFFSET and the
> changed path offset in *CHANGES_OFFSET. If either of these
> - pointers is NULL, do nothing with it. Allocate temporary variables
> - from POOL. */
> + pointers is NULL, do nothing with it. If PACKED is true, REV_FILE
> + should be a packed shard file. Allocate temporary variables from POOL. */
> static svn_error_t *
> get_root_changes_offset(apr_off_t *root_offset,
> apr_off_t *changes_offset,
> apr_file_t *rev_file,
> + svn_fs_t *fs,
> + svn_revnum_t rev,
> apr_pool_t *pool)
> {
> + fs_fs_data_t *ffd = fs->fsap_data;
> apr_off_t offset;
> char buf[64];
> int i, num_bytes;
> apr_size_t len;
> + apr_seek_where_t seek_relative;
> +
> + /* Determine where to seek to in the file.
> +
> + If we've got a pack file, we want to seek to the end of the desired
> + revision. But we don't track that, so we seek to the beginning of the
> + next revision.
> +
> + Unless the next revision is in a different file, in which case, we can
> + just seek to the end of the pack file -- just like we do in the
> + non-packed case. */
> + if ((rev < ffd->max_packed_rev) && ((rev + 1) % ffd->max_files_per_dir != 0))
ditto.
> + {
> + SVN_ERR(get_packed_offset(&offset, fs, rev + 1, pool));
> + seek_relative = APR_SET;
> + }
> + else
> + {
> + seek_relative = APR_END;
> + offset = 0;
> + }
>
> /* We will assume that the last line containing the two offsets
> will never be longer than 64 characters. */
> - offset = 0;
> - SVN_ERR(svn_io_file_seek(rev_file, APR_END, &offset, pool));
> + SVN_ERR(svn_io_file_seek(rev_file, seek_relative, &offset, pool));
>
> offset -= sizeof(buf);
> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
> @@ -2195,7 +2365,6 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **ro
> apr_file_t *revision_file;
> apr_off_t root_offset;
> svn_fs_id_t *root_id;
> - svn_error_t *err;
> svn_boolean_t is_cached;
>
> SVN_ERR(ensure_revision_exists(fs, rev, pool));
> @@ -2205,21 +2374,12 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **ro
> if (is_cached)
> return SVN_NO_ERROR;
>
> - err = svn_io_file_open(&revision_file, svn_fs_fs__path_rev(fs, rev, pool),
> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
> - if (err && APR_STATUS_IS_ENOENT(err->apr_err))
> - {
> - svn_error_clear(err);
> - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
> - _("No such revision %ld"), rev);
> - }
> - else if (err)
> - return err;
> -
> -
> - SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, pool));
> + SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
> + SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, fs, rev,
> + pool));
>
> - SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, root_offset, pool));
> + SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, fs, rev,
> + root_offset, pool));
>
> SVN_ERR(svn_io_file_close(revision_file, pool));
>
> @@ -3734,11 +3894,10 @@ svn_fs_fs__paths_changed(apr_hash_t **ch
>
> SVN_ERR(ensure_revision_exists(fs, rev, pool));
>
> - SVN_ERR(svn_io_file_open(&revision_file, svn_fs_fs__path_rev(fs, rev, pool),
> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
> + SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
>
> - SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file,
> - pool));
> + SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file, fs,
> + rev, pool));
>
> SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool));
>
> @@ -5651,6 +5810,10 @@ svn_fs_fs__create(svn_fs_t *fs,
> && rep_sharing_allowed)
> SVN_ERR(svn_fs_fs__open_rep_cache(fs, fs->pool));
>
> + /* Create the max packed rev file. */
> + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
> + SVN_ERR(svn_io_file_create(path_max_packed_rev(fs, pool), "0\n", pool));
> +
> /* Create the txn-current file if the repository supports
> the transaction sequence file. */
> if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
> @@ -5986,11 +6149,8 @@ recover_body(void *baton, apr_pool_t *po
> if (b->cancel_func)
> SVN_ERR(b->cancel_func(b->cancel_baton));
>
> - SVN_ERR(svn_io_file_open(&rev_file,
> - svn_fs_fs__path_rev(fs, rev, iterpool),
> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
> - iterpool));
> - SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file,
> + SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, iterpool));
> + SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, fs, rev,
> iterpool));
> SVN_ERR(recover_find_max_ids(fs, rev, rev_file, root_offset,
> max_node_id, max_copy_id, iterpool));
> @@ -6504,3 +6664,170 @@ svn_fs_fs__begin_txn(svn_fs_txn_t **txn_
>
> return svn_fs_fs__change_txn_props(*txn_p, props, pool);
> }
> +
> +
> +/****** Packing FSFS shards *********/
> +struct packer_baton
> +{
> + apr_off_t next_offset;
> + svn_stream_t *pack_stream;
> + svn_stream_t *manifest_stream;
> + svn_cancel_func_t cancel_func;
> + void *cancel_baton;
> +};
> +
> +/* Walker function used by pack_shard().
> + This implements svn_io_walk_func_t */
> +static svn_error_t *
> +packer_func(void *baton,
> + const char *path,
> + const apr_finfo_t *finfo,
> + apr_pool_t *pool)
> +{
> + struct packer_baton *pb = baton;
> + svn_stream_t *rev_stream;
> +
> + /* Ignore any directories we find. */
> + if (finfo->filetype == APR_DIR)
> + return SVN_NO_ERROR;
> +
> + /* Update the manifest. */
> + svn_stream_printf(pb->manifest_stream, pool,
> + "%-" APR_STRINGIFY(PACK_MANIFEST_ENTRY_LEN) APR_OFF_T_FMT"\n",
> + pb->next_offset);
> + pb->next_offset += finfo->size;
> +
> + /* Copy all the bits from the rev file to the end of the pack file. */
> + SVN_ERR(svn_stream_open_readonly(&rev_stream, path, pool, pool));
> + return svn_stream_copy3(rev_stream, svn_stream_disown(pb->pack_stream, pool),
> + pb->cancel_func, pb->cancel_baton, pool);
> +}
> +
> +/* Pack a single shard SHARD in REVS_DIR, using POOL for allocations.
> + CANCEL_FUNC and CANCEL_BATON are what you think they are.
> +
> + If for some reason we detect a partial packing already performed, we
> + remove the pack file and start again. */
> +static svn_error_t *
> +pack_shard(const char *revs_dir,
> + const char *fs_path,
> + apr_int64_t shard,
> + int max_files_per_dir,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool)
> +{
> + svn_stream_t *stream;
> + const char *tmp_path;
> + const char *final_path;
> + svn_error_t *err;
> + const char *pack_file_path = svn_path_join(
> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT ".pack",
> + shard), pool);
> + const char *manifest_file_path = svn_path_join(
> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT ".manifest",
> + shard), pool);
> + const char *shard_path = svn_path_join(
> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
> + pool);
> + struct packer_baton pb;
> +
> + /* Remove any existing pack file for this shard, since it is incomplete. */
> + err = svn_io_remove_file(pack_file_path, pool);
> + if (err)
> + {
> + if (APR_STATUS_IS_ENOENT(err->apr_err))
> + svn_error_clear(err);
> + else
> + return err;
> + }
> + else
> + SVN_ERR(svn_io_remove_file(manifest_file_path, pool));
> +
> + /* Create the new pack and manifest files. */
> + SVN_ERR(svn_stream_open_writable(&pb.pack_stream, pack_file_path, pool,
> + pool));
> + SVN_ERR(svn_stream_open_writable(&pb.manifest_stream, manifest_file_path,
> + pool, pool));
> + pb.next_offset = 0;
> + pb.cancel_func = cancel_func;
> + pb.cancel_baton = cancel_baton;
> + SVN_ERR(svn_io_dir_walk(shard_path,
> + APR_FINFO_TYPE | APR_FINFO_NAME | APR_FINFO_SIZE,
> + packer_func, &pb, pool));
> + SVN_ERR(svn_stream_close(pb.manifest_stream));
> + SVN_ERR(svn_stream_close(pb.pack_stream));
> +
> + /* Update the max-pack-rev file to reflect our newly packed shard. */
> + final_path = svn_path_join(fs_path, PATH_MAX_PACKED_REV, pool);
> + SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, fs_path,
> + svn_io_file_del_none, pool, pool));
> + SVN_ERR(svn_stream_printf(stream, pool, "%ld\n",
> + (svn_revnum_t) ((shard + 1) * max_files_per_dir)));
> + SVN_ERR(svn_stream_close(stream));
> + SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, final_path, pool));
> +
> + /* Finally, remove the existing shard directory. */
> + return svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton, pool);
> +}
> +
> +
> +svn_error_t *
> +svn_fs_fs__pack(const char *fs_path,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool)
> +{
> + int format, max_files_per_dir;
> + int completed_shards;
> + apr_int64_t i;
> + svn_revnum_t youngest;
> + apr_pool_t *iterpool;
> + const char *data_path;
> + svn_revnum_t max_packed_rev;
> +
> + SVN_ERR(read_format(&format, &max_files_per_dir,
> + svn_path_join(fs_path, PATH_FORMAT, pool),
> + pool));
> +
> + /* If the repository isn't a new enough format, we don't support packing.
> + Return a friendly error to that effect. */
> + if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
> + return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
> + _("FS format too old to pack, please upgrade."));
> +
> + /* If we aren't using sharding, we can't do any packing, so quit. */
> + if (!max_files_per_dir)
> + return SVN_NO_ERROR;
> +
> + SVN_ERR(read_max_packed_rev(&max_packed_rev,
> + svn_path_join(fs_path, PATH_MAX_PACKED_REV, pool),
> + pool));
> +
> + SVN_ERR(get_youngest(&youngest, fs_path, pool));
> + completed_shards = youngest / max_files_per_dir;
> +
> + /* See if we've already completed all possible shards thus far. */
> + if (max_packed_rev == (completed_shards * max_files_per_dir))
> + return SVN_NO_ERROR;
> +
> + data_path = svn_path_join(fs_path, PATH_REVS_DIR, pool);
> +
> + iterpool = svn_pool_create(pool);
> + for (i = max_packed_rev / max_files_per_dir; i < completed_shards; i++)
> + {
> + svn_pool_clear(iterpool);
> +
> + if (cancel_func)
> + SVN_ERR(cancel_func(cancel_baton));
> +
> + SVN_ERR(pack_shard(data_path, fs_path, i, max_files_per_dir, cancel_func,
> + cancel_baton, iterpool));
> + /* We can't pack revprops, because they aren't immutable :(
> + If we ever do get clever and figure out how to pack revprops,
> + this is the place to do it. */
> + }
> +
> + svn_pool_destroy(iterpool);
> + return SVN_NO_ERROR;
> +}
>
> Modified: trunk/subversion/libsvn_fs_fs/fs_fs.h
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs_fs.h?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/fs_fs.h Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/fs_fs.h Mon Dec 1 08:59:39 2008 (r34502)
> @@ -510,4 +510,16 @@ svn_error_t *
> svn_fs_fs__initialize_caches(svn_fs_t *fs, apr_pool_t *pool);
>
>
> +/* Possibly pack the repository at PATH. This just take full shards, and
> + combines all the revision files into a single one, with a manifest header.
> + Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support.
> +
> + Existing filesystem references need not change. */
> +svn_error_t *
> +svn_fs_fs__pack(const char *fs_path,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool);
> +
> +
> #endif
>
> Modified: trunk/subversion/libsvn_fs_fs/structure
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/structure?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_fs_fs/structure Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_fs_fs/structure Mon Dec 1 08:59:39 2008 (r34502)
> @@ -34,6 +34,8 @@ repository) is:
> revs/ Subdirectory containing revs
> <shard>/ Shard directory, if sharding is in use (see below)
> <revnum> File containing rev <revnum>
> + <shard>.pack Pack file, if the repository has been packed (see below)
> + <shard>.manifest Pack manifest file, if a pack file exists (see below)
> revprops/ Subdirectory containing rev-props
> <shard>/ Shard directory, if sharding is in use (see below)
> <revnum> File containing rev-props for <revnum>
> @@ -55,6 +57,7 @@ repository) is:
> uuid File containing the UUID of the repository
> format File containing the format number of this filesystem
> fsfs.conf Configuration file
> + max-packed-rev File containing the youngest revision to be in a pack file
>
> Files in the revprops directory are in the hash dump format used by
> svn_hash_write.
> @@ -116,39 +119,40 @@ The formats are:
> Format 1, understood by Subversion 1.1+
> Format 2, understood by Subversion 1.4+
> Format 3, understood by Subversion 1.5+
> + Format 4, understood by Subversion 1.6+
>
> The differences between the formats are:
>
> Delta representation in revision files
> Format 1: svndiff0 only
> - Formats 2-3: svndiff0 or svndiff1
> + Formats 2-4: svndiff0 or svndiff1
>
> Format options
> Formats 1-2: none permitted
> - Format 3: "layout" option
> + Format 3-4: "layout" option
>
> Transaction name reuse
> Formats 1-2: transaction names may be reused
> - Format 3: transaction names generated using txn-current file
> + Format 3-4: transaction names generated using txn-current file
>
> Location of proto-rev file and its lock
> Formats 1-2: transactions/<txnid>/rev and
> transactions/<txnid>/rev-lock.
> - Format 3: txn-protorevs/<txnid>.rev and
> + Format 3-4: txn-protorevs/<txnid>.rev and
> txn-protorevs/<txnid>.rev-lock.
>
> Node-ID and copy-ID generation
> Formats 1-2: Node-IDs and copy-IDs are guaranteed to form a
> monotonically increasing base36 sequence using the "current"
> file.
> - Format 3: Node-IDs and copy-IDs use the new revision number to
> + Format 3-4: Node-IDs and copy-IDs use the new revision number to
> ensure uniqueness and the "current" file just contains the
> youngest revision.
>
> Mergeinfo metadata:
> Format 1-2: minfo-here and minfo-count node-revision fields are not
> stored. svn_fs_get_mergeinfo returns an error.
> - Format 3: minfo-here and minfo-count node-revision fields are
> + Format 3-4: minfo-here and minfo-count node-revision fields are
> maintained. svn_fs_get_mergeinfo works.
>
>
> @@ -185,6 +189,16 @@ The known layouts, and the parameters th
> revs/0/ directory will contain revisions 0-999, revs/1/ will contain
> 1000-1999, and so on.
>
> +Packing
> +-------
> +
> +A repository can optionally be "packed" to conserve space on disk. The
> +packing process concatenates all the revision files in each full shard to
> +create pack files. A manifest file is also created for each shard which
> +records the indexes of the corresponding revision files in the pack file.
> +In addition, the original shard is removed, and reads are redirected to the
> +pack file.
You might want to define the format of the manifest file.
> +
> Node-revision IDs
> -----------------
>
>
> Modified: trunk/subversion/libsvn_repos/fs-wrap.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_repos/fs-wrap.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/libsvn_repos/fs-wrap.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/libsvn_repos/fs-wrap.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -633,6 +633,15 @@ svn_repos_fs_get_mergeinfo(svn_mergeinfo
> return SVN_NO_ERROR;
> }
>
> +svn_error_t *
> +svn_repos_fs_pack(svn_repos_t *repos,
> + svn_cancel_func_t cancel_func,
> + void *cancel_baton,
> + apr_pool_t *pool)
> +{
> + return svn_fs_pack(repos->db_path, cancel_func, cancel_baton, pool);
> +}
> +
>
>
> /*
>
> Modified: trunk/subversion/svnadmin/main.c
> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/svnadmin/main.c?pathrev=34502&r1=34501&r2=34502
> ==============================================================================
> --- trunk/subversion/svnadmin/main.c Mon Dec 1 08:31:28 2008 (r34501)
> +++ trunk/subversion/svnadmin/main.c Mon Dec 1 08:59:39 2008 (r34502)
> @@ -200,7 +200,8 @@ static svn_opt_subcommand_t
> subcommand_setrevprop,
> subcommand_setuuid,
> subcommand_upgrade,
> - subcommand_verify;
> + subcommand_verify,
> + subcommand_pack;
>
> enum
> {
> @@ -462,6 +463,12 @@ static const svn_opt_subcommand_desc2_t
> "Verifies the data stored in the repository.\n"),
> {'r', 'q'} },
>
> + {"pack", subcommand_pack, {0}, N_
> + ("usage: svnadmin pack REPOS_PATH\n\n"
> + "Possibly compact the repository into a more effecient storage model.\n"
> + "This may not apply to all repositories, in which case, exit.\n"),
> + {0} },
> +
> { NULL, NULL, {0}, NULL, {0} }
> };
>
> @@ -1115,6 +1122,19 @@ subcommand_setlog(apr_getopt_t *os, void
> }
>
>
> +/* This implements 'svn_opt_subcommand_t'. */
> +static svn_error_t *
> +subcommand_pack(apr_getopt_t *os, void *baton, apr_pool_t *pool)
> +{
> + struct svnadmin_opt_state *opt_state = baton;
> + svn_repos_t *repos;
> +
> + SVN_ERR(open_repos(&repos, opt_state->repository_path, pool));
> +
> + return svn_repos_fs_pack(repos, check_cancel, NULL, pool);
> +}
> +
> +
> /* This implements `svn_opt_subcommand_t'. */
> static svn_error_t *
> subcommand_verify(apr_getopt_t *os, void *baton, apr_pool_t *pool)
>
> Copied: trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c (from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c)
> ==============================================================================
> --- /dev/null 00:00:00 1970 (empty, because file is newly added)
> +++ trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c Mon Dec 1 08:59:39 2008 (r34502, copy of r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c)
> @@ -0,0 +1,325 @@
> +/* fs-pack-test.c --- tests for the filesystem
> + *
> + * ====================================================================
> + * Copyright (c) 2008 CollabNet. All rights reserved.
> + *
> + * This software is licensed as described in the file COPYING, which
> + * you should have received as part of this distribution. The terms
> + * are also available at http://subversion.tigris.org/license-1.html.
> + * If newer versions of this license are posted there, you may use a
> + * newer version instead, at your option.
> + *
> + * This software consists of voluntary contributions made by many
> + * individuals. For exact contribution history, see the revision
> + * history and logs, available at http://subversion.tigris.org/.
> + * ====================================================================
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <apr_pools.h>
> +
> +#include "../svn_test.h"
> +#include "../../libsvn_fs_fs/fs.h"
> +
> +#include "svn_pools.h"
> +#include "svn_fs.h"
> +
> +#include "../svn_test_fs.h"
> +
> +
> +/*-----------------------------------------------------------------*/
> +
> +/** The actual fs-tests called by `make check` **/
> +
> +/* Write the format number and maximum number of files per directory
> + to a new format file in PATH, overwriting a previously existing file.
> +
> + Use POOL for temporary allocation.
> +
> + This implementation is largely stolen from libsvn_fs_fs/fs_fs.c. */
> +static svn_error_t *
> +write_format(const char *path,
> + int format,
> + int max_files_per_dir,
> + apr_pool_t *pool)
> +{
> + const char *contents;
> +
> + path = svn_path_join(path, "format", pool);
> +
> + if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT)
> + {
> + if (max_files_per_dir)
> + contents = apr_psprintf(pool,
> + "%d\n"
> + "layout sharded %d\n",
> + format, max_files_per_dir);
> + else
> + contents = apr_psprintf(pool,
> + "%d\n"
> + "layout linear",
> + format);
> + }
> + else
> + {
> + contents = apr_psprintf(pool, "%d\n", format);
> + }
> +
> + {
> + const char *path_tmp;
> +
> + SVN_ERR(svn_io_write_unique(&path_tmp,
> + svn_path_dirname(path, pool),
> + contents, strlen(contents),
> + svn_io_file_del_none, pool));
> +
> +#ifdef WIN32
> + /* make the destination writable, but only on Windows, because
> + Windows does not let us replace read-only files. */
> + SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool));
> +#endif /* WIN32 */
> +
> + /* rename the temp file as the real destination */
> + SVN_ERR(svn_io_file_rename(path_tmp, path, pool));
> + }
> +
> + /* And set the perms to make it read only */
> + return svn_io_set_file_read_only(path, FALSE, pool);
> +}
> +
> +/* Return the expected contents of "iota" in revision REV. */
> +static const char *
> +get_rev_contents(svn_revnum_t rev, apr_pool_t *pool)
> +{
> + /* Toss in a bunch of magic numbers for spice. */
> + apr_int64_t num = ((rev * 1234353 + 4358) * 4583 + ((rev % 4) << 1)) / 42;
> + return apr_psprintf(pool, "%" APR_INT64_T_FMT "\n", num);
> +}
> +
> +/* Create a packed filesystem in DIR. Set the shard size to SHARD_SIZE
> + and create MAX_REV number of revisions. Use POOL for allocations. */
> +static svn_error_t *
> +create_packed_filesystem(const char *dir,
> + svn_test_opts_t *opts,
> + int max_rev,
> + int shard_size,
> + apr_pool_t *pool)
> +{
> + svn_fs_t *fs;
> + svn_fs_txn_t *txn;
> + svn_fs_root_t *txn_root;
> + const char *conflict;
> + svn_revnum_t after_rev;
> + apr_pool_t *subpool = svn_pool_create(pool);
> +
> + /* Create a filesystem, then close it */
> + SVN_ERR(svn_test__create_fs(&fs, dir, opts, subpool));
> + svn_pool_destroy(subpool);
> +
> + subpool = svn_pool_create(pool);
> +
> + /* Rewrite the format file */
> + SVN_ERR(write_format(dir, SVN_FS_FS__MIN_PACKED_FORMAT,
> + shard_size, subpool));
> +
> + /* Reopen the filesystem */
> + SVN_ERR(svn_fs_open(&fs, dir, NULL, subpool));
> +
> + /* Revision 1: the Greek tree */
> + SVN_ERR(svn_fs_begin_txn(&txn, fs, 0, subpool));
> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, subpool));
> + SVN_ERR(svn_test__create_greek_tree(txn_root, subpool));
> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, subpool));
> +
> + /* Revisions 2-11: A bunch of random changes. */
> + while (after_rev < max_rev + 1)
> + {
> + SVN_ERR(svn_fs_begin_txn(&txn, fs, after_rev, subpool));
> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, subpool));
> + SVN_ERR(svn_test__set_file_contents(txn_root, "iota",
> + get_rev_contents(after_rev + 1,
> + subpool),
> + subpool));
> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, subpool));
> + }
> + svn_pool_destroy(subpool);
> +
> + /* Now pack the FS */
> + return svn_fs_pack(dir, NULL, NULL, pool);
> +}
> +
> +/* Pack a filesystem. */
> +#define REPO_NAME "test-repo-fsfs-pack"
I'm not sure why these all have to be #defines rather than const char
*, const int, etc.
> +#define SHARD_SIZE 7
> +#define MAX_REV 53
> +static svn_error_t *
> +pack_filesystem(const char **msg,
> + svn_boolean_t msg_only,
> + svn_test_opts_t *opts,
> + apr_pool_t *pool)
> +{
> + int i;
> + svn_node_kind_t kind;
> + const char *path;
> + char buf[80];
> + apr_file_t *file;
> + apr_size_t len;
> +
> + *msg = "pack a FSFS filesystem";
> +
> + if (msg_only)
> + return SVN_NO_ERROR;
> +
> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, MAX_REV, SHARD_SIZE,
> + pool));
> +
> + /* Check to see that the pack files exist, and that the rev directories
> + don't. */
> + for (i = 0; i < (MAX_REV + 1) / SHARD_SIZE; i++)
> + {
> + path = svn_path_join_many(pool, REPO_NAME, "revs",
> + apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), NULL);
> +
> + /* These files should exist. */
> + SVN_ERR(svn_io_check_path(path, &kind, pool));
> + if (kind != svn_node_file)
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Expected pack file '%s' not found", path);
> +
> + path = svn_path_join_many(pool, REPO_NAME, "revs",
> + apr_psprintf(pool, "%d.manifest", i / SHARD_SIZE), NULL);
> + SVN_ERR(svn_io_check_path(path, &kind, pool));
> + if (kind != svn_node_file)
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Expected manifest file '%s' not found",
> + path);
> +
> + /* This directory should not exist. */
> + path = svn_path_join_many(pool, REPO_NAME, "revs",
> + apr_psprintf(pool, "%d", i / SHARD_SIZE), NULL);
> + SVN_ERR(svn_io_check_path(path, &kind, pool));
> + if (kind != svn_node_none)
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Unexpected directory '%s' found", path);
> + }
> +
> + /* Ensure the max-packed-rev jives with the above operations. */
> + SVN_ERR(svn_io_file_open(&file,
> + svn_path_join(REPO_NAME, "max-packed-rev", pool),
> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
> + len = sizeof(buf);
> + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
> + SVN_ERR(svn_io_file_close(file, pool));
> + if (SVN_STR_TO_REV(buf) != (MAX_REV / SHARD_SIZE) * SHARD_SIZE)
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Bad 'max-packed-rev' contents");
> +
> + /* Finally, make sure the final revision directory does exist. */
> + path = svn_path_join_many(pool, REPO_NAME, "revs",
> + apr_psprintf(pool, "%d", (i / SHARD_SIZE) + 1), NULL);
> + SVN_ERR(svn_io_check_path(path, &kind, pool));
> + if (kind != svn_node_none)
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Expected directory '%s' not found",
> + path);
> +
> +
> + return SVN_NO_ERROR;
> +}
> +#undef REPO_NAME
> +#undef SHARD_SIZE
> +#undef MAX_REV
> +
> +
> +/* Check reading from a packed filesystem. */
> +#define REPO_NAME "test-repo-read-packed-fs"
> +static svn_error_t *
> +read_packed_fs(const char **msg,
> + svn_boolean_t msg_only,
> + svn_test_opts_t *opts,
> + apr_pool_t *pool)
> +{
> + svn_fs_t *fs;
> + svn_stream_t *rstream;
> + svn_stringbuf_t *rstring;
> + svn_revnum_t i;
> +
> + *msg = "read from a packed FSFS filesystem";
> +
> + if (msg_only)
> + return SVN_NO_ERROR;
> +
> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, 11, 5, pool));
> + SVN_ERR(svn_fs_open(&fs, REPO_NAME, NULL, pool));
> +
> + for (i = 1; i < 12; i++)
> + {
> + svn_fs_root_t *rev_root;
> + svn_stringbuf_t *sb;
> +
> + SVN_ERR(svn_fs_revision_root(&rev_root, fs, i, pool));
> + SVN_ERR(svn_fs_file_contents(&rstream, rev_root, "iota", pool));
> + SVN_ERR(svn_test__stream_to_string(&rstring, rstream, pool));
> +
> + if (i == 1)
> + sb = svn_stringbuf_create("This is the file 'iota'.\n", pool);
> + else
> + sb = svn_stringbuf_create(get_rev_contents(i, pool), pool);
> +
> + if (! svn_stringbuf_compare(rstring, sb))
> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
> + "Bad data in revision %ld.", i);
> + }
> +
> + return SVN_NO_ERROR;
> +}
> +#undef REPO_NAME
> +
> +/* Check reading from a packed filesystem. */
> +#define REPO_NAME "test-repo-commit-packed-fs"
> +static svn_error_t *
> +commit_packed_fs(const char **msg,
> + svn_boolean_t msg_only,
> + svn_test_opts_t *opts,
> + apr_pool_t *pool)
> +{
> + svn_fs_t *fs;
> + svn_fs_txn_t *txn;
> + svn_fs_root_t *txn_root;
> + const char *conflict;
> + svn_revnum_t after_rev;
> +
> + *msg = "commit to a packed FSFS filesystem";
> +
> + if (msg_only)
> + return SVN_NO_ERROR;
> +
> + /* Create the packed FS and open it. */
> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, 11, 5, pool));
> + SVN_ERR(svn_fs_open(&fs, REPO_NAME, NULL, pool));
> +
> + /* Now do a commit. */
> + SVN_ERR(svn_fs_begin_txn(&txn, fs, 12, pool));
> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, pool));
> + SVN_ERR(svn_test__set_file_contents(txn_root, "iota",
> + "How much better is it to get wisdom than gold! and to get "
> + "understanding rather to be chosen than silver!", pool));
> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, pool));
> +
> + return SVN_NO_ERROR;
> +}
> +#undef REPO_NAME
> +
> +/* ------------------------------------------------------------------------ */
> +
> +/* The test table. */
> +
> +struct svn_test_descriptor_t test_funcs[] =
> + {
> + SVN_TEST_NULL,
> + SVN_TEST_PASS(pack_filesystem),
> + SVN_TEST_PASS(read_packed_fs),
> + SVN_TEST_PASS(commit_packed_fs),
> + SVN_TEST_NULL
> + };
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: svn-unsubscribe_at_subversion.tigris.org
> For additional commands, e-mail: svn-help_at_subversion.tigris.org
>
>
--dave
--
David Glasser | glasser@davidglasser.net | http://www.davidglasser.net/
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe_at_subversion.tigris.org
For additional commands, e-mail: dev-help_at_subversion.tigris.org
Received on 2008-12-01 19:11:32 CET