For efficiency reasons, the system has converted the large body of this message into an attachment.
------------------------------------------------------
http://subversion.tigris.org/ds/viewMessage.do?dsForumId=462&dsMessageId=912491
attached mail follows:
I've addressed the following issues in r34510, r34517, r34523, r34530, and r34537.
-Hyrum
David Glasser wrote:
> On Mon, Dec 1, 2008 at 8:59 AM, <hwright_at_tigris.org> wrote:
>> Author: hwright
>> Date: Mon Dec 1 08:59:39 2008
>> New Revision: 34502
>>
>> Log:
>> Merge the fsfs-pack branch to trunk.
>>
>> For a complete log, please see log messages on the branch.
>>
>> Added:
>> trunk/subversion/tests/libsvn_fs_fs/ (props changed)
>> - copied from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/
>> Replaced:
>> trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c
>> - copied unchanged from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c
>> Modified:
>> trunk/ (props changed)
>> trunk/build.conf
>> trunk/subversion/include/svn_fs.h
>> trunk/subversion/include/svn_repos.h
>> trunk/subversion/libsvn_fs/fs-loader.c
>> trunk/subversion/libsvn_fs/fs-loader.h
>> trunk/subversion/libsvn_fs_base/fs.c
>> trunk/subversion/libsvn_fs_fs/caching.c
>> trunk/subversion/libsvn_fs_fs/fs.c
>> trunk/subversion/libsvn_fs_fs/fs.h
>> trunk/subversion/libsvn_fs_fs/fs_fs.c
>> trunk/subversion/libsvn_fs_fs/fs_fs.h
>> trunk/subversion/libsvn_fs_fs/structure
>> trunk/subversion/libsvn_repos/fs-wrap.c
>> trunk/subversion/svnadmin/main.c
>>
>> Merged:
>> /branches/fsfs-pack:r33643-34501
>>
>> Modified: trunk/build.conf
>> URL: http://svn.collab.net/viewvc/svn/trunk/build.conf?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/build.conf Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/build.conf Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -3,7 +3,7 @@
>> #
>> ######################################################################
>> #
>> -# Copyright (c) 2000-2006 CollabNet. All rights reserved.
>> +# Copyright (c) 2000-2008 CollabNet. All rights reserved.
>> #
>> # This software is licensed as described in the file COPYING, which
>> # you should have received as part of this distribution. The terms
>> @@ -611,6 +611,17 @@ libs = libsvn_test libsvn_fs libsvn_fs_b
>> libsvn_subr apriconv apr
>>
>> # ----------------------------------------------------------------------------
>> +# Tests for libsvn_fs_fs
>> +[fs-pack-test]
>> +description = Test fsfs packing in libsvn_fs_fs
>> +type = exe
>> +path = subversion/tests/libsvn_fs_fs
>> +sources = fs-pack-test.c
>> +install = test
>> +libs = libsvn_test libsvn_fs libsvn_fs_fs libsvn_delta
>> + libsvn_subr apriconv apr
>> +
>> +# ----------------------------------------------------------------------------
>> # Tests for libsvn_fs
>>
>> [locks-test]
>> @@ -963,7 +974,7 @@ libs = svn svnserve svnadmin svnlook svn
>> type = project
>> path = build/win32
>> libs = __ALL__
>> - fs-test fs-base-test skel-test key-test strings-reps-test changes-test locks-test
>> + fs-test fs-base-test fs-fsfs-test skel-test key-test strings-reps-test changes-test locks-test
>> repos-test
>> checksum-test compat-test config-test hashdump-test mergeinfo-test opt-test path-test stream-test
>> string-test time-test utf-test target-test error-test cache-test
>>
>> Modified: trunk/subversion/include/svn_fs.h
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/include/svn_fs.h?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/include/svn_fs.h Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/include/svn_fs.h Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -2024,6 +2024,20 @@ svn_error_t *
>> svn_fs_print_modules(svn_stringbuf_t *output,
>> apr_pool_t *pool);
>>
>> +
>> +/**
>> + * Possibly update the filesystem located in the directory @a path
>> + * to use disk space more efficiently.
>> + *
>> + * @since New in 1.6.
>> + */
>> +svn_error_t *
>> +svn_fs_pack(const char *db_path,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool);
>> +
>> +
>> /** @} */
>>
>> #ifdef __cplusplus
>>
>> Modified: trunk/subversion/include/svn_repos.h
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/include/svn_repos.h?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/include/svn_repos.h Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/include/svn_repos.h Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -306,6 +306,20 @@ svn_repos_hotcopy(const char *src_path,
>> svn_boolean_t clean_logs,
>> apr_pool_t *pool);
>>
>> +
>> +/**
>> + * Possibly update the repository, @a repos, to use a more efficient
>> + * filesystem representation. Use @a pool for allocations.
>> + *
>> + * @since New in 1.6.
>> + */
>> +svn_error_t *
>> +svn_repos_fs_pack(svn_repos_t *repos,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool);
>> +
>> +
>> /**
>> * Run database recovery procedures on the repository at @a path,
>> * returning the database to a consistent state. Use @a pool for all
>>
>> Modified: trunk/subversion/libsvn_fs/fs-loader.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs/fs-loader.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs/fs-loader.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs/fs-loader.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -488,6 +488,18 @@ svn_fs_hotcopy(const char *src_path, con
>> }
>>
>> svn_error_t *
>> +svn_fs_pack(const char *path,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool)
>> +{
>> + fs_library_vtable_t *vtable;
>> +
>> + SVN_ERR(fs_library_vtable(&vtable, path, pool));
>> + return vtable->pack(path, cancel_func, cancel_baton, pool);
>> +}
>> +
>> +svn_error_t *
>> svn_fs_recover(const char *path,
>> svn_cancel_func_t cancel_func, void *cancel_baton,
>> apr_pool_t *pool)
>>
>> Modified: trunk/subversion/libsvn_fs/fs-loader.h
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs/fs-loader.h?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs/fs-loader.h Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs/fs-loader.h Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -88,6 +88,8 @@ typedef struct fs_library_vtable_t
>> svn_error_t *(*recover)(svn_fs_t *fs,
>> svn_cancel_func_t cancel_func, void *cancel_baton,
>> apr_pool_t *pool);
>> + svn_error_t *(*pack)(const char *path, svn_cancel_func_t cancel_func,
>> + void *cancel_baton, apr_pool_t *pool);
>>
>> /* Provider-specific functions should go here, even if they could go
>> in an object vtable, so that they are all kept together. */
>>
>> Modified: trunk/subversion/libsvn_fs_base/fs.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_base/fs.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_base/fs.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_base/fs.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -877,6 +877,16 @@ base_bdb_recover(svn_fs_t *fs,
>> return bdb_recover(fs->path, FALSE, pool);
>> }
>>
>> +static svn_error_t *
>> +base_bdb_pack(const char *path,
>> + svn_cancel_func_t cancel,
>> + void *cancel_baton,
>> + apr_pool_t *pool)
>> +{
>> + /* Packing is currently a no op for BDB. */
>> + return SVN_NO_ERROR;
>> +}
>> +
>>
>>
>> /* Running the 'archive' command on a Berkeley DB-based filesystem. */
>> @@ -1327,6 +1337,7 @@ static fs_library_vtable_t library_vtabl
>> base_hotcopy,
>> base_get_description,
>> base_bdb_recover,
>> + base_bdb_pack,
>> base_bdb_logfiles,
>> svn_fs_base__id_parse
>> };
>>
>> Modified: trunk/subversion/libsvn_fs_fs/caching.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/caching.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/caching.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/caching.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -104,6 +104,48 @@ dup_dir_listing(void **out,
>> return SVN_NO_ERROR;
>> }
>>
>> +
>> +/** Caching packed rev offsets. **/
>> +/* Implements svn_cache__serialize_func_t */
>> +static svn_error_t *
>> +offset_serialize(char **data,
>> + apr_size_t *data_len,
>> + void *in,
>> + apr_pool_t *pool)
>> +{
>> + *data = apr_off_t_toa(pool, *((apr_off_t *) in));
>> + *data_len = strlen(*data);
>> + return SVN_NO_ERROR;
>> +}
>> +
>> +/* Implements svn_cache__deserialize_func_t */
>> +static svn_error_t *
>> +offset_deserialize(void **out,
>> + const char *data,
>> + apr_size_t data_len,
>> + apr_pool_t *pool)
>> +{
>> + *out = apr_palloc(pool, sizeof (apr_off_t));
>> + apr_strtoff((apr_off_t *)*out, data, NULL, 0);
>> +
>> + return SVN_NO_ERROR;
>> +}
>> +
>> +/* Implements svn_cache__dup_func_t */
>> +static svn_error_t *
>> +dup_pack_offset(void **out,
>> + void *in,
>> + apr_pool_t *pool)
>> +{
>> + apr_off_t *offset = apr_palloc(pool, sizeof(*offset));
>> +
>> + *offset = *((apr_off_t *) in);
>> +
>> + *out = offset;
>> + return SVN_NO_ERROR;
>> +}
>> +
>> +
>> /* Return a memcache in *MEMCACHE_P for FS if it's configured to use
>> memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean
>> indicating whether cache errors should be returned to the caller or
>> @@ -218,6 +260,25 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
>> SVN_ERR(svn_cache__set_error_handler(ffd->dir_cache,
>> warn_on_cache_errors, fs, pool));
>>
>> + /* Only 16 bytes per entry (a revision number + the corresponding offset). */
>> + if (memcache)
>> + SVN_ERR(svn_cache__create_memcache(&(ffd->packed_offset_cache),
>> + memcache,
>> + offset_serialize,
>> + offset_deserialize,
>> + sizeof(svn_revnum_t),
>> + apr_pstrcat(pool, prefix, "PACK-OFFSET",
>> + NULL),
>> + fs->pool));
>> + else
>> + SVN_ERR(svn_cache__create_inprocess(&(ffd->packed_offset_cache),
>> + dup_pack_offset, sizeof(svn_revnum_t),
>> + 16, 1024, FALSE, fs->pool));
>
> It doesn't matter too much, but the idea behind the size estimation is
> to try to get each page to be about 8K (ie, the minimum pool
> allocation), so you probably want like 500 per page here.
>
>> +
>> + if (! no_handler)
>> + SVN_ERR(svn_cache__set_error_handler(ffd->packed_offset_cache,
>> + warn_on_cache_errors, fs, pool));
>> +
>> if (memcache)
>> {
>> SVN_ERR(svn_cache__create_memcache(&(ffd->fulltext_cache),
>>
>> Modified: trunk/subversion/libsvn_fs_fs/fs.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/fs.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/fs.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -321,6 +321,7 @@ static fs_library_vtable_t library_vtabl
>> fs_hotcopy,
>> fs_get_description,
>> svn_fs_fs__recover,
>> + svn_fs_fs__pack,
>> fs_logfiles
>> };
>>
>>
>> Modified: trunk/subversion/libsvn_fs_fs/fs.h
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs.h?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/fs.h Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/fs.h Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -52,6 +52,8 @@ extern "C" {
>> #define PATH_TXN_CURRENT "txn-current" /* File with next txn key */
>> #define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */
>> #define PATH_LOCKS_DIR "locks" /* Directory of locks */
>> +#define PATH_MAX_PACKED_REV "max-packed-rev" /* Youngest revision which
>> + has been packed. */
>> /* If you change this, look at tests/svn_test_fs.c(maybe_install_fsfs_conf) */
>> #define PATH_CONFIG "fsfs.conf" /* Configuration */
>>
>> @@ -104,6 +106,9 @@ extern "C" {
>> /* The minimum format number that allows rep sharing. */
>> #define SVN_FS_FS__MIN_REP_SHARING_FORMAT 4
>>
>> +/* The minimum format number that supports packed shards. */
>> +#define SVN_FS_FS__MIN_PACKED_FORMAT 4
>> +
>> /* Private FSFS-specific data shared between all svn_txn_t objects that
>> relate to a particular transaction in a filesystem (as identified
>> by transaction id and filesystem UUID). Objects of this type are
>> @@ -218,11 +223,18 @@ typedef struct
>> rep key to svn_string_t. */
>> svn_cache__t *fulltext_cache;
>>
>> + /* Pack manifest cache; maps revision numbers to offsets in their respective
>> + pack files. */
>> + svn_cache__t *packed_offset_cache;
>> +
>> /* Data shared between all svn_fs_t objects for a given filesystem. */
>> fs_fs_shared_data_t *shared;
>>
>> /* The sqlite database used for rep caching. */
>> struct rep_cache_t rep_cache;
>> +
>> + /* The youngest revision in a pack file. */
>> + svn_revnum_t max_packed_rev;
>> } fs_fs_data_t;
>>
>>
>>
>> Modified: trunk/subversion/libsvn_fs_fs/fs_fs.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs_fs.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -103,6 +103,10 @@
>> #define REP_PLAIN "PLAIN"
>> #define REP_DELTA "DELTA"
>>
>> +/* The size of an entry in the pack file manifest, not including terminating
>> + characters. */
>> +#define PACK_MANIFEST_ENTRY_LEN 19
>> +
>> /* Notes:
>>
>> To avoid opening and closing the rev-files all the time, it would
>> @@ -165,6 +169,19 @@ path_lock(svn_fs_t *fs, apr_pool_t *pool
>> }
>>
>> static const char *
>> +path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind,
>> + apr_pool_t *pool)
>> +{
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> +
>> + assert(ffd->max_files_per_dir);
>> + return svn_path_join_many(pool, fs->path, PATH_REVS_DIR,
>> + apr_psprintf(pool, "%ld.%s",
>> + rev / ffd->max_files_per_dir, kind),
>> + NULL);
>> +}
>> +
>> +static const char *
>> path_rev_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
>> {
>> fs_fs_data_t *ffd = fs->fsap_data;
>> @@ -247,6 +264,12 @@ path_txn_next_ids(svn_fs_t *fs, const ch
>> }
>>
>> static APR_INLINE const char *
>> +path_max_packed_rev(svn_fs_t *fs, apr_pool_t *pool)
>> +{
>> + return svn_path_join(fs->path, PATH_MAX_PACKED_REV, pool);
>> +}
>> +
>> +static APR_INLINE const char *
>> path_txn_proto_rev(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
>> {
>> fs_fs_data_t *ffd = fs->fsap_data;
>> @@ -1020,6 +1043,25 @@ write_config(svn_fs_t *fs,
>> }
>>
>> static svn_error_t *
>> +read_max_packed_rev(svn_revnum_t *max_packed_rev,
>> + const char *path,
>> + apr_pool_t *pool)
>> +{
>> + char buf[80];
>> + apr_file_t *file;
>> + apr_size_t len;
>> +
>> + SVN_ERR(svn_io_file_open(&file, path, APR_READ | APR_BUFFERED,
>> + APR_OS_DEFAULT, pool));
>> + len = sizeof(buf);
>> + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
>> + SVN_ERR(svn_io_file_close(file, pool));
>> +
>> + *max_packed_rev = SVN_STR_TO_REV(buf);
>> + return SVN_NO_ERROR;
>> +}
>> +
>> +static svn_error_t *
>> get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool);
>>
>> svn_error_t *
>> @@ -1053,6 +1095,11 @@ svn_fs_fs__open(svn_fs_t *fs, const char
>>
>> SVN_ERR(svn_io_file_close(uuid_file, pool));
>>
>> + /* Read the max packed revision. */
>> + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
>> + SVN_ERR(read_max_packed_rev(&ffd->max_packed_rev,
>> + path_max_packed_rev(fs, pool), pool));
>> +
>> /* Read the configuration file. */
>> SVN_ERR(svn_config_read(&ffd->config,
>> svn_path_join(fs->path, PATH_CONFIG, pool),
>> @@ -1118,6 +1165,10 @@ upgrade_body(void *baton, apr_pool_t *po
>> (svn_path_join(fs->path, PATH_TXN_PROTOS_DIR, pool), pool));
>> }
>>
>> + /* If our filesystem is new enough, write the max packed rev file. */
>> + if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
>> + SVN_ERR(svn_io_file_create(path_max_packed_rev(fs, pool), "0\n", pool));
>> +
>> /* Bump the format file. */
>> return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir,
>> TRUE, pool);
>> @@ -1294,6 +1345,9 @@ svn_fs_fs__hotcopy(const char *src_path,
>> /* Copy the uuid. */
>> SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_UUID, pool));
>>
>> + /* Copy the max packed rev. */
>> + SVN_ERR(svn_io_dir_file_copy(src_path, dst_path, PATH_MAX_PACKED_REV, pool));
>> +
>
> Doesn't this need to be conditional on it existing, or does
> svn_io_dir_file_copy ignore file-not-found errors?
>
>> /* Find the youngest revision from this current file. */
>> SVN_ERR(get_youngest(&youngest, dst_path, pool));
>>
>> @@ -1514,6 +1568,80 @@ ensure_revision_exists(svn_fs_t *fs,
>> _("No such revision %ld"), rev);
>> }
>>
>> +/* Open the correct revision file for REV. If the filesystem FS has
>> + been packed, *FILE will be set to the packed file; otherwise, set *FILE
>> + to the revision file for REV. Use POOL for allocations. */
>> +static svn_error_t *
>> +open_pack_or_rev_file(apr_file_t **file,
>> + svn_fs_t *fs,
>> + svn_revnum_t rev,
>> + apr_pool_t *pool)
>> +{
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> + svn_error_t *err;
>> +
>> + err = svn_io_file_open(file,
>> + rev < ffd->max_packed_rev
>
> If it is the max *packed* rev, then shouldn't this be <=? (This falls
> into the category of "I'm not sure how it worked with this code", if
> I'm correct...)
>
>> + ? path_rev_packed(fs, rev, "pack", pool)
>> + : svn_fs_fs__path_rev(fs, rev, pool),
>> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
>> +
>> + if (err && APR_STATUS_IS_ENOENT(err->apr_err))
>> + {
>> + svn_error_clear(err);
>> + return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
>> + _("No such revision %ld"), rev);
>> + }
>> +
>> + return err;
>> +}
>> +
>> +/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
>> + Use POOL for temporary allocations. */
>> +static svn_error_t *
>> +get_packed_offset(apr_off_t *rev_offset,
>> + svn_fs_t *fs,
>> + svn_revnum_t rev,
>> + apr_pool_t *pool)
>> +{
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> + apr_file_t *manifest_file;
>> + char buf[PACK_MANIFEST_ENTRY_LEN + 1];
>> + apr_off_t rev_index_offset;
>> + svn_boolean_t is_cached;
>> + apr_off_t *cached_rev_offset;
>> + apr_size_t len;
>> +
>> + SVN_ERR(svn_cache__get((void **) &cached_rev_offset, &is_cached,
>> + ffd->packed_offset_cache, &rev, pool));
>> +
>> + if (is_cached)
>> + {
>> + *rev_offset = *cached_rev_offset;
>> + return SVN_NO_ERROR;
>> + }
>> +
>> + /* Open the manifest file. */
>> + SVN_ERR(svn_io_file_open(&manifest_file, path_rev_packed(fs, rev, "manifest",
>> + pool),
>> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
>> +
>> + /* Seek to the correct offset. */
>
> Hmm. These files aren't actually that big, right? I would just read
> the whole manifest into the cache instead of seeking and reading one
> entry (which is likely to be the same amount of IO). Just make sure
> that the cache is a reasonable size...
>
>> + rev_index_offset = (rev % ffd->max_files_per_dir) *
>> + (PACK_MANIFEST_ENTRY_LEN + 1);
>> + SVN_ERR(svn_io_file_seek(manifest_file, APR_SET, &rev_index_offset, pool));
>> +
>> + /* Read the revision offset and close the file. */
>> + len = PACK_MANIFEST_ENTRY_LEN;
>> + SVN_ERR(svn_io_file_read(manifest_file, buf, &len, pool));
>
> Is svn_io_file_read guaranteed to read the entire length, or do you
> need to check len? (Yeah, it's short enough that a partial read seems
> unlikely.)
>
>> + buf[PACK_MANIFEST_ENTRY_LEN] = 0;
>> + SVN_ERR(svn_io_file_close(manifest_file, pool));
>> +
>> + *rev_offset = apr_atoi64(buf);
>> +
>> + return svn_cache__set(ffd->packed_offset_cache, &rev, rev_offset, pool);
>> +}
>> +
>> /* Open the revision file for revision REV in filesystem FS and store
>> the newly opened file in FILE. Seek to location OFFSET before
>> returning. Perform temporary allocations in POOL. */
>> @@ -1524,12 +1652,20 @@ open_and_seek_revision(apr_file_t **file
>> apr_off_t offset,
>> apr_pool_t *pool)
>> {
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> apr_file_t *rev_file;
>>
>> SVN_ERR(ensure_revision_exists(fs, rev, pool));
>>
>> - SVN_ERR(svn_io_file_open(&rev_file, svn_fs_fs__path_rev(fs, rev, pool),
>> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
>> + SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, pool));
>> +
>> + if (rev < ffd->max_packed_rev)
>
> ditto.
>
>> + {
>> + apr_off_t rev_offset;
>> +
>> + SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
>> + offset += rev_offset;
>> + }
>>
>> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
>>
>> @@ -2072,19 +2208,30 @@ read_rep_line(struct rep_args **rep_args
>> _("Malformed representation header"));
>> }
>>
>> -/* Given a revision file REV_FILE, find the Node-ID of the header
>> - located at OFFSET and store it in *ID_P. Allocate temporary
>> - variables from POOL. */
>> +/* Given a revision file REV_FILE, opened to REV in FS, find the Node-ID
>> + of the header located at OFFSET and store it in *ID_P. Allocate
>> + temporary variables from POOL. */
>> static svn_error_t *
>> get_fs_id_at_offset(svn_fs_id_t **id_p,
>> apr_file_t *rev_file,
>> + svn_fs_t *fs,
>> + svn_revnum_t rev,
>> apr_off_t offset,
>> apr_pool_t *pool)
>> {
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> svn_fs_id_t *id;
>> apr_hash_t *headers;
>> const char *node_id_str;
>>
>> + if (rev < ffd->max_packed_rev)
>
> ditto.
>
>> + {
>> + apr_off_t rev_offset;
>> +
>> + SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
>> + offset += rev_offset;
>> + }
>> +
>> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
>>
>> SVN_ERR(read_header_block(&headers,
>> @@ -2109,27 +2256,50 @@ get_fs_id_at_offset(svn_fs_id_t **id_p,
>> }
>>
>>
>> -/* Given an open revision file REV_FILE, locate the trailer that
>> +/* Given an open revision file REV_FILE in FS for REV, locate the trailer that
>> specifies the offset to the root node-id and to the changed path
>> information. Store the root node offset in *ROOT_OFFSET and the
>> changed path offset in *CHANGES_OFFSET. If either of these
>> - pointers is NULL, do nothing with it. Allocate temporary variables
>> - from POOL. */
>> + pointers is NULL, do nothing with it. If PACKED is true, REV_FILE
>> + should be a packed shard file. Allocate temporary variables from POOL. */
>> static svn_error_t *
>> get_root_changes_offset(apr_off_t *root_offset,
>> apr_off_t *changes_offset,
>> apr_file_t *rev_file,
>> + svn_fs_t *fs,
>> + svn_revnum_t rev,
>> apr_pool_t *pool)
>> {
>> + fs_fs_data_t *ffd = fs->fsap_data;
>> apr_off_t offset;
>> char buf[64];
>> int i, num_bytes;
>> apr_size_t len;
>> + apr_seek_where_t seek_relative;
>> +
>> + /* Determine where to seek to in the file.
>> +
>> + If we've got a pack file, we want to seek to the end of the desired
>> + revision. But we don't track that, so we seek to the beginning of the
>> + next revision.
>> +
>> + Unless the next revision is in a different file, in which case, we can
>> + just seek to the end of the pack file -- just like we do in the
>> + non-packed case. */
>> + if ((rev < ffd->max_packed_rev) && ((rev + 1) % ffd->max_files_per_dir != 0))
>
> ditto.
>
>> + {
>> + SVN_ERR(get_packed_offset(&offset, fs, rev + 1, pool));
>> + seek_relative = APR_SET;
>> + }
>> + else
>> + {
>> + seek_relative = APR_END;
>> + offset = 0;
>> + }
>>
>> /* We will assume that the last line containing the two offsets
>> will never be longer than 64 characters. */
>> - offset = 0;
>> - SVN_ERR(svn_io_file_seek(rev_file, APR_END, &offset, pool));
>> + SVN_ERR(svn_io_file_seek(rev_file, seek_relative, &offset, pool));
>>
>> offset -= sizeof(buf);
>> SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
>> @@ -2195,7 +2365,6 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **ro
>> apr_file_t *revision_file;
>> apr_off_t root_offset;
>> svn_fs_id_t *root_id;
>> - svn_error_t *err;
>> svn_boolean_t is_cached;
>>
>> SVN_ERR(ensure_revision_exists(fs, rev, pool));
>> @@ -2205,21 +2374,12 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **ro
>> if (is_cached)
>> return SVN_NO_ERROR;
>>
>> - err = svn_io_file_open(&revision_file, svn_fs_fs__path_rev(fs, rev, pool),
>> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
>> - if (err && APR_STATUS_IS_ENOENT(err->apr_err))
>> - {
>> - svn_error_clear(err);
>> - return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
>> - _("No such revision %ld"), rev);
>> - }
>> - else if (err)
>> - return err;
>> -
>> -
>> - SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, pool));
>> + SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
>> + SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, fs, rev,
>> + pool));
>>
>> - SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, root_offset, pool));
>> + SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, fs, rev,
>> + root_offset, pool));
>>
>> SVN_ERR(svn_io_file_close(revision_file, pool));
>>
>> @@ -3734,11 +3894,10 @@ svn_fs_fs__paths_changed(apr_hash_t **ch
>>
>> SVN_ERR(ensure_revision_exists(fs, rev, pool));
>>
>> - SVN_ERR(svn_io_file_open(&revision_file, svn_fs_fs__path_rev(fs, rev, pool),
>> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
>> + SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
>>
>> - SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file,
>> - pool));
>> + SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file, fs,
>> + rev, pool));
>>
>> SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool));
>>
>> @@ -5651,6 +5810,10 @@ svn_fs_fs__create(svn_fs_t *fs,
>> && rep_sharing_allowed)
>> SVN_ERR(svn_fs_fs__open_rep_cache(fs, fs->pool));
>>
>> + /* Create the max packed rev file. */
>> + if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
>> + SVN_ERR(svn_io_file_create(path_max_packed_rev(fs, pool), "0\n", pool));
>> +
>> /* Create the txn-current file if the repository supports
>> the transaction sequence file. */
>> if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
>> @@ -5986,11 +6149,8 @@ recover_body(void *baton, apr_pool_t *po
>> if (b->cancel_func)
>> SVN_ERR(b->cancel_func(b->cancel_baton));
>>
>> - SVN_ERR(svn_io_file_open(&rev_file,
>> - svn_fs_fs__path_rev(fs, rev, iterpool),
>> - APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
>> - iterpool));
>> - SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file,
>> + SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, iterpool));
>> + SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, fs, rev,
>> iterpool));
>> SVN_ERR(recover_find_max_ids(fs, rev, rev_file, root_offset,
>> max_node_id, max_copy_id, iterpool));
>> @@ -6504,3 +6664,170 @@ svn_fs_fs__begin_txn(svn_fs_txn_t **txn_
>>
>> return svn_fs_fs__change_txn_props(*txn_p, props, pool);
>> }
>> +
>> +
>> +/****** Packing FSFS shards *********/
>> +struct packer_baton
>> +{
>> + apr_off_t next_offset;
>> + svn_stream_t *pack_stream;
>> + svn_stream_t *manifest_stream;
>> + svn_cancel_func_t cancel_func;
>> + void *cancel_baton;
>> +};
>> +
>> +/* Walker function used by pack_shard().
>> + This implements svn_io_walk_func_t */
>> +static svn_error_t *
>> +packer_func(void *baton,
>> + const char *path,
>> + const apr_finfo_t *finfo,
>> + apr_pool_t *pool)
>> +{
>> + struct packer_baton *pb = baton;
>> + svn_stream_t *rev_stream;
>> +
>> + /* Ignore any directories we find. */
>> + if (finfo->filetype == APR_DIR)
>> + return SVN_NO_ERROR;
>> +
>> + /* Update the manifest. */
>> + svn_stream_printf(pb->manifest_stream, pool,
>> + "%-" APR_STRINGIFY(PACK_MANIFEST_ENTRY_LEN) APR_OFF_T_FMT"\n",
>> + pb->next_offset);
>> + pb->next_offset += finfo->size;
>> +
>> + /* Copy all the bits from the rev file to the end of the pack file. */
>> + SVN_ERR(svn_stream_open_readonly(&rev_stream, path, pool, pool));
>> + return svn_stream_copy3(rev_stream, svn_stream_disown(pb->pack_stream, pool),
>> + pb->cancel_func, pb->cancel_baton, pool);
>> +}
>> +
>> +/* Pack a single shard SHARD in REVS_DIR, using POOL for allocations.
>> + CANCEL_FUNC and CANCEL_BATON are what you think they are.
>> +
>> + If for some reason we detect a partial packing already performed, we
>> + remove the pack file and start again. */
>> +static svn_error_t *
>> +pack_shard(const char *revs_dir,
>> + const char *fs_path,
>> + apr_int64_t shard,
>> + int max_files_per_dir,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool)
>> +{
>> + svn_stream_t *stream;
>> + const char *tmp_path;
>> + const char *final_path;
>> + svn_error_t *err;
>> + const char *pack_file_path = svn_path_join(
>> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT ".pack",
>> + shard), pool);
>> + const char *manifest_file_path = svn_path_join(
>> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT ".manifest",
>> + shard), pool);
>> + const char *shard_path = svn_path_join(
>> + revs_dir, apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
>> + pool);
>> + struct packer_baton pb;
>> +
>> + /* Remove any existing pack file for this shard, since it is incomplete. */
>> + err = svn_io_remove_file(pack_file_path, pool);
>> + if (err)
>> + {
>> + if (APR_STATUS_IS_ENOENT(err->apr_err))
>> + svn_error_clear(err);
>> + else
>> + return err;
>> + }
>> + else
>> + SVN_ERR(svn_io_remove_file(manifest_file_path, pool));
>> +
>> + /* Create the new pack and manifest files. */
>> + SVN_ERR(svn_stream_open_writable(&pb.pack_stream, pack_file_path, pool,
>> + pool));
>> + SVN_ERR(svn_stream_open_writable(&pb.manifest_stream, manifest_file_path,
>> + pool, pool));
>> + pb.next_offset = 0;
>> + pb.cancel_func = cancel_func;
>> + pb.cancel_baton = cancel_baton;
>> + SVN_ERR(svn_io_dir_walk(shard_path,
>> + APR_FINFO_TYPE | APR_FINFO_NAME | APR_FINFO_SIZE,
>> + packer_func, &pb, pool));
>> + SVN_ERR(svn_stream_close(pb.manifest_stream));
>> + SVN_ERR(svn_stream_close(pb.pack_stream));
>> +
>> + /* Update the max-pack-rev file to reflect our newly packed shard. */
>> + final_path = svn_path_join(fs_path, PATH_MAX_PACKED_REV, pool);
>> + SVN_ERR(svn_stream_open_unique(&stream, &tmp_path, fs_path,
>> + svn_io_file_del_none, pool, pool));
>> + SVN_ERR(svn_stream_printf(stream, pool, "%ld\n",
>> + (svn_revnum_t) ((shard + 1) * max_files_per_dir)));
>> + SVN_ERR(svn_stream_close(stream));
>> + SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, final_path, pool));
>> +
>> + /* Finally, remove the existing shard directory. */
>> + return svn_io_remove_dir2(shard_path, TRUE, cancel_func, cancel_baton, pool);
>> +}
>> +
>> +
>> +svn_error_t *
>> +svn_fs_fs__pack(const char *fs_path,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool)
>> +{
>> + int format, max_files_per_dir;
>> + int completed_shards;
>> + apr_int64_t i;
>> + svn_revnum_t youngest;
>> + apr_pool_t *iterpool;
>> + const char *data_path;
>> + svn_revnum_t max_packed_rev;
>> +
>> + SVN_ERR(read_format(&format, &max_files_per_dir,
>> + svn_path_join(fs_path, PATH_FORMAT, pool),
>> + pool));
>> +
>> + /* If the repository isn't a new enough format, we don't support packing.
>> + Return a friendly error to that effect. */
>> + if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
>> + return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
>> + _("FS format too old to pack, please upgrade."));
>> +
>> + /* If we aren't using sharding, we can't do any packing, so quit. */
>> + if (!max_files_per_dir)
>> + return SVN_NO_ERROR;
>> +
>> + SVN_ERR(read_max_packed_rev(&max_packed_rev,
>> + svn_path_join(fs_path, PATH_MAX_PACKED_REV, pool),
>> + pool));
>> +
>> + SVN_ERR(get_youngest(&youngest, fs_path, pool));
>> + completed_shards = youngest / max_files_per_dir;
>> +
>> + /* See if we've already completed all possible shards thus far. */
>> + if (max_packed_rev == (completed_shards * max_files_per_dir))
>> + return SVN_NO_ERROR;
>> +
>> + data_path = svn_path_join(fs_path, PATH_REVS_DIR, pool);
>> +
>> + iterpool = svn_pool_create(pool);
>> + for (i = max_packed_rev / max_files_per_dir; i < completed_shards; i++)
>> + {
>> + svn_pool_clear(iterpool);
>> +
>> + if (cancel_func)
>> + SVN_ERR(cancel_func(cancel_baton));
>> +
>> + SVN_ERR(pack_shard(data_path, fs_path, i, max_files_per_dir, cancel_func,
>> + cancel_baton, iterpool));
>> + /* We can't pack revprops, because they aren't immutable :(
>> + If we ever do get clever and figure out how to pack revprops,
>> + this is the place to do it. */
>> + }
>> +
>> + svn_pool_destroy(iterpool);
>> + return SVN_NO_ERROR;
>> +}
>>
>> Modified: trunk/subversion/libsvn_fs_fs/fs_fs.h
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/fs_fs.h?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/fs_fs.h Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/fs_fs.h Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -510,4 +510,16 @@ svn_error_t *
>> svn_fs_fs__initialize_caches(svn_fs_t *fs, apr_pool_t *pool);
>>
>>
>> +/* Possibly pack the repository at PATH. This just take full shards, and
>> + combines all the revision files into a single one, with a manifest header.
>> + Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support.
>> +
>> + Existing filesystem references need not change. */
>> +svn_error_t *
>> +svn_fs_fs__pack(const char *fs_path,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool);
>> +
>> +
>> #endif
>>
>> Modified: trunk/subversion/libsvn_fs_fs/structure
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_fs_fs/structure?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_fs_fs/structure Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_fs_fs/structure Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -34,6 +34,8 @@ repository) is:
>> revs/ Subdirectory containing revs
>> <shard>/ Shard directory, if sharding is in use (see below)
>> <revnum> File containing rev <revnum>
>> + <shard>.pack Pack file, if the repository has been packed (see below)
>> + <shard>.manifest Pack manifest file, if a pack file exists (see below)
>> revprops/ Subdirectory containing rev-props
>> <shard>/ Shard directory, if sharding is in use (see below)
>> <revnum> File containing rev-props for <revnum>
>> @@ -55,6 +57,7 @@ repository) is:
>> uuid File containing the UUID of the repository
>> format File containing the format number of this filesystem
>> fsfs.conf Configuration file
>> + max-packed-rev File containing the youngest revision to be in a pack file
>>
>> Files in the revprops directory are in the hash dump format used by
>> svn_hash_write.
>> @@ -116,39 +119,40 @@ The formats are:
>> Format 1, understood by Subversion 1.1+
>> Format 2, understood by Subversion 1.4+
>> Format 3, understood by Subversion 1.5+
>> + Format 4, understood by Subversion 1.6+
>>
>> The differences between the formats are:
>>
>> Delta representation in revision files
>> Format 1: svndiff0 only
>> - Formats 2-3: svndiff0 or svndiff1
>> + Formats 2-4: svndiff0 or svndiff1
>>
>> Format options
>> Formats 1-2: none permitted
>> - Format 3: "layout" option
>> + Format 3-4: "layout" option
>>
>> Transaction name reuse
>> Formats 1-2: transaction names may be reused
>> - Format 3: transaction names generated using txn-current file
>> + Format 3-4: transaction names generated using txn-current file
>>
>> Location of proto-rev file and its lock
>> Formats 1-2: transactions/<txnid>/rev and
>> transactions/<txnid>/rev-lock.
>> - Format 3: txn-protorevs/<txnid>.rev and
>> + Format 3-4: txn-protorevs/<txnid>.rev and
>> txn-protorevs/<txnid>.rev-lock.
>>
>> Node-ID and copy-ID generation
>> Formats 1-2: Node-IDs and copy-IDs are guaranteed to form a
>> monotonically increasing base36 sequence using the "current"
>> file.
>> - Format 3: Node-IDs and copy-IDs use the new revision number to
>> + Format 3-4: Node-IDs and copy-IDs use the new revision number to
>> ensure uniqueness and the "current" file just contains the
>> youngest revision.
>>
>> Mergeinfo metadata:
>> Format 1-2: minfo-here and minfo-count node-revision fields are not
>> stored. svn_fs_get_mergeinfo returns an error.
>> - Format 3: minfo-here and minfo-count node-revision fields are
>> + Format 3-4: minfo-here and minfo-count node-revision fields are
>> maintained. svn_fs_get_mergeinfo works.
>>
>>
>> @@ -185,6 +189,16 @@ The known layouts, and the parameters th
>> revs/0/ directory will contain revisions 0-999, revs/1/ will contain
>> 1000-1999, and so on.
>>
>> +Packing
>> +-------
>> +
>> +A repository can optionally be "packed" to conserve space on disk. The
>> +packing process concatenates all the revision files in each full shard to
>> +create pack files. A manifest file is also created for each shard which
>> +records the indexes of the corresponding revision files in the pack file.
>> +In addition, the original shard is removed, and reads are redirected to the
>> +pack file.
>
> You might want to define the format of the manifest file.
>
>> +
>> Node-revision IDs
>> -----------------
>>
>>
>> Modified: trunk/subversion/libsvn_repos/fs-wrap.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/libsvn_repos/fs-wrap.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/libsvn_repos/fs-wrap.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/libsvn_repos/fs-wrap.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -633,6 +633,15 @@ svn_repos_fs_get_mergeinfo(svn_mergeinfo
>> return SVN_NO_ERROR;
>> }
>>
>> +svn_error_t *
>> +svn_repos_fs_pack(svn_repos_t *repos,
>> + svn_cancel_func_t cancel_func,
>> + void *cancel_baton,
>> + apr_pool_t *pool)
>> +{
>> + return svn_fs_pack(repos->db_path, cancel_func, cancel_baton, pool);
>> +}
>> +
>>
>>
>> /*
>>
>> Modified: trunk/subversion/svnadmin/main.c
>> URL: http://svn.collab.net/viewvc/svn/trunk/subversion/svnadmin/main.c?pathrev=34502&r1=34501&r2=34502
>> ==============================================================================
>> --- trunk/subversion/svnadmin/main.c Mon Dec 1 08:31:28 2008 (r34501)
>> +++ trunk/subversion/svnadmin/main.c Mon Dec 1 08:59:39 2008 (r34502)
>> @@ -200,7 +200,8 @@ static svn_opt_subcommand_t
>> subcommand_setrevprop,
>> subcommand_setuuid,
>> subcommand_upgrade,
>> - subcommand_verify;
>> + subcommand_verify,
>> + subcommand_pack;
>>
>> enum
>> {
>> @@ -462,6 +463,12 @@ static const svn_opt_subcommand_desc2_t
>> "Verifies the data stored in the repository.\n"),
>> {'r', 'q'} },
>>
>> + {"pack", subcommand_pack, {0}, N_
>> + ("usage: svnadmin pack REPOS_PATH\n\n"
>> + "Possibly compact the repository into a more effecient storage model.\n"
>> + "This may not apply to all repositories, in which case, exit.\n"),
>> + {0} },
>> +
>> { NULL, NULL, {0}, NULL, {0} }
>> };
>>
>> @@ -1115,6 +1122,19 @@ subcommand_setlog(apr_getopt_t *os, void
>> }
>>
>>
>> +/* This implements 'svn_opt_subcommand_t'. */
>> +static svn_error_t *
>> +subcommand_pack(apr_getopt_t *os, void *baton, apr_pool_t *pool)
>> +{
>> + struct svnadmin_opt_state *opt_state = baton;
>> + svn_repos_t *repos;
>> +
>> + SVN_ERR(open_repos(&repos, opt_state->repository_path, pool));
>> +
>> + return svn_repos_fs_pack(repos, check_cancel, NULL, pool);
>> +}
>> +
>> +
>> /* This implements `svn_opt_subcommand_t'. */
>> static svn_error_t *
>> subcommand_verify(apr_getopt_t *os, void *baton, apr_pool_t *pool)
>>
>> Copied: trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c (from r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c)
>> ==============================================================================
>> --- /dev/null 00:00:00 1970 (empty, because file is newly added)
>> +++ trunk/subversion/tests/libsvn_fs_fs/fs-pack-test.c Mon Dec 1 08:59:39 2008 (r34502, copy of r34501, branches/fsfs-pack/subversion/tests/libsvn_fs_fs/fs-pack-test.c)
>> @@ -0,0 +1,325 @@
>> +/* fs-pack-test.c --- tests for the filesystem
>> + *
>> + * ====================================================================
>> + * Copyright (c) 2008 CollabNet. All rights reserved.
>> + *
>> + * This software is licensed as described in the file COPYING, which
>> + * you should have received as part of this distribution. The terms
>> + * are also available at http://subversion.tigris.org/license-1.html.
>> + * If newer versions of this license are posted there, you may use a
>> + * newer version instead, at your option.
>> + *
>> + * This software consists of voluntary contributions made by many
>> + * individuals. For exact contribution history, see the revision
>> + * history and logs, available at http://subversion.tigris.org/.
>> + * ====================================================================
>> + */
>> +
>> +#include <stdlib.h>
>> +#include <string.h>
>> +#include <apr_pools.h>
>> +
>> +#include "../svn_test.h"
>> +#include "../../libsvn_fs_fs/fs.h"
>> +
>> +#include "svn_pools.h"
>> +#include "svn_fs.h"
>> +
>> +#include "../svn_test_fs.h"
>> +
>> +
>> +/*-----------------------------------------------------------------*/
>> +
>> +/** The actual fs-tests called by `make check` **/
>> +
>> +/* Write the format number and maximum number of files per directory
>> + to a new format file in PATH, overwriting a previously existing file.
>> +
>> + Use POOL for temporary allocation.
>> +
>> + This implementation is largely stolen from libsvn_fs_fs/fs_fs.c. */
>> +static svn_error_t *
>> +write_format(const char *path,
>> + int format,
>> + int max_files_per_dir,
>> + apr_pool_t *pool)
>> +{
>> + const char *contents;
>> +
>> + path = svn_path_join(path, "format", pool);
>> +
>> + if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT)
>> + {
>> + if (max_files_per_dir)
>> + contents = apr_psprintf(pool,
>> + "%d\n"
>> + "layout sharded %d\n",
>> + format, max_files_per_dir);
>> + else
>> + contents = apr_psprintf(pool,
>> + "%d\n"
>> + "layout linear",
>> + format);
>> + }
>> + else
>> + {
>> + contents = apr_psprintf(pool, "%d\n", format);
>> + }
>> +
>> + {
>> + const char *path_tmp;
>> +
>> + SVN_ERR(svn_io_write_unique(&path_tmp,
>> + svn_path_dirname(path, pool),
>> + contents, strlen(contents),
>> + svn_io_file_del_none, pool));
>> +
>> +#ifdef WIN32
>> + /* make the destination writable, but only on Windows, because
>> + Windows does not let us replace read-only files. */
>> + SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool));
>> +#endif /* WIN32 */
>> +
>> + /* rename the temp file as the real destination */
>> + SVN_ERR(svn_io_file_rename(path_tmp, path, pool));
>> + }
>> +
>> + /* And set the perms to make it read only */
>> + return svn_io_set_file_read_only(path, FALSE, pool);
>> +}
>> +
>> +/* Return the expected contents of "iota" in revision REV. */
>> +static const char *
>> +get_rev_contents(svn_revnum_t rev, apr_pool_t *pool)
>> +{
>> + /* Toss in a bunch of magic numbers for spice. */
>> + apr_int64_t num = ((rev * 1234353 + 4358) * 4583 + ((rev % 4) << 1)) / 42;
>> + return apr_psprintf(pool, "%" APR_INT64_T_FMT "\n", num);
>> +}
>> +
>> +/* Create a packed filesystem in DIR. Set the shard size to SHARD_SIZE
>> + and create MAX_REV number of revisions. Use POOL for allocations. */
>> +static svn_error_t *
>> +create_packed_filesystem(const char *dir,
>> + svn_test_opts_t *opts,
>> + int max_rev,
>> + int shard_size,
>> + apr_pool_t *pool)
>> +{
>> + svn_fs_t *fs;
>> + svn_fs_txn_t *txn;
>> + svn_fs_root_t *txn_root;
>> + const char *conflict;
>> + svn_revnum_t after_rev;
>> + apr_pool_t *subpool = svn_pool_create(pool);
>> +
>> + /* Create a filesystem, then close it */
>> + SVN_ERR(svn_test__create_fs(&fs, dir, opts, subpool));
>> + svn_pool_destroy(subpool);
>> +
>> + subpool = svn_pool_create(pool);
>> +
>> + /* Rewrite the format file */
>> + SVN_ERR(write_format(dir, SVN_FS_FS__MIN_PACKED_FORMAT,
>> + shard_size, subpool));
>> +
>> + /* Reopen the filesystem */
>> + SVN_ERR(svn_fs_open(&fs, dir, NULL, subpool));
>> +
>> + /* Revision 1: the Greek tree */
>> + SVN_ERR(svn_fs_begin_txn(&txn, fs, 0, subpool));
>> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, subpool));
>> + SVN_ERR(svn_test__create_greek_tree(txn_root, subpool));
>> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, subpool));
>> +
>> + /* Revisions 2-11: A bunch of random changes. */
>> + while (after_rev < max_rev + 1)
>> + {
>> + SVN_ERR(svn_fs_begin_txn(&txn, fs, after_rev, subpool));
>> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, subpool));
>> + SVN_ERR(svn_test__set_file_contents(txn_root, "iota",
>> + get_rev_contents(after_rev + 1,
>> + subpool),
>> + subpool));
>> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, subpool));
>> + }
>> + svn_pool_destroy(subpool);
>> +
>> + /* Now pack the FS */
>> + return svn_fs_pack(dir, NULL, NULL, pool);
>> +}
>> +
>> +/* Pack a filesystem. */
>> +#define REPO_NAME "test-repo-fsfs-pack"
>
> I'm not sure why these all have to be #defines rather than const char
> *, const int, etc.
>
>> +#define SHARD_SIZE 7
>> +#define MAX_REV 53
>> +static svn_error_t *
>> +pack_filesystem(const char **msg,
>> + svn_boolean_t msg_only,
>> + svn_test_opts_t *opts,
>> + apr_pool_t *pool)
>> +{
>> + int i;
>> + svn_node_kind_t kind;
>> + const char *path;
>> + char buf[80];
>> + apr_file_t *file;
>> + apr_size_t len;
>> +
>> + *msg = "pack a FSFS filesystem";
>> +
>> + if (msg_only)
>> + return SVN_NO_ERROR;
>> +
>> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, MAX_REV, SHARD_SIZE,
>> + pool));
>> +
>> + /* Check to see that the pack files exist, and that the rev directories
>> + don't. */
>> + for (i = 0; i < (MAX_REV + 1) / SHARD_SIZE; i++)
>> + {
>> + path = svn_path_join_many(pool, REPO_NAME, "revs",
>> + apr_psprintf(pool, "%d.pack", i / SHARD_SIZE), NULL);
>> +
>> + /* These files should exist. */
>> + SVN_ERR(svn_io_check_path(path, &kind, pool));
>> + if (kind != svn_node_file)
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Expected pack file '%s' not found", path);
>> +
>> + path = svn_path_join_many(pool, REPO_NAME, "revs",
>> + apr_psprintf(pool, "%d.manifest", i / SHARD_SIZE), NULL);
>> + SVN_ERR(svn_io_check_path(path, &kind, pool));
>> + if (kind != svn_node_file)
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Expected manifest file '%s' not found",
>> + path);
>> +
>> + /* This directory should not exist. */
>> + path = svn_path_join_many(pool, REPO_NAME, "revs",
>> + apr_psprintf(pool, "%d", i / SHARD_SIZE), NULL);
>> + SVN_ERR(svn_io_check_path(path, &kind, pool));
>> + if (kind != svn_node_none)
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Unexpected directory '%s' found", path);
>> + }
>> +
>> + /* Ensure the max-packed-rev jives with the above operations. */
>> + SVN_ERR(svn_io_file_open(&file,
>> + svn_path_join(REPO_NAME, "max-packed-rev", pool),
>> + APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
>> + len = sizeof(buf);
>> + SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
>> + SVN_ERR(svn_io_file_close(file, pool));
>> + if (SVN_STR_TO_REV(buf) != (MAX_REV / SHARD_SIZE) * SHARD_SIZE)
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Bad 'max-packed-rev' contents");
>> +
>> + /* Finally, make sure the final revision directory does exist. */
>> + path = svn_path_join_many(pool, REPO_NAME, "revs",
>> + apr_psprintf(pool, "%d", (i / SHARD_SIZE) + 1), NULL);
>> + SVN_ERR(svn_io_check_path(path, &kind, pool));
>> + if (kind != svn_node_none)
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Expected directory '%s' not found",
>> + path);
>> +
>> +
>> + return SVN_NO_ERROR;
>> +}
>> +#undef REPO_NAME
>> +#undef SHARD_SIZE
>> +#undef MAX_REV
>> +
>> +
>> +/* Check reading from a packed filesystem. */
>> +#define REPO_NAME "test-repo-read-packed-fs"
>> +static svn_error_t *
>> +read_packed_fs(const char **msg,
>> + svn_boolean_t msg_only,
>> + svn_test_opts_t *opts,
>> + apr_pool_t *pool)
>> +{
>> + svn_fs_t *fs;
>> + svn_stream_t *rstream;
>> + svn_stringbuf_t *rstring;
>> + svn_revnum_t i;
>> +
>> + *msg = "read from a packed FSFS filesystem";
>> +
>> + if (msg_only)
>> + return SVN_NO_ERROR;
>> +
>> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, 11, 5, pool));
>> + SVN_ERR(svn_fs_open(&fs, REPO_NAME, NULL, pool));
>> +
>> + for (i = 1; i < 12; i++)
>> + {
>> + svn_fs_root_t *rev_root;
>> + svn_stringbuf_t *sb;
>> +
>> + SVN_ERR(svn_fs_revision_root(&rev_root, fs, i, pool));
>> + SVN_ERR(svn_fs_file_contents(&rstream, rev_root, "iota", pool));
>> + SVN_ERR(svn_test__stream_to_string(&rstring, rstream, pool));
>> +
>> + if (i == 1)
>> + sb = svn_stringbuf_create("This is the file 'iota'.\n", pool);
>> + else
>> + sb = svn_stringbuf_create(get_rev_contents(i, pool), pool);
>> +
>> + if (! svn_stringbuf_compare(rstring, sb))
>> + return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
>> + "Bad data in revision %ld.", i);
>> + }
>> +
>> + return SVN_NO_ERROR;
>> +}
>> +#undef REPO_NAME
>> +
>> +/* Check reading from a packed filesystem. */
>> +#define REPO_NAME "test-repo-commit-packed-fs"
>> +static svn_error_t *
>> +commit_packed_fs(const char **msg,
>> + svn_boolean_t msg_only,
>> + svn_test_opts_t *opts,
>> + apr_pool_t *pool)
>> +{
>> + svn_fs_t *fs;
>> + svn_fs_txn_t *txn;
>> + svn_fs_root_t *txn_root;
>> + const char *conflict;
>> + svn_revnum_t after_rev;
>> +
>> + *msg = "commit to a packed FSFS filesystem";
>> +
>> + if (msg_only)
>> + return SVN_NO_ERROR;
>> +
>> + /* Create the packed FS and open it. */
>> + SVN_ERR(create_packed_filesystem(REPO_NAME, opts, 11, 5, pool));
>> + SVN_ERR(svn_fs_open(&fs, REPO_NAME, NULL, pool));
>> +
>> + /* Now do a commit. */
>> + SVN_ERR(svn_fs_begin_txn(&txn, fs, 12, pool));
>> + SVN_ERR(svn_fs_txn_root(&txn_root, txn, pool));
>> + SVN_ERR(svn_test__set_file_contents(txn_root, "iota",
>> + "How much better is it to get wisdom than gold! and to get "
>> + "understanding rather to be chosen than silver!", pool));
>> + SVN_ERR(svn_fs_commit_txn(&conflict, &after_rev, txn, pool));
>> +
>> + return SVN_NO_ERROR;
>> +}
>> +#undef REPO_NAME
>> +
>> +/* ------------------------------------------------------------------------ */
>> +
>> +/* The test table. */
>> +
>> +struct svn_test_descriptor_t test_funcs[] =
>> + {
>> + SVN_TEST_NULL,
>> + SVN_TEST_PASS(pack_filesystem),
>> + SVN_TEST_PASS(read_packed_fs),
>> + SVN_TEST_PASS(commit_packed_fs),
>> + SVN_TEST_NULL
>> + };
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: svn-unsubscribe_at_subversion.tigris.org
>> For additional commands, e-mail: svn-help_at_subversion.tigris.org
>>
>>
>
> --dave
>
Received on 2008-12-02 21:14:15 CET