The rather long thread about when to use transactions, how often to
checkpoint, and the impact of fsync on performance prompted me to come
up with a little demonstratoin. This patch does the following:
* Adds a mechanism for passing configuration options to the FS layer
in a way that's independent of the actual FS implementation
* Adds an option to "svnadmin create" to configure the FS so that
fsyncs on transaction commit are disabled
* Uses this feature in all the FS and Python tests.
This patch reduces "make check" time over ra_local on my Windows box
from 56 to 34 minutes. I'd like to ask people to do two things: measure
the effects of this patch on their machines, and especially comment on
the FS configuration mechanism.
The new "svnadmin create" flag would be useful for cvs2svn'ing large
repositories, and "svnadmin load"ing them, too, not just for testing.
The administrator could remove the nosync flag from DB_CONFIG before
putting the newly-created repository into production.
The log message (if I were to commit this patch) would look a bit like
this (see, Karl, I /told/ you :svn_config_create should be publc :-) :
* include/svn_fs.h (svn_fs_new): New parameter "fs_config". All callers
changed.
* include/svn_repos.h (svn_repos_open, svn_repos_create): New parameter
"fs_config". All callers changed.
* include/svn_config.h (svn_config_create): New function.
* libsvn_subr/config.c (svn_config_create, svn_config_read): Extract the
svn_config_t allocation bits from svn_config_read into svn_config_create.
* libsvn_fs/fs.h (svn_fs_t): New member "config".
* libsvn_fs/fs.c (svn_fs_new): Initialise the "config" member of the
svn_fs_t.
(svn_fs_create_berkeley): Add "set_flags DB_TXN_NOSYNC" to DB_CONFIG if
the nosync option was set in fs->config.
* libsvn_repos/repos.c (svn_repos_create, get_repos): Pass fs_config to
svn_fs_new. All callers changed.
* svnadmin/main.c: New "create" option --bdb-txn-nosync. If set, pass
the nosync option to the repos layer when creating the repository.
* tests/fs-helpers.c, tests/libsvn_fs/fs-test.c: Pass the nosync option
to svn_fs_new and svn_repos_create.
* tests/clients/cmdline/svntest/main.py (create_repos): Pass
--bdb-txn-nosync to "svnadmin create".
--
Brane Čibej <brane_at_xbc.nu> http://www.xbc.nu/brane/
Index: subversion/svnadmin/main.c
===================================================================
--- subversion/svnadmin/main.c (revision 5012)
+++ subversion/svnadmin/main.c (working copy)
@@ -83,7 +83,8 @@
svnadmin__on_disk_template,
svnadmin__in_repos_template,
svnadmin__ignore_uuid,
- svnadmin__force_uuid
+ svnadmin__force_uuid,
+ svnadmin__bdb_txn_nosync
};
/* Option codes and descriptions.
@@ -125,6 +126,9 @@
{"force-uuid", svnadmin__force_uuid, 0,
"set repos UUID to that found in stream, if any."},
+ {"bdb-txn-nosync", svnadmin__bdb_txn_nosync, 0,
+ "disable fsync at database transaction commit [Berkeley DB]."},
+
{NULL}
};
@@ -137,7 +141,8 @@
{"create", subcommand_create, {0},
"usage: svnadmin create REPOS_PATH\n\n"
"Create a new, empty repository at REPOS_PATH.\n",
- {svnadmin__on_disk_template, svnadmin__in_repos_template} },
+ {svnadmin__on_disk_template, svnadmin__in_repos_template,
+ svnadmin__bdb_txn_nosync} },
{"createtxn", subcommand_createtxn, {0},
"usage: svnadmin createtxn REPOS_PATH -r REVISION\n\n"
@@ -216,6 +221,7 @@
svn_boolean_t incremental; /* --incremental */
svn_boolean_t follow_copies; /* --copies */
svn_boolean_t quiet; /* --quiet */
+ svn_boolean_t bdb_txn_nosync; /* --bdb-txn-nosync */
enum svn_repos_load_uuid uuid_action; /* --ignore-uuid,
--force-uuid */
const char *on_disk;
@@ -229,11 +235,18 @@
struct svnadmin_opt_state *opt_state = baton;
svn_repos_t *repos;
apr_hash_t *config;
-
+ svn_config_t *fs_config = NULL;
+
+ if (opt_state->bdb_txn_nosync)
+ {
+ fs_config = svn_config_create (pool);
+ svn_config_set (fs_config, "fs:berkeley-db", "DB_TXN_NOSYNC", "1");
+ }
+
SVN_ERR (svn_config_get_config (&config, pool));
SVN_ERR (svn_repos_create (&repos, opt_state->repository_path,
opt_state->on_disk, opt_state->in_repos,
- config, pool));
+ config, fs_config, pool));
return SVN_NO_ERROR;
}
@@ -255,7 +268,7 @@
return svn_error_createf (SVN_ERR_CL_ARG_PARSING_ERROR, NULL,
"only one revision allowed");
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
SVN_ERR (svn_fs_begin_txn (&txn, fs, opt_state->start_revision.value.number,
pool));
@@ -277,7 +290,7 @@
lower = SVN_INVALID_REVNUM,
upper = SVN_INVALID_REVNUM;
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
/* ### We only handle revision numbers right now, not dates. */
@@ -351,7 +364,7 @@
svn_repos_t *repos;
svn_stream_t *stdin_stream, *stdout_stream;
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
/* Read the stream from STDIN. Users can redirect a file. */
SVN_ERR (create_stdio_stream (&stdin_stream,
@@ -394,7 +407,7 @@
*(const char **)apr_array_push(paths) =
svn_path_internal_style (path_utf8, pool);
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
svn_fs_youngest_rev (&youngest_rev, fs, pool);
SVN_ERR (svn_fs_revision_root (&rev_root, fs, youngest_rev, pool));
@@ -420,7 +433,7 @@
apr_array_header_t *txns;
int i;
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
SVN_ERR (svn_fs_list_transactions (&txns, fs, pool));
@@ -453,7 +466,7 @@
/* Since db transactions may have been replayed, it's nice to tell
people what the latest revision is. It also proves that the
recovery actually worked. */
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
SVN_ERR (svn_fs_youngest_rev (&youngest_rev, svn_repos_fs (repos), pool));
printf ("The latest repos revision is %"
SVN_REVNUM_T_FMT ".\n", youngest_rev);
@@ -474,7 +487,7 @@
int i;
apr_pool_t *subpool = svn_pool_create (pool);
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
SVN_ERR (svn_opt_parse_all_args (&args, os, pool));
@@ -533,7 +546,7 @@
NULL, pool));
/* open the filesystem */
- SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, pool));
+ SVN_ERR (svn_repos_open (&repos, opt_state->repository_path, NULL, pool));
fs = svn_repos_fs (repos);
/* set the revision property */
@@ -676,6 +689,9 @@
case svnadmin__force_uuid:
opt_state.uuid_action = svn_repos_load_uuid_force;
break;
+ case svnadmin__bdb_txn_nosync:
+ opt_state.bdb_txn_nosync = TRUE;
+ break;
default:
{
subcommand_help (NULL, NULL, pool);
Index: subversion/include/svn_fs.h
===================================================================
--- subversion/include/svn_fs.h (revision 5012)
+++ subversion/include/svn_fs.h (working copy)
@@ -30,6 +30,7 @@
#include "svn_error.h"
#include "svn_delta.h"
#include "svn_io.h"
+#include "svn_config.h"
#ifdef __cplusplus
@@ -44,17 +45,23 @@
typedef struct svn_fs_t svn_fs_t;
-/** Create a new filesystem object.
+/** Create a new filesystem object in @a pool.
*
- * Create a new filesystem object in @a pool. It doesn't refer to any
- * actual repository yet; you need to invoke @c svn_fs_open_* or
- * @c svn_fs_create_* on it for that to happen.
+ * It doesn't refer to any actual repository yet; you need to invoke
+ * @c svn_fs_open_* or @c svn_fs_create_* on it for that to happen. If
+ * @a fs_config is not @c NULL, the options it contains modify the
+ * behaviour of the filesystem. The interpretation of @a fs_config is
+ * specific to the filesystem back-end.
*
- * NOTE: you probably don't want to use this directly, especially not
+ * @note The lifetime of @a fs_config must not be shorter than @a
+ * pool's. It's a good idea to allocate @ fs_config from @a pool or
+ * one of its ancestors.
+ *
+ * @note You probably don't want to use this directly, especially not
* if it's followed immediately by a call to @c svn_fs_open_berkeley().
* Take a look at @c svn_repos_open() instead.
*/
-svn_fs_t *svn_fs_new (apr_pool_t *pool);
+svn_fs_t *svn_fs_new (svn_config_t *fs_config, apr_pool_t *pool);
/** The type of a warning callback function.
Index: subversion/include/svn_repos.h
===================================================================
--- subversion/include/svn_repos.h (revision 5012)
+++ subversion/include/svn_repos.h (working copy)
@@ -29,6 +29,7 @@
#include "svn_delta.h"
#include "svn_types.h"
#include "svn_error.h"
+#include "svn_config.h"
#ifdef __cplusplus
@@ -46,8 +47,8 @@
/** Set @a *repos_p to a repository object for the repository at @a path.
*
- * Set @a *repos_p to a repository object for the repository at @a path.
- * Allocate @a *repos_p in @a pool.
+ * Allocate @a *repos_p in @a pool. @a fs_config is passed to the
+ * filesystem, and may be NULL.
*
* Acquires a shared lock on the repository, and attaches a cleanup
* function to @a pool to remove the lock. If no lock can be acquired,
@@ -56,6 +57,7 @@
*/
svn_error_t *svn_repos_open (svn_repos_t **repos_p,
const char *path,
+ svn_config_t *fs_config,
apr_pool_t *pool);
/** Create a new Subversion repository at @a path.
@@ -81,12 +83,15 @@
*
* @a config is a client configuration hash of @c svn_config_t * items
* keyed on config category names, and may be NULL.
+ *
+ * @a fs_config is passed to the filesystem, and may be NULL.
*/
svn_error_t *svn_repos_create (svn_repos_t **repos_p,
const char *path,
const char *on_disk_template,
const char *in_repos_template,
apr_hash_t *config,
+ svn_config_t *fs_config,
apr_pool_t *pool);
/** Destroy the Subversion repository found at @a path, using @a pool for any
Index: subversion/include/svn_config.h
===================================================================
--- subversion/include/svn_config.h (revision 5012)
+++ subversion/include/svn_config.h (working copy)
@@ -76,6 +76,13 @@
#define SVN_CONFIG_OPTION_TEMPLATE_ROOT "template-root"
+/** Create an empty @c svn_config_t object.
+ *
+ * Allocate the object from @pool.
+ */
+svn_config_t *svn_config_create (apr_pool_t *pool);
+
+
/** Read configuration information from the standard sources and
* return it in the hash @a *cfg_hash.
*
Index: subversion/libsvn_fs/fs.c
===================================================================
--- subversion/libsvn_fs/fs.c (revision 5012)
+++ subversion/libsvn_fs/fs.c (working copy)
@@ -310,7 +310,7 @@
/* Allocating and freeing filesystem objects. */
svn_fs_t *
-svn_fs_new (apr_pool_t *parent_pool)
+svn_fs_new (svn_config_t *fs_config, apr_pool_t *parent_pool)
{
svn_fs_t *new_fs;
@@ -324,6 +324,7 @@
}
new_fs->warning = default_warning_func;
+ new_fs->config = fs_config;
apr_pool_cleanup_register (new_fs->pool, new_fs,
cleanup_fs_apr,
@@ -452,6 +453,15 @@
"set_lg_bsize 262144\n"
"set_lg_max 1048576\n";
+ static const char dbconfig_txn_nosync[] =
+ "#\n"
+ "# Disable fsync of log files on transaction commit. Read the\n"
+ "# documentation abtou DB_TXN_NOSYNC at:\n"
+ "#\n"
+ "# http://www.sleepycat.com/docs/api_c/env_set_flags.html\n"
+ "#\n"
+ "set_flags DB_TXN_NOSYNC\n";
+
SVN_ERR (svn_io_file_open (&dbconfig_file, dbconfig_file_name,
APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
fs->pool));
@@ -462,6 +472,23 @@
return svn_error_createf (apr_err, 0,
"writing to `%s'", dbconfig_file_name);
+ if (fs->config)
+ {
+ const char *value;
+
+ svn_config_get (fs->config, &value,
+ "fs:berkeley-db", "DB_TXN_NOSYNC", NULL);
+ if (value != NULL)
+ {
+ apr_err = apr_file_write_full (dbconfig_file, dbconfig_txn_nosync,
+ sizeof (dbconfig_txn_nosync) - 1,
+ NULL);
+ if (apr_err != APR_SUCCESS)
+ return svn_error_createf (apr_err, 0,
+ "writing to `%s'", dbconfig_file_name);
+ }
+ }
+
apr_err = apr_file_close (dbconfig_file);
if (apr_err != APR_SUCCESS)
return svn_error_createf (apr_err, 0,
Index: subversion/libsvn_fs/fs.h
===================================================================
--- subversion/libsvn_fs/fs.h (revision 5012)
+++ subversion/libsvn_fs/fs.h (working copy)
@@ -59,6 +59,9 @@
pass through to it. */
svn_fs_warning_callback_t warning;
void *warning_baton;
+
+ /* The filesystem configuration. */
+ svn_config_t *config;
};
Index: subversion/libsvn_subr/config.c
===================================================================
--- subversion/libsvn_subr/config.c (revision 5012)
+++ subversion/libsvn_subr/config.c (working copy)
@@ -69,12 +69,9 @@
-svn_error_t *
-svn_config_read (svn_config_t **cfgp, const char *file,
- svn_boolean_t must_exist, apr_pool_t *pool)
+svn_config_t *svn_config_create (apr_pool_t *pool)
{
svn_config_t *cfg = apr_palloc (pool, sizeof (*cfg));
- svn_error_t *err;
cfg->sections = apr_hash_make (pool);
cfg->pool = pool;
@@ -82,6 +79,18 @@
cfg->x_values = FALSE;
cfg->tmp_key = svn_stringbuf_create ("", pool);
+ return cfg;
+}
+
+
+
+svn_error_t *
+svn_config_read (svn_config_t **cfgp, const char *file,
+ svn_boolean_t must_exist, apr_pool_t *pool)
+{
+ svn_config_t *cfg = svn_config_create (pool);
+ svn_error_t *err;
+
/* Yes, this is platform-specific code in Subversion, but there's no
practical way to migrate it into APR, as it's simultaneously
Subversion-specific and Windows-specific. Even if we eventually
Index: subversion/libsvn_ra_local/split_url.c
===================================================================
--- subversion/libsvn_ra_local/split_url.c (revision 5012)
+++ subversion/libsvn_ra_local/split_url.c (working copy)
@@ -104,7 +104,7 @@
while (1)
{
/* Attempt to open a repository at URL. */
- err = svn_repos_open (repos, candidate_url, pool);
+ err = svn_repos_open (repos, candidate_url, NULL, pool);
/* Hey, cool, we were successful. Stop looping. */
if (err == SVN_NO_ERROR)
Index: subversion/svnlook/main.c
===================================================================
--- subversion/svnlook/main.c (revision 5012)
+++ subversion/svnlook/main.c (working copy)
@@ -974,7 +974,8 @@
{
svnlook_ctxt_t *baton = apr_pcalloc (pool, sizeof (*baton));
- SVN_ERR (svn_repos_open (&(baton->repos), opt_state->repos_path, pool));
+ SVN_ERR (svn_repos_open (&(baton->repos), opt_state->repos_path,
+ NULL, pool));
baton->fs = svn_repos_fs (baton->repos);
baton->show_ids = opt_state->show_ids;
baton->no_diff_deleted = opt_state->no_diff_deleted;
Index: subversion/tests/libsvn_fs/fs-test.c
===================================================================
--- subversion/tests/libsvn_fs/fs-test.c (revision 5012)
+++ subversion/tests/libsvn_fs/fs-test.c (working copy)
@@ -305,15 +305,18 @@
svn_boolean_t msg_only,
apr_pool_t *pool)
{
+ svn_fs_t *fs;
svn_error_t *err;
- svn_fs_t *fs = svn_fs_new (pool);
+ svn_config_t *fs_config = svn_config_create (pool);
+ svn_config_set (fs_config, "fs:berkeley-db", "DB_TXN_NOSYNC", "1");
+ fs = svn_fs_new (fs_config, pool);
*msg = "call functions with unopened filesystem and check errors";
if (msg_only)
return SVN_NO_ERROR;
- fs = svn_fs_new (pool);
+ fs = svn_fs_new (fs_config, pool);
err = svn_fs_set_berkeley_errcall (fs, berkeley_error_handler);
SVN_ERR (check_no_fs_error (err));
Index: subversion/tests/fs-helpers.c
===================================================================
--- subversion/tests/fs-helpers.c (revision 5012)
+++ subversion/tests/fs-helpers.c (working copy)
@@ -55,7 +55,10 @@
svn_error_t *
svn_test__fs_new (svn_fs_t **fs_p, apr_pool_t *pool)
{
- *fs_p = svn_fs_new (pool);
+ svn_config_t *fs_config = svn_config_create (pool);
+ svn_config_set (fs_config, "fs:berkeley-db", "DB_TXN_NOSYNC", "1");
+
+ *fs_p = svn_fs_new (fs_config, pool);
if (! *fs_p)
return svn_error_create (SVN_ERR_FS_GENERAL, NULL,
"Couldn't alloc a new fs object.");
@@ -105,6 +108,7 @@
apr_pool_t *pool)
{
apr_finfo_t finfo;
+ svn_config_t *fs_config;
/* If there's already a repository named NAME, delete it. Doing
things this way means that repositories stick around after a
@@ -120,7 +124,10 @@
"there is already a file named `%s'", name);
}
- SVN_ERR (svn_repos_create (repos_p, name, NULL, NULL, NULL, pool));
+ fs_config = svn_config_create (pool);
+ svn_config_set (fs_config, "fs:berkeley-db", "DB_TXN_NOSYNC", "1");
+ SVN_ERR (svn_repos_create (repos_p, name, NULL, NULL, NULL,
+ fs_config, pool));
/* Provide a handler for Berkeley DB error messages. */
SVN_ERR (svn_fs_set_berkeley_errcall (svn_repos_fs (*repos_p),
Index: subversion/tests/clients/cmdline/svntest/main.py
===================================================================
--- subversion/tests/clients/cmdline/svntest/main.py (revision 5012)
+++ subversion/tests/clients/cmdline/svntest/main.py (working copy)
@@ -281,7 +281,7 @@
if not(os.path.exists(path)):
os.makedirs(path) # this creates all the intermediate dirs, if neccessary
- run_svnadmin("create", path)
+ run_svnadmin("create", path, "--bdb-txn-nosync")
# make the repos world-writeable, for mod_dav_svn's sake.
chmod_tree(path, 0666, 0666)
Index: subversion/libsvn_repos/repos.c
===================================================================
--- subversion/libsvn_repos/repos.c (revision 5012)
+++ subversion/libsvn_repos/repos.c (working copy)
@@ -25,7 +25,6 @@
#include "svn_path.h"
#include "svn_fs.h"
#include "svn_repos.h"
-#include "svn_config.h"
#include "svn_private_config.h" /* for SVN_TEMPLATE_ROOT_DIR */
#include "repos.h"
@@ -761,6 +760,7 @@
const char *on_disk_template,
const char *in_repos_template,
apr_hash_t *config,
+ svn_config_t *fs_config,
apr_pool_t *pool)
{
svn_repos_t *repos;
@@ -831,7 +831,7 @@
/* The on-disk structure should be built now. */
/* Initialize the filesystem object. */
- repos->fs = svn_fs_new (pool);
+ repos->fs = svn_fs_new (fs_config, pool);
/* Create a Berkeley DB environment for the filesystem. */
SVN_ERR (svn_fs_create_berkeley (repos->fs, repos->db_path));
@@ -888,6 +888,7 @@
const char *path,
int locktype,
svn_boolean_t open_fs,
+ svn_config_t *fs_config,
apr_pool_t *pool)
{
apr_status_t apr_err;
@@ -904,7 +905,7 @@
init_repos_dirs (repos, pool);
/* Initialize the filesystem object. */
- repos->fs = svn_fs_new (pool);
+ repos->fs = svn_fs_new (fs_config, pool);
/* Open up the Berkeley filesystem. */
if (open_fs)
@@ -955,6 +956,7 @@
svn_error_t *
svn_repos_open (svn_repos_t **repos_p,
const char *path,
+ svn_config_t *fs_config,
apr_pool_t *pool)
{
/* Fetch a repository object initialized with a shared read/write
@@ -963,6 +965,7 @@
SVN_ERR (get_repos (repos_p, path,
APR_FLOCK_SHARED,
TRUE, /* open the db into repos->fs. */
+ fs_config,
pool));
return SVN_NO_ERROR;
@@ -1049,6 +1052,7 @@
SVN_ERR (get_repos (&repos, path,
APR_FLOCK_EXCLUSIVE,
FALSE, /* don't try to open the db yet. */
+ NULL, /* No special configuration for recovery. */
subpool));
/* Recover the database to a consistent state. */
Index: subversion/svnserve/serve.c
===================================================================
--- subversion/svnserve/serve.c (revision 5012)
+++ subversion/svnserve/serve.c (working copy)
@@ -875,7 +875,7 @@
candidate = full_path;
while (1)
{
- err = svn_repos_open(repos, candidate, pool);
+ err = svn_repos_open(repos, candidate, NULL, pool);
if (err == SVN_NO_ERROR)
break;
if (!*candidate || strcmp(candidate, "/") == 0)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@subversion.tigris.org
For additional commands, e-mail: dev-help@subversion.tigris.org
Received on Sat Feb 22 04:49:10 2003