Index: ./libsvn_fs/strings-table.c =================================================================== --- ./libsvn_fs/strings-table.c +++ ./libsvn_fs/strings-table.c Wed Feb 27 08:39:10 2002 @@ -36,6 +36,11 @@ DB *strings; DB_ERR (db_create (&strings, env, 0)); + + /* Enable duplicate keys. This allows the data to be spread out across + multiple records. Note: this must occur before ->open(). */ + DB_ERR (strings->set_flags (strings, DB_DUP)); + DB_ERR (strings->open (strings, "strings", 0, DB_BTREE, create ? (DB_CREATE | DB_EXCL) : 0, 0666)); @@ -60,6 +65,101 @@ /*** Storing and retrieving strings. ***/ +static svn_error_t * +locate_key (apr_size_t *length, + DBC **cursor, + DBT *query, + svn_fs_t *fs, + trail_t *trail) +{ + int db_err; + DBT result; + + SVN_ERR (DB_WRAP (fs, "creating cursor for reading a string", + fs->strings->cursor (fs->strings, trail->db_txn, + cursor, 0))); + + /* Set up the DBT for reading the length of the record. */ + svn_fs__clear_dbt (&result); + result.ulen = 0; + result.flags |= DB_DBT_USERMEM; + + /* Advance the cursor to the key that we're looking for. */ + db_err = (*cursor)->c_get (*cursor, query, &result, DB_SET); + + /* We don't need to svn_fs__track_dbt() the result, because nothing + was allocated in it. */ + + /* If there's no such node, return an appropriately specific error. */ + if (db_err == DB_NOTFOUND) + { + (*cursor)->c_close (*cursor); + return svn_error_createf + (SVN_ERR_FS_NO_SUCH_STRING, 0, 0, fs->pool, + "locate_key: no such string `%s'", query->data); + } + if (db_err) + { + DBT rerun; + + if (db_err != ENOMEM) + { + (*cursor)->c_close (*cursor); + return DB_WRAP (fs, "could not move cursor", db_err); + } + + /* We got an ENOMEM (typical since we have a zero length buf), so + we need to re-run the operation to make it happen. */ + svn_fs__clear_dbt (&rerun); + rerun.flags |= DB_DBT_USERMEM | DB_DBT_PARTIAL; + SVN_ERR (DB_WRAP (fs, "rerunning cursor move", + (*cursor)->c_get (*cursor, query, &rerun, DB_SET))); + } + + /* ### this cast might not be safe? */ + *length = (apr_size_t) result.size; + + return SVN_NO_ERROR; +} + +static int +get_next_length (apr_size_t *length, DBC *cursor, DBT *query) +{ + DBT result; + int db_err; + + /* Set up the DBT for reading the length of the record. */ + svn_fs__clear_dbt (&result); + result.ulen = 0; + result.flags |= DB_DBT_USERMEM; + + /* Note: this may change the QUERY DBT, but that's okay: we're going + to be sticking with the same key anyways. */ + db_err = cursor->c_get (cursor, query, &result, DB_NEXT_DUP); + + /* Note that we exit on DB_NOTFOUND. The caller uses that to end a loop. */ + if (db_err) + { + DBT rerun; + + if (db_err != ENOMEM) + { + cursor->c_close (cursor); + return db_err; + } + + /* We got an ENOMEM (typical since we have a zero length buf), so + we need to re-run the operation to make it happen. */ + svn_fs__clear_dbt (&rerun); + rerun.flags |= DB_DBT_USERMEM | DB_DBT_PARTIAL; + db_err = cursor->c_get (cursor, query, &rerun, DB_NEXT_DUP); + } + + /* ### this cast might not be safe? */ + *length = (apr_size_t) result.size; + return db_err; +} + svn_error_t * svn_fs__string_read (svn_fs_t *fs, const char *key, @@ -70,53 +170,112 @@ { int db_err; DBT query, result; + DBC *cursor; + apr_size_t length; + + svn_fs__str_to_dbt (&query, (char *) key); + + SVN_ERR (locate_key (&length, &cursor, &query, fs, trail)); + + /* Seek through the records for this key, trying to find the record that + includes OFFSET. Note that we don't require reading from more than + one record since we're allowed to return partial reads. */ + while (length <= offset) + { + offset -= length; + db_err = get_next_length (&length, cursor, &query); + + /* No more records? They tried to read past the end. */ + if (db_err == DB_NOTFOUND) + { + *len = 0; + return SVN_NO_ERROR; + } + if (db_err) + return DB_WRAP (fs, "reading string", db_err); + } + + /* The current record contains OFFSET. Fetch the contents now. Note that + OFFSET has been moved to be relative to this record. The length could + quite easily extend past this record, but no big deal. We also keep + the DB_DBT_PARTIAL to read little pieces at this location. */ svn_fs__clear_dbt (&result); result.data = buf; result.ulen = *len; result.doff = offset; result.dlen = *len; result.flags |= (DB_DBT_USERMEM | DB_DBT_PARTIAL); + db_err = cursor->c_get (cursor, &query, &result, DB_CURRENT); + if (db_err) + goto cursor_error; + + /* Done with the cursor. */ + SVN_ERR (DB_WRAP (fs, "closing string-reading cursor", + cursor->c_close (cursor))); - db_err = fs->strings->get (fs->strings, trail->db_txn, - svn_fs__str_to_dbt (&query, (char *) key), - &result, 0); + *len = result.size; + return SVN_NO_ERROR; - /* If there's no such node, return an appropriately specific error. */ - if (db_err == DB_NOTFOUND) - return svn_error_createf - (SVN_ERR_FS_NO_SUCH_STRING, 0, 0, fs->pool, - "svn_fs__string_read: no such string `%s'", key); +cursor_error: + /* An error occurred somewhere. Close the cursor and return the error. */ + cursor->c_close (cursor); + return DB_WRAP (fs, "reading string", db_err); +} - /* Handle any other error conditions. */ - SVN_ERR (DB_WRAP (fs, "reading string", db_err)); - { - /* ### ugly hack!! +/* Get the current 'next-key' value and bump the record. */ +static svn_error_t * +get_key_and_bump (svn_fs_t *fs, const char **key, trail_t *trail) +{ + DBC *cursor; + char next_key[200]; + apr_size_t key_len; + int db_err; + DBT query; + DBT result; - Thanks to what is believed to be a bug in Berkeley DB 3.2.9, - reading off the end of this end (if it is stored in a - B_OVERFLOW page, as BDB defines it) will not return the proper - value in results.size, so for now we will hack around that by - calculating the legal sizes for ourselves. - - This section of code *should* just be: - - *len = result.size; - - and is certainly intended to someday return to that. */ - apr_size_t size; - - SVN_ERR (svn_fs__string_size (&size, fs, key, trail)); - if (offset >= size) - *len = 0; - else if (*len > (size - offset)) - *len = size - offset; - } + /* ### todo: see issue #409 for why bumping the key as part of this + trail is problematic. */ - return SVN_NO_ERROR; -} + /* Open a cursor and move it to the 'next-key' value. We can then fetch + the contents and use the cursor to overwrite those contents. Since + this database allows duplicates, we can't do an arbitrary 'put' to + write the new value -- that would append, not overwrite. */ + + SVN_ERR (DB_WRAP (fs, "creating cursor for reading a string", + fs->strings->cursor (fs->strings, trail->db_txn, + &cursor, 0))); + + /* Advance the cursor to 'next-key' and read it. */ + + db_err = cursor->c_get (cursor, + svn_fs__str_to_dbt (&query, + (char *) svn_fs__next_key_key), + svn_fs__result_dbt (&result), + DB_SET); + if (db_err) + { + cursor->c_close (cursor); + return DB_WRAP (fs, "getting next-key value", db_err); + } + + svn_fs__track_dbt (&result, trail->pool); + *key = apr_pstrndup (trail->pool, result.data, result.size); + + /* Bump to future key. */ + key_len = result.size; + svn_fs__next_key (result.data, &key_len, next_key); + + /* Shove the new key back into the database, at the cursor position. */ + db_err = cursor->c_put (cursor, &query, + svn_fs__str_to_dbt (&result, (char *) next_key), + DB_CURRENT); + cursor->c_close (cursor); + + return DB_WRAP (fs, "bumping next string key", db_err); +} svn_error_t * svn_fs__string_append (svn_fs_t *fs, @@ -126,61 +285,21 @@ trail_t *trail) { DBT query, result; - apr_size_t offset = 0; - int db_err; /* If the passed-in key is NULL, we graciously generate a new string using the value of the `next-key' record in the strings table. */ if (*key == NULL) { - char next_key[200]; - apr_size_t key_len; - - /* ### todo: see issue #409 for why bumping the key as part of this - trail is problematic. */ - - /* Get the current value associated with `next-key'. */ - svn_fs__str_to_dbt (&query, (char *) svn_fs__next_key_key); - SVN_ERR (DB_WRAP (fs, "allocating new string (getting next-key)", - fs->strings->get (fs->strings, - trail->db_txn, - &query, - svn_fs__result_dbt (&result), - 0))); - - svn_fs__track_dbt (&result, trail->pool); - *key = apr_pstrndup (trail->pool, result.data, result.size); - - /* Bump to future key. */ - key_len = result.size; - svn_fs__next_key (result.data, &key_len, next_key); - db_err = fs->strings->put - (fs->strings, trail->db_txn, - svn_fs__str_to_dbt (&query, (char *) svn_fs__next_key_key), - svn_fs__str_to_dbt (&result, (char *) next_key), - 0); - - SVN_ERR (DB_WRAP (fs, "bumping next string key", db_err)); - } - else - { - /* Get the current size of the record. */ - SVN_ERR (svn_fs__string_size (&offset, fs, *key, trail)); + SVN_ERR (get_key_and_bump (fs, key, trail)); } - /* Append to it. */ - svn_fs__clear_dbt (&result); - result.data = (char *) buf; - result.size = len; - result.doff = offset; - result.dlen = len; - result.flags |= (DB_DBT_USERMEM | DB_DBT_PARTIAL); - + /* Store a new record into the database. */ SVN_ERR (DB_WRAP (fs, "appending string", fs->strings->put (fs->strings, trail->db_txn, svn_fs__str_to_dbt (&query, (char *) (*key)), - &result, 0))); + svn_fs__set_dbt (&result, (void *) buf, len), + 0))); return SVN_NO_ERROR; } @@ -194,14 +313,10 @@ int db_err; DBT query, result; - svn_fs__clear_dbt (&result); - result.data = 0; - result.size = 0; - result.flags |= DB_DBT_USERMEM; + svn_fs__str_to_dbt (&query, (char *)key); - db_err = fs->strings->put (fs->strings, trail->db_txn, - svn_fs__str_to_dbt (&query, (char *) key), - &result, 0); + /* Torch the prior contents */ + db_err = fs->strings->del (fs->strings, trail->db_txn, &query, 0); /* If there's no such node, return an appropriately specific error. */ if (db_err == DB_NOTFOUND) @@ -212,7 +327,15 @@ /* Handle any other error conditions. */ SVN_ERR (DB_WRAP (fs, "clearing string", db_err)); - return SVN_NO_ERROR; + /* Shove empty data back in for this key. */ + svn_fs__clear_dbt (&result); + result.data = 0; + result.size = 0; + result.flags |= DB_DBT_USERMEM; + + return DB_WRAP (fs, "storing empty contents", + fs->strings->put (fs->strings, trail->db_txn, + &query, &result, 0)); } @@ -223,32 +346,33 @@ trail_t *trail) { int db_err; - DBT query, result; + DBT query; + DBC *cursor; + apr_size_t length; + apr_size_t total; - svn_fs__clear_dbt (&result); - result.ulen = 0; - result.flags |= DB_DBT_USERMEM; + svn_fs__str_to_dbt (&query, (char *) key); - db_err = fs->strings->get (fs->strings, trail->db_txn, - svn_fs__str_to_dbt (&query, (char *) key), - &result, 0); + SVN_ERR (locate_key (&length, &cursor, &query, fs, trail)); - /* We don't need to svn_fs__track_dbt() the result, because nothing - was allocated in it. */ - - /* If there's no such node, return an appropriately specific error. */ - if (db_err == DB_NOTFOUND) - return svn_error_createf - (SVN_ERR_FS_NO_SUCH_STRING, 0, 0, fs->pool, - "svn_fs__string_size: no such string `%s'", key); + total = length; + while (1) + { + db_err = get_next_length (&length, cursor, &query); - if (db_err && db_err != ENOMEM) - return DB_WRAP (fs, "reading string", db_err); + /* No more records? Then return the total length. */ + if (db_err == DB_NOTFOUND) + { + *size = total; + return SVN_NO_ERROR; + } + if (db_err) + return DB_WRAP (fs, "fetching string length", db_err); - /* kff todo: how can we know this cast is safe? */ - *size = (apr_size_t) result.size; + total += length; + } - return SVN_NO_ERROR; + /* NOTREACHED */ } @@ -282,28 +406,63 @@ const char *key, trail_t *trail) { - apr_size_t size, offset; - char buf[10000]; + DBT query; + DBT result; + DBT copykey; + DBC *cursor; + int db_err; - offset = 0; - *new_key = NULL; - while (1) - { - /* Reset size to maximum chunk size. */ - size = sizeof (buf); + SVN_ERR (get_key_and_bump (fs, new_key, trail)); + + SVN_ERR (DB_WRAP (fs, "creating cursor for reading a string", + fs->strings->cursor (fs->strings, trail->db_txn, + &cursor, 0))); - /* Read SIZE bytes from the record, starting at OFFSET. */ - SVN_ERR (svn_fs__string_read (fs, key, buf, offset, &size, trail)); + svn_fs__str_to_dbt (&query, (char *) key); + svn_fs__str_to_dbt (©key, (char *) *new_key); - /* Now, write SIZE bytes into the new record. */ - SVN_ERR (svn_fs__string_append (fs, new_key, size, buf, trail)); - offset += size; + svn_fs__clear_dbt (&result); + + /* Move to the first record and fetch its data (under BDB's mem mgmt). */ + db_err = cursor->c_get (cursor, &query, &result, DB_SET); + if (db_err) + { + cursor->c_close (cursor); + return DB_WRAP (fs, "getting next-key value", db_err); + } - /* If we didn't get the whole chunk, we must be finished. */ - if (size < sizeof (buf)) + while (1) + { + /* ### can we pass a BDB-provided buffer to another BDB function? + ### they are supposed to have a duration up to certain points + ### of calling back into BDB, but I'm not sure what the exact + ### rules are. it is definitely nicer to use BDB buffers here + ### to simplify things and reduce copies, but... hrm. + */ + + /* Write the data to the database */ + db_err = fs->strings->put (fs->strings, trail->db_txn, + ©key, &result, 0); + if (db_err) + { + cursor->c_close (cursor); + return DB_WRAP (fs, "writing copied data", db_err); + } + + /* Read the next chunk. Terminate loop if we're done. */ + svn_fs__clear_dbt (&result); + db_err = cursor->c_get (cursor, &query, &result, DB_NEXT_DUP); + if (db_err == DB_NOTFOUND) break; + if (db_err) + { + cursor->c_close (cursor); + return DB_WRAP (fs, "fetching string data for a copy", db_err); + } } + cursor->c_close (cursor); + return SVN_NO_ERROR; } Index: ./libsvn_fs/tree.c =================================================================== --- ./libsvn_fs/tree.c +++ ./libsvn_fs/tree.c Wed Feb 27 07:37:57 2002 @@ -2507,7 +2507,7 @@ /* Check to see if we need to purge the portion of the contents that have been written thus far. */ - if ((! window) || (tb->target_string->len > SVN_FS_WRITE_BUFFER_SIZE)) + if ((! window) || (tb->target_string->len > 500000)) { apr_size_t len = tb->target_string->len; svn_stream_write (tb->target_stream, Index: ./tests/libsvn_fs/strings-reps-test.c =================================================================== --- ./tests/libsvn_fs/strings-reps-test.c +++ ./tests/libsvn_fs/strings-reps-test.c Wed Feb 27 07:18:22 2002 @@ -326,25 +326,31 @@ /* Check the string size. */ SVN_ERR (svn_fs__string_size (&size, fs, key, trail)); if (size != expected_len) - return svn_error_create (SVN_ERR_FS_GENERAL, 0, NULL, trail->pool, - "record has unexpected size"); + return svn_error_createf (SVN_ERR_FS_GENERAL, 0, NULL, trail->pool, + "record has unexpected size " + "(got %" APR_SIZE_T_FMT ", " + "expected %" APR_SIZE_T_FMT ")", + size, expected_len); /* Read the string back in 100-byte chunks. */ text = svn_stringbuf_create ("", trail->pool); while (1) { - size = 100; + size = sizeof (buf); SVN_ERR (svn_fs__string_read (fs, key, buf, offset, &size, trail)); - svn_stringbuf_appendbytes (text, buf, size); - if (size < 100) + if (size == 0) break; + svn_stringbuf_appendbytes (text, buf, size); offset += size; } /* Check the size and contents of the read data. */ if (text->len != expected_len) - return svn_error_create (SVN_ERR_FS_GENERAL, 0, NULL, trail->pool, - "record read returned unexpected size"); + return svn_error_createf (SVN_ERR_FS_GENERAL, 0, NULL, trail->pool, + "record read returned unexpected size " + "(got %" APR_SIZE_T_FMT ", " + "expected %" APR_SIZE_T_FMT ")", + size, expected_len); if (memcmp (expected_text, text->data, expected_len)) return svn_error_create (SVN_ERR_FS_GENERAL, 0, NULL, trail->pool, "record read returned unexpected data");