lock in svn_fs__bdb_get_node_revision

From: Bastian Blank <waldi_at_debian.org>
Date: 2004-05-07 21:30:29 CEST

Hi folks

Debian-installer uses a subversion repository and regulary suffers from
deadlocks which blocks any repository access.
The repository is highly frequented and uses svnserve for anonymous
access. Each of that process falls into a select loop in this case.

I simulated the load on my apache and found one of the following
backtraces on every apache process:

#5 0x4026fe45 in __os_sleep_4002 () from /usr/lib/libdb-4.2.so
#6 0x4026ff6f in __os_yield_4002 () from /usr/lib/libdb-4.2.so
#7 0x401d0730 in __db_tas_mutex_lock_4002 () from /usr/lib/libdb-4.2.so
#8 0x4025a181 in __lock_get_4002 () from /usr/lib/libdb-4.2.so
#9 0x4025937f in __lock_vec_4002 () from /usr/lib/libdb-4.2.so
#10 0x4023d0aa in __db_lget_4002 () from /usr/lib/libdb-4.2.so
#11 0x401e10b6 in __bam_search_4002 () from /usr/lib/libdb-4.2.so
#12 0x401d6b30 in __bam_c_rget_4002 () from /usr/lib/libdb-4.2.so
#13 0x401d3e95 in __bam_c_dup_4002 () from /usr/lib/libdb-4.2.so
#14 0x402310b9 in __db_c_get_4002 () from /usr/lib/libdb-4.2.so
#15 0x40238cd1 in __db_get_4002 () from /usr/lib/libdb-4.2.so
#16 0x40238b37 in __db_get_pp_4002 () from /usr/lib/libdb-4.2.so
#17 0x4061ebc0 in svn_fs__bdb_get_node_revision () from /usr/lib/libsvn_fs-1.so.0
#18 0x4062191d in svn_fs__dag_get_fs () from /usr/lib/libsvn_fs-1.so.0
#19 0x40621aae in svn_fs__dag_get_node () from /usr/lib/libsvn_fs-1.so.0
#20 0x40622dcb in svn_fs__dag_txn_root () from /usr/lib/libsvn_fs-1.so.0
#21 0x4062cb35 in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#22 0x4062cf99 in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#23 0x40630555 in svn_fs_make_dir () from /usr/lib/libsvn_fs-1.so.0
#24 0x4062bf6c in svn_fs_list_transactions () from /usr/lib/libsvn_fs-1.so.0
#25 0x4062c0b3 in svn_fs__retry_txn () from /usr/lib/libsvn_fs-1.so.0
#26 0x406306d8 in svn_fs_delete () from /usr/lib/libsvn_fs-1.so.0
#27 0x4060eed7 in svn_repos_replay () from /usr/lib/libsvn_repos-1.so.0
#28 0x4060f1b4 in svn_repos_set_path () from /usr/lib/libsvn_repos-1.so.0
#29 0x405f12c8 in dav_svn__update_report () from /usr/lib/apache2/modules/mod_dav_svn.so
#30 0x405f3612 in dav_svn_checkin () from /usr/lib/apache2/modules/mod_dav_svn.so
#31 0x405ca637 in dav_add_response () from /usr/lib/apache2/modules/mod_dav.so

#5 0x4026fe45 in __os_sleep_4002 () from /usr/lib/libdb-4.2.so
#6 0x4026ff6f in __os_yield_4002 () from /usr/lib/libdb-4.2.so
#7 0x401d0730 in __db_tas_mutex_lock_4002 () from /usr/lib/libdb-4.2.so
#8 0x4025a181 in __lock_get_4002 () from /usr/lib/libdb-4.2.so
#9 0x4025937f in __lock_vec_4002 () from /usr/lib/libdb-4.2.so
#10 0x4023d0aa in __db_lget_4002 () from /usr/lib/libdb-4.2.so
#11 0x401e10b6 in __bam_search_4002 () from /usr/lib/libdb-4.2.so
#12 0x401d6b30 in __bam_c_rget_4002 () from /usr/lib/libdb-4.2.so
#13 0x401d3e95 in __bam_c_dup_4002 () from /usr/lib/libdb-4.2.so
#14 0x402310b9 in __db_c_get_4002 () from /usr/lib/libdb-4.2.so
#15 0x40238cd1 in __db_get_4002 () from /usr/lib/libdb-4.2.so
#16 0x40238b37 in __db_get_pp_4002 () from /usr/lib/libdb-4.2.so
#17 0x4061ebc0 in svn_fs__bdb_get_node_revision () from /usr/lib/libsvn_fs-1.so.0
#18 0x4062191d in svn_fs__dag_get_fs () from /usr/lib/libsvn_fs-1.so.0
#19 0x40621aae in svn_fs__dag_get_node () from /usr/lib/libsvn_fs-1.so.0
#20 0x4062370b in svn_fs__dag_remove_node () from /usr/lib/libsvn_fs-1.so.0
#21 0x4062b62b in svn_fs_open_txn () from /usr/lib/libsvn_fs-1.so.0
#22 0x4062bf6c in svn_fs_list_transactions () from /usr/lib/libsvn_fs-1.so.0
#23 0x4062c0b3 in svn_fs__retry_txn () from /usr/lib/libsvn_fs-1.so.0
#24 0x4062b7db in svn_fs_open_txn () from /usr/lib/libsvn_fs-1.so.0
#25 0x4062b780 in svn_fs_open_txn () from /usr/lib/libsvn_fs-1.so.0
#26 0x4062b8b2 in svn_fs_purge_txn () from /usr/lib/libsvn_fs-1.so.0
#27 0x4062bad5 in svn_fs_abort_txn () from /usr/lib/libsvn_fs-1.so.0
#28 0x4060f8fb in svn_repos_abort_report () from /usr/lib/libsvn_repos-1.so.0
#29 0x4060f897 in svn_repos_finish_report () from /usr/lib/libsvn_repos-1.so.0
#30 0x405f1487 in dav_svn__update_report () from /usr/lib/apache2/modules/mod_dav_svn.so
#31 0x405f3612 in dav_svn_checkin () from /usr/lib/apache2/modules/mod_dav_svn.so
#32 0x405ca637 in dav_add_response () from /usr/lib/apache2/modules/mod_dav.so

#5 0x4026fe45 in __os_sleep_4002 () from /usr/lib/libdb-4.2.so
#6 0x4026ff6f in __os_yield_4002 () from /usr/lib/libdb-4.2.so
#7 0x401d0730 in __db_tas_mutex_lock_4002 () from /usr/lib/libdb-4.2.so
#8 0x4025a181 in __lock_get_4002 () from /usr/lib/libdb-4.2.so
#9 0x4025937f in __lock_vec_4002 () from /usr/lib/libdb-4.2.so
#10 0x4023d0aa in __db_lget_4002 () from /usr/lib/libdb-4.2.so
#11 0x401e10b6 in __bam_search_4002 () from /usr/lib/libdb-4.2.so
#12 0x401d6b30 in __bam_c_rget_4002 () from /usr/lib/libdb-4.2.so
#13 0x401d3e95 in __bam_c_dup_4002 () from /usr/lib/libdb-4.2.so
#14 0x402310b9 in __db_c_get_4002 () from /usr/lib/libdb-4.2.so
#15 0x40238cd1 in __db_get_4002 () from /usr/lib/libdb-4.2.so
#16 0x40238b37 in __db_get_pp_4002 () from /usr/lib/libdb-4.2.so
#17 0x4061ebc0 in svn_fs__bdb_get_node_revision () from /usr/lib/libsvn_fs-1.so.0
#18 0x4061e9fc in svn_fs__bdb_new_successor_id () from /usr/lib/libsvn_fs-1.so.0
#19 0x406276c7 in svn_fs__create_successor () from /usr/lib/libsvn_fs-1.so.0
#20 0x40623199 in svn_fs__dag_clone_root () from /usr/lib/libsvn_fs-1.so.0
#21 0x4062cb99 in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#22 0x4062d4f7 in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#23 0x4062d33f in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#24 0x4062d33f in svn_fs_revision_root_revision () from /usr/lib/libsvn_fs-1.so.0
#25 0x406305d1 in svn_fs_make_dir () from /usr/lib/libsvn_fs-1.so.0
#26 0x4062bf6c in svn_fs_list_transactions () from /usr/lib/libsvn_fs-1.so.0
#27 0x4062c0b3 in svn_fs__retry_txn () from /usr/lib/libsvn_fs-1.so.0
#28 0x406306d8 in svn_fs_delete () from /usr/lib/libsvn_fs-1.so.0
#29 0x4060eed7 in svn_repos_replay () from /usr/lib/libsvn_repos-1.so.0
#30 0x4060f1b4 in svn_repos_set_path () from /usr/lib/libsvn_repos-1.so.0
#31 0x405f12c8 in dav_svn__update_report () from /usr/lib/apache2/modules/mod_dav_svn.so
#32 0x405f3612 in dav_svn_checkin () from /usr/lib/apache2/modules/mod_dav_svn.so
#33 0x405ca637 in dav_add_response () from /usr/lib/apache2/modules/mod_dav.so

#5 0x4026fe45 in __os_sleep_4002 () from /usr/lib/libdb-4.2.so
#6 0x4026ff6f in __os_yield_4002 () from /usr/lib/libdb-4.2.so
#7 0x401d0730 in __db_tas_mutex_lock_4002 () from /usr/lib/libdb-4.2.so
#8 0x4025a181 in __lock_get_4002 () from /usr/lib/libdb-4.2.so
#9 0x4025937f in __lock_vec_4002 () from /usr/lib/libdb-4.2.so
#10 0x4023d0aa in __db_lget_4002 () from /usr/lib/libdb-4.2.so
#11 0x401e10b6 in __bam_search_4002 () from /usr/lib/libdb-4.2.so
#12 0x401d6b30 in __bam_c_rget_4002 () from /usr/lib/libdb-4.2.so
#13 0x401d3e95 in __bam_c_dup_4002 () from /usr/lib/libdb-4.2.so
#14 0x402310b9 in __db_c_get_4002 () from /usr/lib/libdb-4.2.so
#15 0x40238cd1 in __db_get_4002 () from /usr/lib/libdb-4.2.so
#16 0x40238b37 in __db_get_pp_4002 () from /usr/lib/libdb-4.2.so
#17 0x4061ebc0 in svn_fs__bdb_get_node_revision () from /usr/lib/libsvn_fs-1.so.0
#18 0x4062191d in svn_fs__dag_get_fs () from /usr/lib/libsvn_fs-1.so.0
#19 0x40621aae in svn_fs__dag_get_node () from /usr/lib/libsvn_fs-1.so.0
#20 0x40622d55 in svn_fs__dag_revision_root () from /usr/lib/libsvn_fs-1.so.0
#21 0x4062c734 in svn_fs_txn_root () from /usr/lib/libsvn_fs-1.so.0
#22 0x4062bf6c in svn_fs_list_transactions () from /usr/lib/libsvn_fs-1.so.0
#23 0x4062c0b3 in svn_fs__retry_txn () from /usr/lib/libsvn_fs-1.so.0
#24 0x4062c7e8 in svn_fs_revision_root () from /usr/lib/libsvn_fs-1.so.0
#25 0x405f1a37 in dav_svn_get_safe_cr () from /usr/lib/apache2/modules/mod_dav_svn.so
#26 0x405ef395 in dav_svn_create_version_resource () from /usr/lib/apache2/modules/mod_dav_svn.so
#27 0x405ef8a0 in dav_svn_create_version_resource () from /usr/lib/apache2/modules/mod_dav_svn.so
#28 0x405f045a in dav_svn_create_version_resource () from /usr/lib/apache2/modules/mod_dav_svn.so
#29 0x4060804a in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#30 0x406088fd in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#31 0x40607fd2 in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#32 0x406088fd in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#33 0x40607fd2 in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#34 0x406088fd in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#35 0x406072e9 in svn_repos_dir_delta () from /usr/lib/libsvn_repos-1.so.0
#36 0x4060f83d in svn_repos_delete_path () from /usr/lib/libsvn_repos-1.so.0
#37 0x4060f882 in svn_repos_finish_report () from /usr/lib/libsvn_repos-1.so.0
#38 0x405f1487 in dav_svn__update_report () from /usr/lib/apache2/modules/mod_dav_svn.so
#39 0x405f3612 in dav_svn_checkin () from /usr/lib/apache2/modules/mod_dav_svn.so
#40 0x405ca637 in dav_add_response () from /usr/lib/apache2/modules/mod_dav.so

I may have missed processes which also access the repository.

A strace on a "svn ls" shows the following:

[...]
stat64("/org/org.eu.waldi/svn/debian.test/db/log.0000000180", {st_mode=S_IFREG|0660, st_size=718384, ...}) = 0
open("/org/org.eu.waldi/svn/debian.test/db/log.0000000180", O_RDWR|O_CREAT|O_LARGEFILE, 0666) = 12
fcntl64(12, F_SETFD, FD_CLOEXEC) = 0
read(12, "1\376\17\0\34\0\0\0$(\21\324\210\t\4\0\10\0\0\0\0\0\20"..., 28) = 28
_llseek(12, 718384, [718384], SEEK_SET) = 0
write(12, "\374\365\n\0R\0\0\0\254\271\237\310\2\0\0\0\347\311\2\200"..., 1002) = 1002
fsync(12) = 0
pwrite(11, "\0\0\0\0\1\0\0\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0\t\0\0"..., 4096, 0) = 4096
pwrite(10, "\264\0\0\0o_\4\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0\t\0\0"..., 4096, 0) = 4096
pwrite(9, "\263\0\0\0\301\300\n\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0"..., 4096, 0) = 4096
pwrite(8, "\264\0\0\0\256\301\6\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0"..., 4096, 0) = 4096
pwrite(7, "\250\0\0\0\262V\1\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0\t"..., 4096, 0) = 4096
pwrite(6, "\263\0\0\0\304\20\17\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0"..., 4096, 0) = 4096
pwrite(5, "\247\0\0\0\250\257\f\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0"..., 4096, 0) = 4096
pwrite(4, "\263\0\0\0\226\"\t\0\0\0\0\0b1\5\0\t\0\0\0\0\20\0\0\0\t"..., 4096, 0) = 4096
fsync(11) = 0
fsync(10) = 0
fsync(9) = 0
fsync(8) = 0
fsync(7) = 0
fsync(6) = 0
fsync(5) = 0
fsync(4) = 0
time(NULL) = 1083949863
_llseek(12, 719386, [719386], SEEK_SET) = 0
write(12, "\362\371\n\0R\0\0\0\265Fwt\2\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 53930) = 53930
fsync(12) = 0
[...]
brk(0) = 0x80c7000
brk(0x80e8000) = 0x80e8000
old_mmap(NULL, 249856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40aed000
old_mmap(NULL, 249856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40b2a000
munmap(0x40b2a000, 249856) = 0
munmap(0x40aed000, 249856) = 0
select(0, NULL, NULL, NULL, {0, 1000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 2000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 4000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 8000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {0, 10000}) = 0 (Timeout)

This does not end until the process gets a SIGKILL.

Bastian

-- 
Beam me up, Scotty, there's no intelligent life down here!

application/pgp-signature attachment: Digital signature

Received on Sat May 8 01:20:41 2004

This message: [ Message body ]
Next message: Karl Robillard: "Re: File not found: transaction 'ct', path 'trunk/Setup'"
Previous message: Henderson, Michael D: "FS Backend Storage"
Next in thread: C. Michael Pilato: "Re: lock in svn_fs__bdb_get_node_revision"
Reply: C. Michael Pilato: "Re: lock in svn_fs__bdb_get_node_revision"

Contemporary messages sorted: [ By Date ] [ By Thread ] [ By Subject ] [ By Author ] [ By messages with attachments ]