Subversion checkout performance to NFS share is very slow

lpwevers · 10-08-2014, 09:03 AM

Dear experts,

Hopefully one (or more) of you is going to be able to help me solve an issue with the performance of subversion. I'm faced with the challenge of making operations like "svn co" perform fast.

For example if I do a checkout to local disk, total execution time is around 17 seconds. Checking out the same repository to an NFS share is about 450 seconds: (Data calculated using strace analyzer http://clusterbuffer.wikifoundry.com...ext+Generation)
Local

Code:

----------------
-- Time Stats --
----------------
Elapsed Time for run: 63.523103 (secs)
Total IO Time: 17.695617 (secs)
Total IO Time Counter: 1115045
   Percentage of Total Time = 27.856978%

Remote

Code:

----------------
-- Time Stats --
----------------
Elapsed Time for run: 442.366395 (secs)
Total IO Time: 368.821888 (secs)
Total IO Time Counter: 1114166
   Percentage of Total Time = 83.374753%

I did play around with various NFS options like rsize, wsize, noatime, noacl and the server side no_subtree_ckeck, and tried using nfs caching (using cachefilesd) but all to no avail. All times with NFS remain well above 400 seconds.

Looking in the strace output I find loads and loads of activity on wc.db-journal so I expect the actual culprit is SQLite, but that's just a guess on my side.

Code:

1412764425.197310 stat("/tmp/v3.1.8/.svn/wc.db", {st_mode=S_IFREG|0644, st_size=34816, ...}) = 0 <0.000014>
1412764425.197369 open("/tmp/v3.1.8/.svn/wc.db-journal", O_RDWR|O_CREAT|O_CLOEXEC, 0644) = 7 <0.000065>
1412764425.197462 fstat(7, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000008>
1412764425.197499 geteuid()             = 46457 <0.000007>
1412764425.197528 lseek(7, 0, SEEK_SET) = 0 <0.000007>
1412764425.197555 write(7, "\331\325\5\371 \241c\327\377\377\377\377\2035\0262\0\0\0\"\0\0\2\0\0\0\4\0\0\0\0\0"..., 512) = 512 <0.000018>
1412764425.197602 lseek(7, 512, SEEK_SET) = 512 <0.000007>
1412764425.197628 write(7, "\0\0\0\31", 4) = 4 <0.000011>
1412764425.197660 lseek(7, 516, SEEK_SET) = 516 <0.000007>
1412764425.197687 write(7, "\n\0\0\0\3\3\316\0\3\372\3\353\3\316\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000020>
1412764425.197739 lseek(7, 1540, SEEK_SET) = 1540 <0.000007>
1412764425.197765 write(7, "\2035\0262", 4) = 4 <0.000010>
1412764425.197804 lseek(7, 1544, SEEK_SET) = 1544 <0.000007>
1412764425.197831 write(7, "\0\0\0\33", 4) = 4 <0.000010>
1412764425.197863 lseek(7, 1548, SEEK_SET) = 1548 <0.000007>
1412764425.197889 write(7, "\n\0\0\0\3\3\303\0\3\371\3\351\3\303\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000010>
1412764425.197926 lseek(7, 2572, SEEK_SET) = 2572 <0.000007>
1412764425.197951 write(7, "\2035\0262", 4) = 4 <0.000010>
1412764425.197988 lseek(7, 2576, SEEK_SET) = 2576 <0.000007>
1412764425.198015 write(7, "\0\0\0\34", 4) = 4 <0.000010>
1412764425.198047 lseek(7, 2580, SEEK_SET) = 2580 <0.000007>
1412764425.198073 write(7, "\n\0\0\0\3\3\354\0\3\372\3\363\3\354\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000010>
1412764425.198110 lseek(7, 3604, SEEK_SET) = 3604 <0.000006>
1412764425.198151 write(7, "\2035\0262", 4) = 4 <0.000012>
1412764425.198196 lseek(7, 3608, SEEK_SET) = 3608 <0.000007>
1412764425.198223 write(7, "\0\0\0\30", 4) = 4 <0.000010>
1412764425.198256 lseek(7, 3612, SEEK_SET) = 3612 <0.000007>
1412764425.198282 write(7, "\r\0\0\0\3\2\372\0\3\300\3o\2\372\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000016>
1412764425.198325 lseek(7, 4636, SEEK_SET) = 4636 <0.000007>
1412764425.198351 write(7, "\2035\26\227", 4) = 4 <0.000010>
1412764425.198404 fcntl(3, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0 <0.000010>
1412764425.198439 fcntl(3, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0 <0.000009>
1412764425.198472 lseek(7, 4640, SEEK_SET) = 4640 <0.000007>
1412764425.198498 write(7, "\0\0\0\1", 4) = 4 <0.000011>
1412764425.198530 lseek(7, 4644, SEEK_SET) = 4644 <0.000007>
1412764425.198556 write(7, "SQLite format 3\0\4\0\1\1\0@  \0\0\0\5\0\0\0\""..., 1024) = 1024 <0.000010>
1412764425.198592 lseek(7, 5668, SEEK_SET) = 5668 <0.000007>
1412764425.198626 write(7, "\2035\26\316", 4) = 4 <0.000020>
1412764425.198681 lseek(3, 0, SEEK_SET) = 0 <0.000007>
1412764425.198727 write(3, "SQLite format 3\0\4\0\1\1\0@  \0\0\0\6\0\0\0\""..., 1024) = 1024 <0.000022>
1412764425.198787 lseek(3, 23552, SEEK_SET) = 23552 <0.000009>
1412764425.198831 write(3, "\r\0\0\0\4\2b\0\3\300\3o\2\372\2b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000022>
1412764425.198911 lseek(3, 24576, SEEK_SET) = 24576 <0.000015>
1412764425.198968 write(3, "\n\0\0\0\4\3\247\0\3\372\3\353\3\316\3\247\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000022>
1412764425.199044 lseek(3, 26624, SEEK_SET) = 26624 <0.000014>
1412764425.199092 write(3, "\n\0\0\0\4\3\205\0\3\371\3\351\3\303\3\205\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000010>
1412764425.199132 lseek(3, 27648, SEEK_SET) = 27648 <0.000008>
1412764425.199176 write(3, "\n\0\0\0\4\3\345\0\3\372\3\363\3\354\3\345\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024 <0.000011>
1412764425.199217 close(7)              = 0 <0.000009>
1412764425.199247 unlink("/tmp/v3.1.8/.svn/wc.db-journal") = 0 <0.000029>
1412764425.199303 fcntl(3, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0 <0.000009>
1412764425.199336 fcntl(3, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=2}) = 0 <0.000008>
1412764425.199369 fcntl(3, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0 <0.000010>

Does anyone have an idea what else I can do to increase the performance? I'm kinda stuck here.

Many thanks in advance for any advice.

Keith Hedger · 10-10-2014, 06:10 PM

checking out to a mounted share is always going to be slower than a local disk because you have to download the file which takes time and network acces then the copied file is sent over the network to the nfs share which takes time and network access, there is a similar thread to this but using samba, the only way to get local speeds when downloading to a share ist ssh into the machine that has the drive connected loccaly and do the download from there, that way the file access is all local, much faster

lpwevers · 10-13-2014, 12:41 AM

Quote:

Originally Posted by Keith Hedger

checking out to a mounted share is always going to be slower than a local disk because you have to download the file which takes time and network acces then the copied file is sent over the network to the nfs share which takes time and network access, there is a similar thread to this but using samba, the only way to get local speeds when downloading to a share ist ssh into the machine that has the drive connected loccaly and do the download from there, that way the file access is all local, much faster

I accecpt the fact that checking out to NFS is slower then doing a local checkout. However, the difference here is way too big for me here. Besides, if I copy the locally checked out version to the NFS share, that takes just a few secs.

Also the machine that I do the SVN checkout on, is actually a VM, that has it's 'local' disk on an NFS share. So, even if we're doing the checkout to local disk, we are in fact still checking out to NFS. It's just that the kernel handles it as if it were a locally attached disk. So it must be something in the way NFS works.

Keith Hedger · 10-13-2014, 03:16 AM

Quote:

Originally Posted by lpwevers

... Also the machine that I do the SVN checkout on, is actually a VM, that has it's 'local' disk on an NFS share. So, even if we're doing ...

I think there is your problem a vm is never going to be as fast as the real machine it is being hosted on, not by a long shot, it's very little to do with nfs.

lpwevers · 10-14-2014, 01:11 AM

Quote:

Originally Posted by Keith Hedger

I think there is your problem a vm is never going to be as fast as the real machine it is being hosted on, not by a long shot, it's very little to do with nfs.

Well, I don't agree with you in this case. If I do the checkout on this VM and let it extract to local storage (which in fact is NFS) then it takes about 63 seconds to complete the run. Then, on the same VM, I do the same checkout but now to an NFS share, it takes around 442 seconds. So there must be something in the way NFS is handled.

talishka · 04-06-2016, 08:36 AM

I'm facing the same problem.. it's too slow. No news or ideas about this?