[Gluster-devel] Speeding up file creation

[Erik Osterman]

I've configured a cluster with replication that uses most of the 
advanced features you've implemented including io-threads, afr, 
readahead, and writebehind. I am very satisfied with the write 
performance, but the file creation performance leaves much to be 
desired. What can we do to speed this up?

Creating 100 empty files

# time for i in {1..100}; do touch test.$i;done
real    0m46.913s
user    0m0.023s
sys     0m0.067s

That's about 0.500 seconds just to create an empty file.


In general, what do you advise for tuning the performance of 
reading/writing tons of tiny files. Can the client use io-threads to 
improve performance? Right now, our application stuffs all the tiny 
files in a single directory. Eventually, we were planning on hashing 
them out to directories. Would hashing them out into multiple 
directories positively and significantly affect the performance of 
GlusterFS?



Best,

Erik Osterman





For what it's worth, here are my configurations:

#
# Master
#
volume posix0
 type storage/posix                   # POSIX FS translator
 option directory /home/glusterfs        # Export this directory
end-volume

volume brick0
 type performance/io-threads
 option thread-count 8
 option queue-limit 1024
 subvolumes posix0
end-volume

### Add network serving capability to above brick.
volume server
 type protocol/server
 option transport-type tcp/server     # For TCP/IP transport
# option bind-address 192.168.1.10     # Default is to listen on all 
interfaces
 option listen-port 6996               # Default is 6996
 option client-volume-filename /etc/glusterfs/client.vol
 subvolumes brick0
 option auth.ip.brick0.allow *         # access to "brick" volume
end-volume



#
# Mirror
#
volume posix0
 type storage/posix                   # POSIX FS translator
 option directory /home/glusterfs     # Export this directory
end-volume

volume mirror0
 type performance/io-threads
 option thread-count 8
 option queue-limit 1024
 subvolumes posix0
end-volume

### Add network serving capability to above brick.
volume server
 type protocol/server
 option transport-type tcp/server     # For TCP/IP transport
# option bind-address 192.168.1.11     # Default is to listen on all 
interfaces
 option listen-port 6996               # Default is 6996
 option client-volume-filename /etc/glusterfs/client.vol
 subvolumes mirror0
 option auth.ip.mirror0.allow *         # access to "brick" volume
end-volume


#
# Client
#

### Add client feature and attach to remote subvolume of server
volume brick0
 type protocol/client
 option transport-type tcp/client     # for TCP/IP transport
 option remote-host 216.182.237.155   # IP address of the remote brick 
server
 option remote-port 6996              # default server port is 6996
 option remote-subvolume brick0        # name of the remote volume
end-volume

### Add client feature and attach to remote mirror of brick0
volume mirror0
 type protocol/client
 option transport-type tcp/client     # for TCP/IP transport
 option remote-host 216.55.170.26      # IP address of the remote 
mirror server
 option remote-port 6996              # default server port is 6996
 option remote-subvolume mirror0        # name of the remote volume
end-volume

### Add AFR feature to brick
volume afr0
 type cluster/afr
 subvolumes brick0 mirror0
 option replicate *:2                 # All files 2 copies (RAID-1)
end-volume

### Add unify feature to cluster the servers. Associate an
### appropriate scheduler that matches your I/O demand.
volume bricks
 type cluster/unify
 subvolumes afr0
### ** Round Robin (RR) Scheduler **
 option scheduler rr
 option rr.limits.min-free-disk 2GB
end-volume

### Add performance feature
volume writebehind
 type performance/write-behind
 option aggregate-size 131072 # aggregate block size in bytes
 subvolumes bricks
end-volume

### Add performance feature
volume readahead
 type performance/read-ahead
 option page-size 131072
 option page-count 16
 subvolumes writebehind
end-volume