findutils-patches
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Findutils-patches] [PATCH 1/4] Reduce memory consumption of fts.


From: James Youngman
Subject: [Findutils-patches] [PATCH 1/4] Reduce memory consumption of fts.
Date: Sun, 21 Aug 2011 00:10:17 +0100

* gnulib: update to latest, mainly to include a bugfix (gnulib
commit 47cb657eca1abf2c26c32c8ce03def994a3ee37c) which limits the
memory consumed by fts.
* find/testsuite/sv-34079.sh: New test, verifying that memory
growth in ftsfind does not grow indefinitely as the size of
directories increases.
* find/testsuite/Makefile.am (test_shell_progs): Added sv-34079.sh.
---
 ChangeLog                  |   11 ++++++
 find/testsuite/Makefile.am |    2 +-
 find/testsuite/sv-34079.sh |   78 ++++++++++++++++++++++++++++++++++++++++++++
 gnulib                     |    2 +-
 4 files changed, 91 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 412d219..da5ce0b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2011-08-20  James Youngman  <address@hidden>
+
+       Reduce memory consumption of fts.
+       * gnulib: update to latest, mainly to include a bugfix (gnulib
+       commit 47cb657eca1abf2c26c32c8ce03def994a3ee37c) which limits the
+       memory consumed by fts.
+       * find/testsuite/sv-34079.sh: New test, verifying that memory
+       growth in ftsfind does not grow indefinitely as the size of
+       directories increases.
+       * find/testsuite/Makefile.am (test_shell_progs): Added sv-34079.sh.
+
 2011-06-29  James Youngman  <address@hidden>
 
        Fix some constness warnings when dealing with -printf formats.
diff --git a/find/testsuite/Makefile.am b/find/testsuite/Makefile.am
index b461fb6..a1e49b8 100644
--- a/find/testsuite/Makefile.am
+++ b/find/testsuite/Makefile.am
@@ -247,7 +247,7 @@ EXTRA_DIST_GOLDEN = \
        test_escapechars.golden
 
 test_shell_progs = sv-bug-32043.sh test_escapechars.sh test_escape_c.sh \
-       test_inode.sh
+       test_inode.sh sv-34079.sh
 
 EXTRA_DIST = $(EXTRA_DIST_EXP) $(EXTRA_DIST_XO) $(EXTRA_DIST_GOLDEN) \
        $(test_shell_progs) binary_locations.sh
diff --git a/find/testsuite/sv-34079.sh b/find/testsuite/sv-34079.sh
new file mode 100755
index 0000000..ea6aea6
--- /dev/null
+++ b/find/testsuite/sv-34079.sh
@@ -0,0 +1,78 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# This test verifies that find does not have excessive memory consumption
+# even for large directories.   It's not executed by default; it will only
+# run if the environment variable RUN_VERY_EXPENSIVE_TESTS is set.
+
+testname="$(basename $0)"
+
+. "${srcdir}"/binary_locations.sh
+
+make_test_data() {
+    d="$1"
+    (
+       cd "$1" && echo "Creating test data in $(pwd -P)" >&2 || exit 1
+       max=400
+       for i in $(seq 0 400)
+       do
+           printf "\r%03d/%03d" $i $max >&2
+           for j in $(seq 0 10000)
+           do
+               printf "%03d_%04d " $i $j
+           done
+       done | xargs sh -c 'touch "$@" || exit 255' fnord || exit 1
+       printf "\rTest files created.\n" >&2
+    )
+}
+
+
+if [[ -n "${RUN_VERY_EXPENSIVE_TESTS}" ]]; then
+    if outdir=$(mktemp -d); then
+       # Create some test files.
+       bad=""
+       printf "Generating test data in %s (this may take some time...):\n" \
+           "${outdir}" >&2
+       if make_test_data "${outdir}"; then
+           # We don't check oldfind, as it uses savedir, meaning that
+           # it stores all the directory entries.  Hence the excessive
+           # memory consumption bug applies to oldfind even though it is
+           # not using fts.
+           exe="${ftsfind}"
+           echo "Checking memory consumption of ${exe}..." >&2
+           if ( ulimit -v 50000 && ${exe} "${outdir}" >/dev/null; ); then
+               echo "Memory consumption of ${exe} is reasonable" >&2
+           else
+               bad="${bad}${bad:+\n}Memory consumption of ${exe} is too high"
+           fi
+       else
+           bad="failed to set up the test in ${outdir}"
+       fi
+       rm -rf "${outdir}" || exit 1
+       if [[ -n "${bad}" ]]; then
+           echo "${bad}" >&2
+           exit 1
+       fi
+    else
+       echo "FAIL: could not create a test output file." >&2
+       exit 1
+    fi
+else
+    echo "${testname} was not run because" '${RUN_VERY_EXPENSIVE_TESTS}' \
+       "is unset."
+fi
+
diff --git a/gnulib b/gnulib
index 414111a..372ef2a 160000
--- a/gnulib
+++ b/gnulib
@@ -1 +1 @@
-Subproject commit 414111a8d8020459f65506d4c229d4071ad9d51a
+Subproject commit 372ef2a0e94ec6ee85b5fc4bab763154ec11420d
-- 
1.7.2.5




reply via email to

[Prev in Thread] Current Thread [Next in Thread]