version 8.10

* NEWS: Record release date.

.prev-version

@@ -1 +1 @@
-8.8
+8.9

HACKING

@@ -468,10 +468,8 @@ versions of gcc and the linux kernel, and modern GNU tools.
 Ensure that your changes are indented properly.
 ===============================================
 Format the code the way GNU indent does.
-In a file with the "indent-tabs-mode: nil" directive at the end,
-running "indent --no-tabs" should induce no change.
-With other files, there will be some existing differences.
-Try not to add any more.
+Filtering most source files through "indent --no-tabs" should
+induce no change in indentation.  Try not to add any more.
 
 
 Avoid trailing white space
@@ -484,13 +482,11 @@ Do not add any more trailing blanks anywhere.  While "make syntax-check"
 will alert you if you slip up, it's better to nip any problem in the
 bud, as you're typing.  A good way to help you adapt to this rule is
 to configure your editor to highlight any offending characters in the
-files you edit.  If you use Emacs, customize its font-lock mode (FIXME:
-provide more detail) or try one of its whitespace packages.  This appears
-to be the one that will end up in emacs 23:
+files you edit.  If you use Emacs, customize its font-lock mode
+or use its WhiteSpace mode:
 
     http://www.emacswiki.org/emacs/WhiteSpace
 
-[that page says its version also works with emacs 21 and 22]
 If you use vim, add this to ~/.vimrc:
 
     let c_space_errors=1

NEWS

@@ -1,5 +1,46 @@
 GNU coreutils NEWS                                    -*- outline -*-
 
+* Noteworthy changes in release 8.10 (2011-02-04) [stable]
+
+** Bug fixes
+
+  du would abort with a failed assertion when two conditions are met:
+  part of the hierarchy being traversed is moved to a higher level in the
+  directory tree, and there is at least one more command line directory
+  argument following the one containing the moved sub-tree.
+  [bug introduced in coreutils-5.1.0]
+
+  join --header now skips the ordering check for the first line
+  even if the other file is empty.  [bug introduced in coreutils-8.5]
+
+  rm -f no longer fails for EINVAL or EILSEQ on file systems that
+  reject file names invalid for that file system.
+
+  uniq -f NUM no longer tries to process fields after end of line.
+  [bug introduced in coreutils-7.0]
+
+** New features
+
+  cp now copies sparse files efficiently on file systems with FIEMAP
+  support (ext4, btrfs, xfs, ocfs2).  Before, it had to read 2^20 bytes
+  when copying a 1MiB sparse file.  Now, it copies bytes only for the
+  non-sparse sections of a file.  Similarly, to induce a hole in the
+  output file, it had to detect a long sequence of zero bytes.  Now,
+  it knows precisely where each hole in an input file is, and can
+  reproduce them efficiently in the output file.  mv also benefits
+  when it resorts to copying, e.g., between file systems.
+
+  join now supports -o 'auto' which will automatically infer the
+  output format from the first line in each file, to ensure
+  the same number of fields are output for each line.
+
+** Changes in behavior
+
+  join no longer reports disorder when one of the files is empty.
+  This allows one to use join as a field extractor like:
+  join -a1 -o 1.3,1.1 - /dev/null
+
+
 * Noteworthy changes in release 8.9 (2011-01-04) [stable]
 
 ** Bug fixes

README-release

@@ -29,13 +29,15 @@ FIXME: enable excluded programs like arch? to get their manual pages?
   Run the following on at least one SELinux-enabled (enforcing) and
   one non-SELinux system:
 
-    make distcheck
-    make -j1 check RUN_VERY_EXPENSIVE_TESTS=yes RUN_EXPENSIVE_TESTS=yes
-    sudo env PATH="$PATH" NON_ROOT_USERNAME=$USER make -k check-root
+    n=$(( ($(nproc) + 1) / 2 ))
+    sudo env PATH="$PATH" NON_ROOT_USERNAME=$USER make -k -j$(nproc) check-root\
+      && make distcheck \
+      && make -j$n check RUN_VERY_EXPENSIVE_TESTS=yes RUN_EXPENSIVE_TESTS=yes
 
-  Note the -j1 above.  If you use -jN, for larger N, some of the expensive
-  tests are likely to interfere with concurrent performance-measuring or
-  timing-sensitive tests, resulting in spurious failures.
+  Note that the use of -j$n tells make to use approximately half of the
+  available processing units.  If you use -jN, for larger N, some of the
+  expensive tests are likely to interfere with concurrent performance-measuring
+  or timing-sensitive tests, resulting in spurious failures.
 
   If "make distcheck" doesn't run "make syntax-check" for you, then run
   it manually:

bootstrap.conf

@@ -61,6 +61,7 @@ gnulib_modules="
   dirfd
   dirname
   do-release-commit-and-tag
+  dtoastr
   dup2
   environ
   error
@@ -124,6 +125,7 @@ gnulib_modules="
   isblank
   lchmod
   lchown
+  ldtoastr
   lib-ignore
   linebuffer
   link
@@ -170,6 +172,7 @@ gnulib_modules="
   quotearg
   randint
   randperm
+  read-file
   readlink
   readtokens
   readtokens0

cfg.mk

@@ -27,7 +27,7 @@ bootstrap-tools = autoconf,automake,gnulib,bison
 # Now that we have better tests, make this the default.
 export VERBOSE = yes
 
-old_NEWS_hash = 5147e339b49d47675c34d0362060e973
+old_NEWS_hash = f2ebf9f1f16209f7a4b9927a755956fa
 
 # Add an exemption for sc_makefile_at_at_check.
 _makefile_at_at_check_exceptions = ' && !/^cu_install_program =/'
@@ -80,7 +80,7 @@ sc_root_tests:
 	@if test -d tests \
 	      && grep check-root tests/Makefile.am>/dev/null 2>&1; then \
 	t1=sc-root.expected; t2=sc-root.actual;				\
-	grep -nl '^require_root_$$'					\
+	grep -nl '^ *require_root_$$'					\
 	  $$($(VC_LIST) tests) |sed s,tests/,, |sort > $$t1;		\
 	sed -n '/^root_tests =[	 ]*\\$$/,/[^\]$$/p'			\
 	  $(srcdir)/tests/Makefile.am					\

doc/coreutils.texi

@@ -217,6 +217,7 @@ Common Options
 * Exit status::                  Indicating program success or failure
 * Backup options::               Backup options
 * Block size::                   Block size
+* Floating point::               Floating point number representation
 * Signal specifications::        Specifying signals
 * Disambiguating names and IDs:: chgrp and chown owner and group syntax
 * Random sources::               Sources of random data
@@ -624,7 +625,7 @@ Remove any trailing slashes from each @var{source} argument.
 @macro mayConflictWithShellBuiltIn{cmd}
 @cindex conflicts with shell built-ins
 @cindex built-in shell commands, conflicts with
-Due to shell aliases and built-in @command{\cmd\} command, using an
+Due to shell aliases and built-in @command{\cmd\} functions, using an
 unadorned @command{\cmd\} interactively or in a script may get you
 different functionality than that described here.  Invoke it via
 @command{env} (i.e., @code{env \cmd\ @dots{}}) to avoid interference
@@ -729,6 +730,7 @@ name.
 * Exit status::                 Indicating program success or failure.
 * Backup options::              -b -S, in some programs.
 * Block size::                  BLOCK_SIZE and --block-size, in some programs.
+* Floating point::              Floating point number representation.
 * Signal specifications::       Specifying signals using the --signal option.
 * Disambiguating names and IDs:: chgrp and chown owner and group syntax
 * Random sources::              --random-source, in some programs.
@@ -1011,6 +1013,34 @@ set.  The @option{-h} or @option{--human-readable} option is equivalent to
 @option{--block-size=human-readable}.  The @option{--si} option is
 equivalent to @option{--block-size=si}.
 
+@node Floating point
+@section Floating point numbers
+@cindex floating point
+@cindex IEEE floating point
+
+Commands that accept or produce floating point numbers employ the
+floating point representation of the underlying system, and suffer
+from rounding error, overflow, and similar floating-point issues.
+Almost all modern systems use IEEE-754 floating point, and it is
+typically portable to assume IEEE-754 behavior these days.  IEEE-754
+has positive and negative infinity, distinguishes positive from
+negative zero, and uses special values called NaNs to represent
+invalid computations such as dividing zero by itself.  For more
+information, please see David Goldberg's paper
+@uref{http://@/www.validlab.com/@/goldberg/@/paper.pdf, What Every
+Computer Scientist Should Know About Floating-Point Arithmetic}.
+
+@vindex LC_NUMERIC
+Commands that accept floating point numbers as options, operands or
+input use the standard C functions @code{strtod} and @code{strtold} to
+convert from text to floating point numbers.  These floating point
+numbers therefore can use scientific notation like @code{1.0e-34} and
+@code{-10e100}.  Modern C implementations also accept hexadecimal
+floating point numbers such as @code{-0x.ep-3}, which stands for
+@minus{}14/16 times @math{2^-3}, which equals @minus{}0.109375.  The
+@env{LC_NUMERIC} locale determines the decimal-point character.
+@xref{Parsing of Floats,,, libc, The GNU C Library Reference Manual}.
+
 @node Signal specifications
 @section Signal specifications
 @cindex signals, specifying
@@ -1880,7 +1910,7 @@ named character, ignoring high-order bit
 @item d
 signed decimal
 @item f
-floating point
+floating point (@pxref{Floating point})
 @item o
 octal
 @item u
@@ -2820,8 +2850,7 @@ During one iteration, every specified file is checked to see if it has
 changed size.
 Historical implementations of @command{tail} have required that
 @var{number} be an integer.  However, GNU @command{tail} accepts
-an arbitrary floating point number (using a period before any
-fractional digits).
+an arbitrary floating point number.  @xref{Floating point}.
 When @command{tail} uses inotify, this polling-related option is ignored.
 
 @itemx --pid=@var{pid}
@@ -3883,11 +3912,8 @@ the final result, after the throwing away.))
 @opindex --sort
 @cindex general numeric sort
 @vindex LC_NUMERIC
-Sort numerically, using the standard C function @code{strtold} to convert
-a prefix of each line to a long double-precision floating point number.
-This allows floating point numbers to be specified in scientific notation,
-like @code{1.0e-34} and @code{10e100}.
-The @env{LC_NUMERIC} locale determines the decimal-point character.
+Sort numerically, converting a prefix of each line to a long
+double-precision floating point number.  @xref{Floating point}.
 Do not report overflow, underflow, or conversion errors.
 Use the following collating sequence:
 
@@ -4761,11 +4787,17 @@ If there is an error it exits with nonzero status.
 @macro checkOrderOption{cmd}
 If the @option{--check-order} option is given, unsorted inputs will
 cause a fatal error message.  If the option @option{--nocheck-order}
-is given, unsorted inputs will never cause an error message.  If
-neither of these options is given, wrongly sorted inputs are diagnosed
-only if an input file is found to contain unpairable lines.  If an
-input file is diagnosed as being unsorted, the @command{\cmd\} command
-will exit with a nonzero status (and the output should not be used).
+is given, unsorted inputs will never cause an error message.  If neither
+of these options is given, wrongly sorted inputs are diagnosed
+only if an input file is found to contain unpairable
+@ifset JOIN_COMMAND
+lines, and when both input files are non empty.
+@end ifset
+@ifclear JOIN_COMMAND
+lines.
+@end ifclear
+If an input file is diagnosed as being unsorted, the @command{\cmd\}
+command will exit with a nonzero status (and the output should not be used).
 
 Forcing @command{\cmd\} to process wrongly sorted input files
 containing unpairable lines by specifying @option{--nocheck-order} is
@@ -5474,11 +5506,26 @@ Select for printing only the fields listed in @var{field-list}.
 Fields are separated by a TAB character by default.  Also print any
 line that contains no delimiter character, unless the
 @option{--only-delimited} (@option{-s}) option is specified.
-Note @command{cut} does not support specifying runs of whitespace as a
-delimiter, so to achieve that common functionality one can pre-process
-with @command{tr} like:
+
+Note @command{awk} supports more sophisticated field processing,
+and by default will use (and discard) runs of blank characters to
+separate fields, and ignore leading and trailing blanks.
 @example
-tr -s '[:blank:]' '\t' | cut -f@dots{}
+@verbatim
+awk '{print $2}'    # print the second field
+awk '{print $NF-1}' # print the penultimate field
+awk '{print $2,$1}' # reorder the first two fields
+@end verbatim
+@end example
+
+In the unlikely event that @command{awk} is unavailable,
+one can use the @command{join} command, to process blank
+characters as @command{awk} does above.
+@example
+@verbatim
+join -a1 -o 1.2     - /dev/null # print the second field
+join -a1 -o 1.2,1.1 - /dev/null # reorder the first two fields
+@end verbatim
 @end example
 
 @item -d @var{input_delim_byte}
@@ -5646,7 +5693,9 @@ c c1 c2
 b b1 b2
 @end example
 
+@set JOIN_COMMAND
 @checkOrderOption{join}
+@clear JOIN_COMMAND
 
 The defaults are:
 @itemize
@@ -5675,8 +5724,8 @@ Do not check that both input files are in sorted order.  This is the default.
 
 @item -e @var{string}
 @opindex -e
-Replace those output fields that are missing in the input with
-@var{string}.
+Replace those output fields that are missing in the input with @var{string}.
+I.E. missing fields specified with the @option{-12jo} options.
 
 @item --header
 @opindex --header
@@ -5707,10 +5756,17 @@ Join on field @var{field} (a positive integer) of file 2.
 Equivalent to @option{-1 @var{field} -2 @var{field}}.
 
 @item -o @var{field-list}
-Construct each output line according to the format in @var{field-list}.
-Each element in @var{field-list} is either the single character @samp{0} or
-has the form @var{m.n} where the file number, @var{m}, is @samp{1} or
-@samp{2} and @var{n} is a positive field number.
+@itemx -o auto
+If the keyword @samp{auto} is specified, infer the output format from
+the first line in each file.  This is the same as the default output format
+but also ensures the same number of fields are output for each line.
+Missing fields are replaced with the @option{-e} option and extra fields
+are discarded.
+
+Otherwise, construct each output line according to the format in
+@var{field-list}.  Each element in @var{field-list} is either the single
+character @samp{0} or has the form @var{m.n} where the file number, @var{m},
+is @samp{1} or @samp{2} and @var{n} is a positive field number.
 
 A field specification of @samp{0} denotes the join field.
 In most cases, the functionality of the @samp{0} field spec
@@ -5850,8 +5906,9 @@ Control-I.
 @item \v
 Control-K.
 @item \@var{ooo}
-The character with the value given by @var{ooo}, which is 1 to 3
-octal digits,
+The 8-bit character with the value given by @var{ooo}, which is 1 to 3
+octal digits.  Note that @samp{\400} is interpreted as the two-byte
+sequence, @samp{\040} @samp{0}.
 @item \\
 A backslash.
 @end table
@@ -8891,6 +8948,20 @@ your hard disk, you could give a command like this:
 shred --verbose /dev/sda5
 @end example
 
+On modern disks, a single pass should be adequate,
+and it will take one third the time of the default three-pass approach.
+
+@example
+# 1 pass, write pseudo-random data; 3x faster than the default
+shred --verbose -n1 /dev/sda5
+@end example
+
+To be on the safe side, use at least one pass that overwrites using
+pseudo-random data.  I.e., don't be tempted to use @samp{-n0 --zero},
+in case some disk controller optimizes the process of writing blocks
+of all zeros, and thereby does not clear all bytes in a block.
+Some SSDs may do just that.
+
 A @var{file} of @samp{-} denotes standard output.
 The intended use of this is to shred a removed temporary file.
 For example:
@@ -11080,10 +11151,12 @@ vertical tab
 backslash
 @item \0@var{nnn}
 the eight-bit value that is the octal number @var{nnn}
-(zero to three octal digits)
+(zero to three octal digits), if @var{nnn} is
+a nine-bit value, the ninth bit is ignored
 @item \@var{nnn}
 the eight-bit value that is the octal number @var{nnn}
-(one to three octal digits)
+(one to three octal digits), if @var{nnn} is
+a nine-bit value, the ninth bit is ignored
 @item \x@var{hh}
 the eight-bit value that is the hexadecimal number @var{hh}
 (one or two hexadecimal digits)
@@ -11164,7 +11237,8 @@ one.
 @command{printf} has an additional directive, @samp{%b}, which prints its
 argument string with @samp{\} escapes interpreted in the same way as in
 the @var{format} string, except that octal escapes are of the form
-@samp{\0@var{ooo}} where @var{ooo} is 0 to 3 octal digits.
+@samp{\0@var{ooo}} where @var{ooo} is 0 to 3 octal digits.  If
+@samp{\@var{ooo}} is nine-bit value, ignore the ninth bit.
 If a precision is also given, it limits the number of bytes printed
 from the converted string.
 
@@ -11191,13 +11265,17 @@ digits, but is printed according to the @env{LC_NUMERIC} category of the
 current locale.  For example, in a locale whose radix character is a
 comma, the command @samp{printf %g 3.14} outputs @samp{3,14} whereas
 the command @samp{printf %g 3,14} is an error.
+@xref{Floating point}.
 
 @kindex \@var{ooo}
 @kindex \x@var{hh}
 @command{printf} interprets @samp{\@var{ooo}} in @var{format} as an octal number
-(if @var{ooo} is 1 to 3 octal digits) specifying a character to print,
+(if @var{ooo} is 1 to 3 octal digits) specifying a byte to print,
 and @samp{\x@var{hh}} as a hexadecimal number (if @var{hh} is 1 to 2 hex
 digits) specifying a character to print.
+Note however that when @samp{\@var{ooo}} specifies a number larger than 255,
+@command{printf} ignores the ninth bit.
+For example, @samp{printf '\400'} is equivalent to @samp{printf '\0'}.
 
 @kindex \uhhhh
 @kindex \Uhhhhhhhh
@@ -15643,8 +15721,7 @@ days
 Historical implementations of @command{sleep} have required that
 @var{number} be an integer, and only accepted a single argument
 without a suffix.  However, GNU @command{sleep} accepts
-arbitrary floating point numbers (using a period before any fractional
-digits).
+arbitrary floating point numbers.  @xref{Floating point}.
 
 The only options are @option{--help} and @option{--version}.  @xref{Common
 options}.
@@ -15744,8 +15821,7 @@ When @var{increment} is not specified, it defaults to @samp{1},
 even when @var{first} is larger than @var{last}.
 @var{first} also defaults to @samp{1}.  So @code{seq 1} prints
 @samp{1}, but @code{seq 0} and @code{seq 10 5} produce no output.
-Floating-point numbers
-may be specified (using a period before any fractional digits).
+Floating-point numbers may be specified.  @xref{Floating point}.
 
 The program accepts the following options.  Also see @ref{Common options}.
 Options must precede operands.
@@ -15822,7 +15898,8 @@ of @code{%x}.
 
 On most systems, seq can produce whole-number output for values up to
 at least @math{2^{53}}.  Larger integers are approximated.  The details
-differ depending on your floating-point implementation, but a common
+differ depending on your floating-point implementation.
+@xref{Floating point}.  A common
 case is that @command{seq} works with integers through @math{2^{64}},
 and larger integers may not be numerically correct:
 

src/Makefile.am

@@ -145,6 +145,7 @@ noinst_HEADERS =	\
   copy.h		\
   cp-hash.h		\
   dircolors.h		\
+  fiemap.h		\
   find-mount-point.h	\
   fs.h			\
   group-list.h		\
@@ -449,7 +450,7 @@ uninstall-local:
 	  fi; \
 	fi
 
-copy_sources = copy.c cp-hash.c
+copy_sources = copy.c cp-hash.c extent-scan.c extent-scan.h
 
 # Use `ginstall' in the definition of PROGRAMS and in dependencies to avoid
 # confusion with the `install' target.  The install rule transforms `ginstall'

src/chcon.c

@@ -226,7 +226,7 @@ process_file (FTS *fts, FTSENT *ent)
               /* Tell fts not to traverse into this hierarchy.  */
               fts_set (fts, ent, FTS_SKIP);
               /* Ensure that we do not process "/" on the second visit.  */
-              ignore_ptr (fts_read (fts));
+              ignore_value (fts_read (fts));
               return false;
             }
           return true;

src/chmod.c

@@ -248,7 +248,7 @@ process_file (FTS *fts, FTSENT *ent)
       /* Tell fts not to traverse into this hierarchy.  */
       fts_set (fts, ent, FTS_SKIP);
       /* Ensure that we do not process "/" on the second visit.  */
-      ignore_ptr (fts_read (fts));
+      ignore_value (fts_read (fts));
       return false;
     }
 

src/chown-core.c

@@ -271,7 +271,7 @@ change_file_owner (FTS *fts, FTSENT *ent,
               /* Tell fts not to traverse into this hierarchy.  */
               fts_set (fts, ent, FTS_SKIP);
               /* Ensure that we do not process "/" on the second visit.  */
-              ignore_ptr (fts_read (fts));
+              ignore_value (fts_read (fts));
               return false;
             }
           return true;

src/copy.c

@@ -19,6 +19,7 @@
 #include <config.h>
 #include <stdio.h>
 #include <assert.h>
+#include <sys/ioctl.h>
 #include <sys/types.h>
 #include <selinux/selinux.h>
 
@@ -35,6 +36,7 @@
 #include "buffer-lcm.h"
 #include "copy.h"
 #include "cp-hash.h"
+#include "extent-scan.h"
 #include "error.h"
 #include "fcntl--.h"
 #include "file-set.h"
@@ -61,8 +63,6 @@
 # include "verror.h"
 #endif
 
-#include <sys/ioctl.h>
-
 #ifndef HAVE_FCHOWN
 # define HAVE_FCHOWN false
 # define fchown(fd, uid, gid) (-1)
@@ -130,6 +130,124 @@ utimens_symlink (char const *file, struct timespec const *timespec)
   return err;
 }
 
+/* Copy the regular file open on SRC_FD/SRC_NAME to DST_FD/DST_NAME,
+   honoring the MAKE_HOLES setting and using the BUF_SIZE-byte buffer
+   BUF for temporary storage.  Copy no more than MAX_N_READ bytes.
+   Return true upon successful completion;
+   print a diagnostic and return false upon error.
+   Note that for best results, BUF should be "well"-aligned.
+   BUF must have sizeof(uintptr_t)-1 bytes of additional space
+   beyond BUF[BUF_SIZE-1].
+   Set *LAST_WRITE_MADE_HOLE to true if the final operation on
+   DEST_FD introduced a hole.  Set *TOTAL_N_READ to the number of
+   bytes read.  */
+static bool
+sparse_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
+             bool make_holes,
+             char const *src_name, char const *dst_name,
+             uintmax_t max_n_read, off_t *total_n_read,
+             bool *last_write_made_hole)
+{
+  typedef uintptr_t word;
+  *last_write_made_hole = false;
+  *total_n_read = 0;
+
+  while (max_n_read)
+    {
+      word *wp = NULL;
+
+      ssize_t n_read = read (src_fd, buf, MIN (max_n_read, buf_size));
+      if (n_read < 0)
+        {
+          if (errno == EINTR)
+            continue;
+          error (0, errno, _("reading %s"), quote (src_name));
+          return false;
+        }
+      if (n_read == 0)
+        break;
+      max_n_read -= n_read;
+      *total_n_read += n_read;
+
+      if (make_holes)
+        {
+          char *cp;
+
+          /* Sentinel to stop loop.  */
+          buf[n_read] = '\1';
+#ifdef lint
+          /* Usually, buf[n_read] is not the byte just before a "word"
+             (aka uintptr_t) boundary.  In that case, the word-oriented
+             test below (*wp++ == 0) would read some uninitialized bytes
+             after the sentinel.  To avoid false-positive reports about
+             this condition (e.g., from a tool like valgrind), set the
+             remaining bytes -- to any value.  */
+          memset (buf + n_read + 1, 0, sizeof (word) - 1);
+#endif
+
+          /* Find first nonzero *word*, or the word with the sentinel.  */
+
+          wp = (word *) buf;
+          while (*wp++ == 0)
+            continue;
+
+          /* Find the first nonzero *byte*, or the sentinel.  */
+
+          cp = (char *) (wp - 1);
+          while (*cp++ == 0)
+            continue;
+
+          if (cp <= buf + n_read)
+            /* Clear to indicate that a normal write is needed. */
+            wp = NULL;
+          else
+            {
+              /* We found the sentinel, so the whole input block was zero.
+                 Make a hole.  */
+              if (lseek (dest_fd, n_read, SEEK_CUR) < 0)
+                {
+                  error (0, errno, _("cannot lseek %s"), quote (dst_name));
+                  return false;
+                }
+              *last_write_made_hole = true;
+            }
+        }
+
+      if (!wp)
+        {
+          size_t n = n_read;
+          if (full_write (dest_fd, buf, n) != n)
+            {
+              error (0, errno, _("writing %s"), quote (dst_name));
+              return false;
+            }
+          *last_write_made_hole = false;
+
+          /* It is tempting to return early here upon a short read from a
+             regular file.  That would save the final read syscall for each
+             file.  Unfortunately that doesn't work for certain files in
+             /proc with linux kernels from at least 2.6.9 .. 2.6.29.  */
+        }
+    }
+
+  return true;
+}
+
+/* If the file ends with a `hole' (i.e., if sparse_copy set wrote_hole_at_eof),
+   call this function to record the length of the output file.  */
+static bool
+sparse_copy_finalize (int dest_fd, char const *dst_name)
+{
+  off_t len = lseek (dest_fd, 0, SEEK_CUR);
+  if (0 <= len && ftruncate (dest_fd, len) < 0)
+    {
+      error (0, errno, _("truncating %s"), quote (dst_name));
+      return false;
+    }
+
+  return true;
+}
+
 /* Perform the O(1) btrfs clone operation, if possible.
    Upon success, return 0.  Otherwise, return -1 and set errno.  */
 static inline int
@@ -149,6 +267,162 @@ clone_file (int dest_fd, int src_fd)
 #endif
 }
 
+/* Write N_BYTES zero bytes to file descriptor FD.  Return true if successful.
+   Upon write failure, set errno and return false.  */
+static bool
+write_zeros (int fd, uint64_t n_bytes)
+{
+  static char *zeros;
+  static size_t nz = IO_BUFSIZE;
+
+  /* Attempt to use a relatively large calloc'd source buffer for
+     efficiency, but if that allocation fails, resort to a smaller
+     statically allocated one.  */
+  if (zeros == NULL)
+    {
+      static char fallback[1024];
+      zeros = calloc (nz, 1);
+      if (zeros == NULL)
+        {
+          zeros = fallback;
+          nz = sizeof fallback;
+        }
+    }
+
+  while (n_bytes)
+    {
+      uint64_t n = MIN (nz, n_bytes);
+      if ((full_write (fd, zeros, n)) != n)
+        return false;
+      n_bytes -= n;
+    }
+
+  return true;
+}
+
+/* Perform an efficient extent copy, if possible.  This avoids
+   the overhead of detecting holes in hole-introducing/preserving
+   copy, and thus makes copying sparse files much more efficient.
+   Upon a successful copy, return true.  If the initial extent scan
+   fails, set *NORMAL_COPY_REQUIRED to true and return false.
+   Upon any other failure, set *NORMAL_COPY_REQUIRED to false and
+   return false.  */
+static bool
+extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
+             off_t src_total_size, bool make_holes,
+             char const *src_name, char const *dst_name,
+             bool *require_normal_copy)
+{
+  struct extent_scan scan;
+  off_t last_ext_start = 0;
+  uint64_t last_ext_len = 0;
+
+  /* Keep track of the output position.
+     We may need this at the end, for a final ftruncate.  */
+  off_t dest_pos = 0;
+
+  extent_scan_init (src_fd, &scan);
+
+  *require_normal_copy = false;
+  bool wrote_hole_at_eof = true;
+  do
+    {
+      bool ok = extent_scan_read (&scan);
+      if (! ok)
+        {
+          if (scan.hit_final_extent)
+            break;
+
+          if (scan.initial_scan_failed)
+            {
+              *require_normal_copy = true;
+              return false;
+            }
+
+          error (0, errno, _("%s: failed to get extents info"),
+                 quote (src_name));
+          return false;
+        }
+
+      unsigned int i;
+      for (i = 0; i < scan.ei_count; i++)
+        {
+          off_t ext_start = scan.ext_info[i].ext_logical;
+          uint64_t ext_len = scan.ext_info[i].ext_length;
+
+          if (lseek (src_fd, ext_start, SEEK_SET) < 0)
+            {
+              error (0, errno, _("cannot lseek %s"), quote (src_name));
+            fail:
+              extent_scan_free (&scan);
+              return false;
+            }
+
+          if (make_holes)
+            {
+              if (lseek (dest_fd, ext_start, SEEK_SET) < 0)
+                {
+                  error (0, errno, _("cannot lseek %s"), quote (dst_name));
+                  goto fail;
+                }
+            }
+          else
+            {
+              /* When not inducing holes and when there is a hole between
+                 the end of the previous extent and the beginning of the
+                 current one, write zeros to the destination file.  */
+              if (last_ext_start + last_ext_len < ext_start)
+                {
+                  uint64_t hole_size = (ext_start
+                                        - last_ext_start
+                                        - last_ext_len);
+                  if (! write_zeros (dest_fd, hole_size))
+                    {
+                      error (0, errno, _("%s: write failed"), quote (dst_name));
+                      goto fail;
+                    }
+                }
+            }
+
+          last_ext_start = ext_start;
+          last_ext_len = ext_len;
+
+          off_t n_read;
+          if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
+                              make_holes, src_name, dst_name,
+                              ext_len, &n_read,
+                              &wrote_hole_at_eof))
+            return false;
+
+          dest_pos = ext_start + n_read;
+        }
+
+      /* Release the space allocated to scan->ext_info.  */
+      extent_scan_free (&scan);
+
+    }
+  while (! scan.hit_final_extent);
+
+  /* When the source file ends with a hole, we have to do a little more work,
+     since the above copied only up to and including the final extent.
+     In order to complete the copy, we may have to insert a hole or write
+     zeros in the destination corresponding to the source file's hole-at-EOF.
+
+     In addition, if the final extent was a block of zeros at EOF and we've
+     just converted them to a hole in the destination, we must call ftruncate
+     here in order to record the proper length in the destination.  */
+  if ((dest_pos < src_total_size || wrote_hole_at_eof)
+      && (make_holes
+          ? ftruncate (dest_fd, src_total_size)
+          : ! write_zeros (dest_fd, src_total_size - dest_pos)))
+    {
+      error (0, errno, _("failed to extend %s"), quote (dst_name));
+      return false;
+    }
+
+  return true;
+}
+
 /* FIXME: describe */
 /* FIXME: rewrite this to use a hash table so we avoid the quadratic
    performance hit that's probably noticeable only on trees deeper
@@ -648,7 +922,6 @@ copy_reg (char const *src_name, char const *dst_name,
   if (data_copy_required)
     {
       typedef uintptr_t word;
-      off_t n_read_total = 0;
 
       /* Choose a suitable buffer size; it may be adjusted later.  */
       size_t buf_alignment = lcm (getpagesize (), sizeof (word));
@@ -656,7 +929,6 @@ copy_reg (char const *src_name, char const *dst_name,
       size_t buf_size = io_blksize (sb);
 
       /* Deal with sparse files.  */
-      bool last_write_made_hole = false;
       bool make_holes = false;
 
       if (S_ISREG (sb.st_mode))
@@ -705,106 +977,37 @@ copy_reg (char const *src_name, char const *dst_name,
       buf_alloc = xmalloc (buf_size + buf_alignment_slop);
       buf = ptr_align (buf_alloc, buf_alignment);
 
-      while (true)
+      bool normal_copy_required;
+      /* Perform an efficient extent-based copy, falling back to the
+         standard copy only if the initial extent scan fails.  If the
+         '--sparse=never' option is specified, write all data but use
+         any extents to read more efficiently.  */
+      if (extent_copy (source_desc, dest_desc, buf, buf_size,
+                       src_open_sb.st_size, make_holes,
+                       src_name, dst_name, &normal_copy_required))
+        goto preserve_metadata;
+
+      if (! normal_copy_required)
         {
-          word *wp = NULL;
-
-          ssize_t n_read = read (source_desc, buf, buf_size);
-          if (n_read < 0)
-            {
-#ifdef EINTR
-              if (errno == EINTR)
-                continue;
-#endif
-              error (0, errno, _("reading %s"), quote (src_name));
-              return_val = false;
-              goto close_src_and_dst_desc;
-            }
-          if (n_read == 0)
-            break;
-
-          n_read_total += n_read;
-
-          if (make_holes)
-            {
-              char *cp;
-
-              /* Sentinel to stop loop.  */
-              buf[n_read] = '\1';
-#ifdef lint
-              /* Usually, buf[n_read] is not the byte just before a "word"
-                 (aka uintptr_t) boundary.  In that case, the word-oriented
-                 test below (*wp++ == 0) would read some uninitialized bytes
-                 after the sentinel.  To avoid false-positive reports about
-                 this condition (e.g., from a tool like valgrind), set the
-                 remaining bytes -- to any value.  */
-              memset (buf + n_read + 1, 0, sizeof (word) - 1);
-#endif
-
-              /* Find first nonzero *word*, or the word with the sentinel.  */
-
-              wp = (word *) buf;
-              while (*wp++ == 0)
-                continue;
-
-              /* Find the first nonzero *byte*, or the sentinel.  */
-
-              cp = (char *) (wp - 1);
-              while (*cp++ == 0)
-                continue;
-
-              if (cp <= buf + n_read)
-                /* Clear to indicate that a normal write is needed. */
-                wp = NULL;
-              else
-                {
-                  /* We found the sentinel, so the whole input block was zero.
-                     Make a hole.  */
-                  if (lseek (dest_desc, n_read, SEEK_CUR) < 0)
-                    {
-                      error (0, errno, _("cannot lseek %s"), quote (dst_name));
-                      return_val = false;
-                      goto close_src_and_dst_desc;
-                    }
-                  last_write_made_hole = true;
-                }
-            }
-
-          if (!wp)
-            {
-              size_t n = n_read;
-              if (full_write (dest_desc, buf, n) != n)
-                {
-                  error (0, errno, _("writing %s"), quote (dst_name));
-                  return_val = false;
-                  goto close_src_and_dst_desc;
-                }
-              last_write_made_hole = false;
-
-              /* It is tempting to return early here upon a short read from a
-                 regular file.  That would save the final read syscall for each
-                 file.  Unfortunately that doesn't work for certain files in
-                 /proc with linux kernels from at least 2.6.9 .. 2.6.29.  */
-            }
+          return_val = false;
+          goto close_src_and_dst_desc;
         }
 
-      /* If the file ends with a `hole', we need to do something to record
-         the length of the file.  On modern systems, calling ftruncate does
-         the job.  On systems without native ftruncate support, we have to
-         write a byte at the ending position.  Otherwise the kernel would
-         truncate the file at the end of the last write operation.  */
-
-      if (last_write_made_hole)
+      off_t n_read;
+      bool wrote_hole_at_eof;
+      if ( ! sparse_copy (source_desc, dest_desc, buf, buf_size,
+                          make_holes, src_name, dst_name,
+                          UINTMAX_MAX, &n_read,
+                          &wrote_hole_at_eof)
+           || (wrote_hole_at_eof &&
+               ! sparse_copy_finalize (dest_desc, dst_name)))
         {
-          if (ftruncate (dest_desc, n_read_total) < 0)
-            {
-              error (0, errno, _("truncating %s"), quote (dst_name));
-              return_val = false;
-              goto close_src_and_dst_desc;
-            }
+          return_val = false;
+          goto close_src_and_dst_desc;
         }
     }
 
+preserve_metadata:
   if (x->preserve_timestamps)
     {
       struct timespec timespec[2];

src/du.c

@@ -63,8 +63,11 @@ extern bool fts_debug;
 /* A set of dev/ino pairs.  */
 static struct di_set *di_set;
 
-/* Define a class for collecting directory information. */
+/* Keep track of the preceding "level" (depth in hierarchy)
+   from one call of process_file to the next.  */
+static size_t prev_level;
 
+/* Define a class for collecting directory information. */
 struct duinfo
 {
   /* Size of files in directory.  */
@@ -399,7 +402,6 @@ process_file (FTS *fts, FTSENT *ent)
   struct duinfo dui;
   struct duinfo dui_to_print;
   size_t level;
-  static size_t prev_level;
   static size_t n_alloc;
   /* First element of the structure contains:
      The sum of the st_size values of all entries in the single directory
@@ -582,10 +584,15 @@ du_files (char **files, int bit_flags)
             {
               if (errno != 0)
                 {
-                  /* FIXME: try to give a better message  */
-                  error (0, errno, _("fts_read failed"));
+                  error (0, errno, _("fts_read failed: %s"),
+                         quotearg_colon (fts->fts_path));
                   ok = false;
                 }
+
+              /* When exiting this loop early, be careful to reset the
+                 global, prev_level, used in process_file.  Otherwise, its
+                 (level == prev_level - 1) assertion could fail.  */
+              prev_level = 0;
               break;
             }
           FTS_CROSS_CHECK (fts);

src/extent-scan.c

@@ -0,0 +1,117 @@
+/* extent-scan.c -- core functions for scanning extents
+   Copyright (C) 2010-2011 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+   Written by Jie Liu (jeff.liu@oracle.com).  */
+
+#include <config.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <assert.h>
+
+#include "system.h"
+#include "extent-scan.h"
+
+#ifndef HAVE_FIEMAP
+# include "fiemap.h"
+#endif
+
+/* Allocate space for struct extent_scan, initialize the entries if
+   necessary and return it as the input argument of extent_scan_read().  */
+extern void
+extent_scan_init (int src_fd, struct extent_scan *scan)
+{
+  scan->fd = src_fd;
+  scan->ei_count = 0;
+  scan->scan_start = 0;
+  scan->initial_scan_failed = false;
+  scan->hit_final_extent = false;
+}
+
+#ifdef __linux__
+# ifndef FS_IOC_FIEMAP
+#  define FS_IOC_FIEMAP _IOWR ('f', 11, struct fiemap)
+# endif
+/* Call ioctl(2) with FS_IOC_FIEMAP (available in linux 2.6.27) to
+   obtain a map of file extents excluding holes.  */
+extern bool
+extent_scan_read (struct extent_scan *scan)
+{
+  union { struct fiemap f; char c[4096]; } fiemap_buf;
+  struct fiemap *fiemap = &fiemap_buf.f;
+  struct fiemap_extent *fm_extents = &fiemap->fm_extents[0];
+  enum { count = (sizeof fiemap_buf - sizeof *fiemap) / sizeof *fm_extents };
+  verify (count != 0);
+
+  /* This is required at least to initialize fiemap->fm_start,
+     but also serves (in mid 2010) to appease valgrind, which
+     appears not to know the semantics of the FIEMAP ioctl. */
+  memset (&fiemap_buf, 0, sizeof fiemap_buf);
+
+  fiemap->fm_start = scan->scan_start;
+  fiemap->fm_flags = FIEMAP_FLAG_SYNC;
+  fiemap->fm_extent_count = count;
+  fiemap->fm_length = FIEMAP_MAX_OFFSET - scan->scan_start;
+
+  /* Fall back to the standard copy if call ioctl(2) failed for the
+     the first time.  */
+  if (ioctl (scan->fd, FS_IOC_FIEMAP, fiemap) < 0)
+    {
+      if (scan->scan_start == 0)
+        scan->initial_scan_failed = true;
+      return false;
+    }
+
+  /* If 0 extents are returned, then more get_extent_table() are not needed.  */
+  if (fiemap->fm_mapped_extents == 0)
+    {
+      scan->hit_final_extent = true;
+      return false;
+    }
+
+  scan->ei_count = fiemap->fm_mapped_extents;
+  scan->ext_info = xnmalloc (scan->ei_count, sizeof (struct extent_info));
+
+  unsigned int i;
+  for (i = 0; i < scan->ei_count; i++)
+    {
+      assert (fm_extents[i].fe_logical <= OFF_T_MAX);
+
+      scan->ext_info[i].ext_logical = fm_extents[i].fe_logical;
+      scan->ext_info[i].ext_length = fm_extents[i].fe_length;
+      scan->ext_info[i].ext_flags = fm_extents[i].fe_flags;
+    }
+
+  i--;
+  if (scan->ext_info[i].ext_flags & FIEMAP_EXTENT_LAST)
+    {
+      scan->hit_final_extent = true;
+      return true;
+    }
+
+  scan->scan_start = fm_extents[i].fe_logical + fm_extents[i].fe_length;
+
+  return true;
+}
+#else
+extern bool
+extent_scan_read (struct extent_scan *scan ATTRIBUTE_UNUSED)
+{
+  scan->initial_scan_failed = true;
+  errno = ENOTSUP;
+  return false;
+}
+#endif

src/extent-scan.h

@@ -0,0 +1,68 @@
+/* core functions for efficient reading sparse files
+   Copyright (C) 2010-2011 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+   Written by Jie Liu (jeff.liu@oracle.com).  */
+
+#ifndef EXTENT_SCAN_H
+# define EXTENT_SCAN_H
+
+/* Structure used to store information of each extent.  */
+struct extent_info
+{
+  /* Logical offset of an extent.  */
+  off_t ext_logical;
+
+  /* Extent length.  */
+  uint64_t ext_length;
+
+  /* Extent flags, use it for FIEMAP only, or set it to zero.  */
+  uint32_t ext_flags;
+};
+
+/* Structure used to reserve extent scan information per file.  */
+struct extent_scan
+{
+  /* File descriptor of extent scan run against.  */
+  int fd;
+
+  /* Next scan start offset.  */
+  off_t scan_start;
+
+  /* How many extent info returned for a scan.  */
+  uint32_t ei_count;
+
+  /* If true, fall back to a normal copy, either set by the
+     failure of ioctl(2) for FIEMAP or lseek(2) with SEEK_DATA.  */
+  bool initial_scan_failed;
+
+  /* If true, the total extent scan per file has been finished.  */
+  bool hit_final_extent;
+
+  /* Extent information: a malloc'd array of ei_count structs.  */
+  struct extent_info *ext_info;
+};
+
+void extent_scan_init (int src_fd, struct extent_scan *scan);
+
+bool extent_scan_read (struct extent_scan *scan);
+
+static inline void
+extent_scan_free (struct extent_scan *scan)
+{
+  free (scan->ext_info);
+}
+
+#endif /* EXTENT_SCAN_H */

src/fiemap.h

@@ -0,0 +1,102 @@
+/* FS_IOC_FIEMAP ioctl infrastructure.
+   Some portions copyright (C) 2007 Cluster File Systems, Inc
+   Authors: Mark Fasheh <mfasheh@suse.com>
+            Kalpak Shah <kalpak.shah@sun.com>
+            Andreas Dilger <adilger@sun.com>.  */
+
+/* Copy from kernel, modified to respect GNU code style by Jie Liu.  */
+
+#ifndef _LINUX_FIEMAP_H
+# define _LINUX_FIEMAP_H
+
+# include <stdint.h>
+
+struct fiemap_extent
+{
+  /* Logical offset in bytes for the start of the extent
+     from the beginning of the file.  */
+  uint64_t fe_logical;
+
+  /* Physical offset in bytes for the start of the extent
+     from the beginning of the disk.  */
+  uint64_t fe_physical;
+
+  /* Length in bytes for this extent.  */
+  uint64_t fe_length;
+
+  uint64_t fe_reserved64[2];
+
+  /* FIEMAP_EXTENT_* flags for this extent.  */
+  uint32_t fe_flags;
+
+  uint32_t fe_reserved[3];
+};
+
+struct fiemap
+{
+  /* Logical offset(inclusive) at which to start mapping(in).  */
+  uint64_t fm_start;
+
+  /* Logical length of mapping which userspace wants(in).  */
+  uint64_t fm_length;
+
+  /* FIEMAP_FLAG_* flags for request(in/out).  */
+  uint32_t fm_flags;
+
+  /* Number of extents that were mapped(out).  */
+  uint32_t fm_mapped_extents;
+
+  /* Size of fm_extents array(in).  */
+  uint32_t fm_extent_count;
+
+  uint32_t fm_reserved;
+
+  /* Array of mapped extents(out).  */
+  struct fiemap_extent fm_extents[0];
+};
+
+/* The maximum offset can be mapped for a file.  */
+# define FIEMAP_MAX_OFFSET       (~0ULL)
+
+/* Sync file data before map.  */
+# define FIEMAP_FLAG_SYNC        0x00000001
+
+/* Map extented attribute tree.  */
+# define FIEMAP_FLAG_XATTR       0x00000002
+
+# define FIEMAP_FLAGS_COMPAT     (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR)
+
+/* Last extent in file.  */
+# define FIEMAP_EXTENT_LAST              0x00000001
+
+/* Data location unknown.  */
+# define FIEMAP_EXTENT_UNKNOWN           0x00000002
+
+/* Location still pending, Sets EXTENT_UNKNOWN.  */
+# define FIEMAP_EXTENT_DELALLOC          0x00000004
+
+/* Data can not be read while fs is unmounted.  */
+# define FIEMAP_EXTENT_ENCODED           0x00000008
+
+/* Data is encrypted by fs.  Sets EXTENT_NO_BYPASS.  */
+# define FIEMAP_EXTENT_DATA_ENCRYPTED    0x00000080
+
+/* Extent offsets may not be block aligned.  */
+# define FIEMAP_EXTENT_NOT_ALIGNED       0x00000100
+
+/* Data mixed with metadata.  Sets EXTENT_NOT_ALIGNED.  */
+# define FIEMAP_EXTENT_DATA_INLINE       0x00000200
+
+/* Multiple files in block.  Set EXTENT_NOT_ALIGNED.  */
+# define FIEMAP_EXTENT_DATA_TAIL         0x00000400
+
+/* Space allocated, but not data (i.e. zero).  */
+# define FIEMAP_EXTENT_UNWRITTEN         0x00000800
+
+/* File does not natively support extents.  Result merged for efficiency.  */
+# define FIEMAP_EXTENT_MERGED		0x00001000
+
+/* Space shared with other files.  */
+# define FIEMAP_EXTENT_SHARED            0x00002000
+
+#endif

src/install.c

@@ -24,6 +24,7 @@
 #include <pwd.h>
 #include <grp.h>
 #include <selinux/selinux.h>
+#include <sys/wait.h>
 
 #include "system.h"
 #include "backupfile.h"
@@ -48,8 +49,6 @@
 
 #define AUTHORS proper_name ("David MacKenzie")
 
-#include <sys/wait.h>
-
 static int selinux_enabled = 0;
 static bool use_default_selinux_context = true;
 

src/join.c

@@ -112,6 +112,13 @@ static bool issued_disorder_warning[2];
 /* Empty output field filler.  */
 static char const *empty_filler;
 
+/* Whether to ensure the same number of fields are output from each line.  */
+static bool autoformat;
+/* The number of fields to output for each line.
+   Only significant when autoformat is true.  */
+static size_t autocount_1;
+static size_t autocount_2;
+
 /* Field to join on; SIZE_MAX means they haven't been determined yet.  */
 static size_t join_field_1 = SIZE_MAX;
 static size_t join_field_2 = SIZE_MAX;
@@ -210,7 +217,8 @@ else fields are separated by CHAR.  Any FIELD is a field number counted\n\
 from 1.  FORMAT is one or more comma or blank separated specifications,\n\
 each being `FILENUM.FIELD' or `0'.  Default FORMAT outputs the join field,\n\
 the remaining fields from FILE1, the remaining fields from FILE2, all\n\
-separated by CHAR.\n\
+separated by CHAR.  If FORMAT is the keyword 'auto', then the first\n\
+line of each file determines the number of fields output for each line.\n\
 \n\
 Important: FILE1 and FILE2 must be sorted on the join fields.\n\
 E.g., use ` sort -k 1b,1 ' if `join' has no options,\n\
@@ -527,6 +535,27 @@ prfield (size_t n, struct line const *line)
     fputs (empty_filler, stdout);
 }
 
+/* Output all the fields in line, other than the join field.  */
+
+static void
+prfields (struct line const *line, size_t join_field, size_t autocount)
+{
+  size_t i;
+  size_t nfields = autoformat ? autocount : line->nfields;
+  char output_separator = tab < 0 ? ' ' : tab;
+
+  for (i = 0; i < join_field && i < nfields; ++i)
+    {
+      putchar (output_separator);
+      prfield (i, line);
+    }
+  for (i = join_field + 1; i < nfields; ++i)
+    {
+      putchar (output_separator);
+      prfield (i, line);
+    }
+}
+
 /* Print the join of LINE1 and LINE2.  */
 
 static void
@@ -534,6 +563,8 @@ prjoin (struct line const *line1, struct line const *line2)
 {
   const struct outlist *outlist;
   char output_separator = tab < 0 ? ' ' : tab;
+  size_t field;
+  struct line const *line;
 
   outlist = outlist_head.next;
   if (outlist)
@@ -543,9 +574,6 @@ prjoin (struct line const *line1, struct line const *line2)
       o = outlist;
       while (1)
         {
-          size_t field;
-          struct line const *line;
-
           if (o->file == 0)
             {
               if (line1 == &uni_blank)
@@ -574,37 +602,24 @@ prjoin (struct line const *line1, struct line const *line2)
     }
   else
     {
-      size_t i;
-
       if (line1 == &uni_blank)
         {
-          struct line const *t;
-          t = line1;
-          line1 = line2;
-          line2 = t;
+          line = line2;
+          field = join_field_2;
         }
-      prfield (join_field_1, line1);
-      for (i = 0; i < join_field_1 && i < line1->nfields; ++i)
+      else
         {
-          putchar (output_separator);
-          prfield (i, line1);
-        }
-      for (i = join_field_1 + 1; i < line1->nfields; ++i)
-        {
-          putchar (output_separator);
-          prfield (i, line1);
+          line = line1;
+          field = join_field_1;
         }
 
-      for (i = 0; i < join_field_2 && i < line2->nfields; ++i)
-        {
-          putchar (output_separator);
-          prfield (i, line2);
-        }
-      for (i = join_field_2 + 1; i < line2->nfields; ++i)
-        {
-          putchar (output_separator);
-          prfield (i, line2);
-        }
+      /* Output the join field.  */
+      prfield (field, line);
+
+      /* Output other fields.  */
+      prfields (line1, join_field_1, autocount_1);
+      prfields (line2, join_field_2, autocount_2);
+
       putchar ('\n');
     }
 }
@@ -627,13 +642,23 @@ join (FILE *fp1, FILE *fp2)
   initseq (&seq2);
   getseq (fp2, &seq2, 2);
 
-  if (join_header_lines && seq1.count && seq2.count)
+  if (autoformat)
     {
-      prjoin (seq1.lines[0], seq2.lines[0]);
+      autocount_1 = seq1.count ? seq1.lines[0]->nfields : 0;
+      autocount_2 = seq2.count ? seq2.lines[0]->nfields : 0;
+    }
+
+  if (join_header_lines && (seq1.count || seq2.count))
+    {
+      struct line const *hline1 = seq1.count ? seq1.lines[0] : &uni_blank;
+      struct line const *hline2 = seq2.count ? seq2.lines[0] : &uni_blank;
+      prjoin (hline1, hline2);
       prevline[0] = NULL;
       prevline[1] = NULL;
-      advance_seq (fp1, &seq1, true, 1);
-      advance_seq (fp2, &seq2, true, 2);
+      if (seq1.count)
+        advance_seq (fp1, &seq1, true, 1);
+      if (seq2.count)
+        advance_seq (fp2, &seq2, true, 2);
     }
 
   while (seq1.count && seq2.count)
@@ -711,7 +736,7 @@ join (FILE *fp1, FILE *fp2)
         seq2.count = 0;
     }
 
-  /* If the user did not specify --check-order, then we read the
+  /* If the user did not specify --nocheck-order, then we read the
      tail ends of both inputs to verify that they are in order.  We
      skip the rest of the tail once we have issued a warning for that
      file, unless we actually need to print the unpairable lines.  */
@@ -726,7 +751,8 @@ join (FILE *fp1, FILE *fp2)
     {
       if (print_unpairables_1)
         prjoin (seq1.lines[0], &uni_blank);
-      seen_unpairable = true;
+      if (seq2.count)
+        seen_unpairable = true;
       while (get_line (fp1, &line, 1))
         {
           if (print_unpairables_1)
@@ -740,7 +766,8 @@ join (FILE *fp1, FILE *fp2)
     {
       if (print_unpairables_2)
         prjoin (&uni_blank, seq2.lines[0]);
-      seen_unpairable = true;
+      if (seq1.count)
+        seen_unpairable = true;
       while (get_line (fp2, &line, 2))
         {
           if (print_unpairables_2)
@@ -1037,8 +1064,13 @@ main (int argc, char **argv)
           break;
 
         case 'o':
-          add_field_list (optarg);
-          optc_status = MIGHT_BE_O_ARG;
+          if (STREQ (optarg, "auto"))
+            autoformat = true;
+          else
+            {
+              add_field_list (optarg);
+              optc_status = MIGHT_BE_O_ARG;
+            }
           break;
 
         case 't':

src/pr.c

@@ -2695,7 +2695,7 @@ char_to_clump (char c)
               width = 2;
               chars = 2;
               *s++ = '^';
-              *s++ = c ^ 0100;
+              *s = c ^ 0100;
             }
           else
             {

src/ptx.c

@@ -22,13 +22,14 @@
 #include <getopt.h>
 #include <sys/types.h>
 #include "system.h"
+#include <regex.h>
 #include "argmatch.h"
 #include "diacrit.h"
 #include "error.h"
 #include "fadvise.h"
 #include "quote.h"
 #include "quotearg.h"
-#include "regex.h"
+#include "read-file.h"
 #include "stdio--.h"
 #include "xstrtol.h"
 
@@ -62,10 +63,6 @@
    options.  Many of the "int" values below should be "size_t" or
    something else like that.  */
 
-/* Reallocation step when swallowing non regular files.  The value is not
-   the actual reallocation step, but its base two logarithm.  */
-#define SWALLOW_REALLOC_LOG 12
-
 /* Program options.  */
 
 enum Format
@@ -511,88 +508,21 @@ initialize_regex (void)
 static void
 swallow_file_in_memory (const char *file_name, BLOCK *block)
 {
-  int file_handle;		/* file descriptor number */
-  struct stat stat_block;	/* stat block for file */
-  size_t allocated_length;	/* allocated length of memory buffer */
   size_t used_length;		/* used length in memory buffer */
-  int read_length;		/* number of character gotten on last read */
 
   /* As special cases, a file name which is NULL or "-" indicates standard
      input, which is already opened.  In all other cases, open the file from
      its name.  */
   bool using_stdin = !file_name || !*file_name || STREQ (file_name, "-");
   if (using_stdin)
-    file_handle = STDIN_FILENO;
+    block->start = fread_file (stdin, &used_length);
   else
-    if ((file_handle = open (file_name, O_RDONLY)) < 0)
-      error (EXIT_FAILURE, errno, "%s", file_name);
+    block->start = read_file (file_name, &used_length);
 
-  /* If the file is a plain, regular file, allocate the memory buffer all at
-     once and swallow the file in one blow.  In other cases, read the file
-     repeatedly in smaller chunks until we have it all, reallocating memory
-     once in a while, as we go.  */
+  if (!block->start)
+    error (EXIT_FAILURE, errno, "%s", quote (using_stdin ? "-" : file_name));
 
-  if (fstat (file_handle, &stat_block) < 0)
-    error (EXIT_FAILURE, errno, "%s", file_name);
-
-  if (S_ISREG (stat_block.st_mode))
-    {
-      size_t in_memory_size;
-
-      fdadvise (file_handle, 0, 0, FADVISE_SEQUENTIAL);
-
-      block->start = xmalloc ((size_t) stat_block.st_size);
-
-      if ((in_memory_size = read (file_handle,
-                                  block->start, (size_t) stat_block.st_size))
-          != stat_block.st_size)
-        {
-#if MSDOS
-          /* On MSDOS, in memory size may be smaller than the file
-             size, because of end of line conversions.  But it can
-             never be smaller than half the file size, because the
-             minimum is when all lines are empty and terminated by
-             CR+LF.  */
-          if (in_memory_size != (size_t)-1
-              && in_memory_size >= stat_block.st_size / 2)
-            block->start = xrealloc (block->start, in_memory_size);
-          else
-#endif /* not MSDOS */
-
-            error (EXIT_FAILURE, errno, "%s", file_name);
-        }
-      block->end = block->start + in_memory_size;
-    }
-  else
-    {
-      block->start = xmalloc ((size_t) 1 << SWALLOW_REALLOC_LOG);
-      used_length = 0;
-      allocated_length = (1 << SWALLOW_REALLOC_LOG);
-
-      while (read_length = read (file_handle,
-                                 block->start + used_length,
-                                 allocated_length - used_length),
-             read_length > 0)
-        {
-          used_length += read_length;
-          if (used_length == allocated_length)
-            {
-              allocated_length += (1 << SWALLOW_REALLOC_LOG);
-              block->start
-                = xrealloc (block->start, allocated_length);
-            }
-        }
-
-      if (read_length < 0)
-        error (EXIT_FAILURE, errno, "%s", file_name);
-
-      block->end = block->start + used_length;
-    }
-
-  /* Close the file, but only if it was not the standard input.  */
-
-  if (! using_stdin && close (file_handle) != 0)
-    error (EXIT_FAILURE, errno, "%s", file_name);
+  block->end = block->start + used_length;
 }
 
 /* Sort and search routines.  */

src/remove.c

@@ -25,6 +25,7 @@
 #include "error.h"
 #include "euidaccess-stat.h"
 #include "file-type.h"
+#include "ignore-value.h"
 #include "quote.h"
 #include "remove.h"
 #include "root-dev-ino.h"
@@ -377,10 +378,18 @@ nonexistent_file_errno (int errnum)
      exist, but be (in)accessible only via too long a symlink chain.
      Likewise for ENAMETOOLONG, since rm -f ./././.../foo may fail
      if the "..." part expands to a long enough sequence of "./"s,
-     even though ./foo does indeed exist.  */
+     even though ./foo does indeed exist.
+
+     Another case to consider is when a particular name is invalid for
+     a given file system.  In 2011, smbfs returns EINVAL, but the next
+     revision of POSIX will require EILSEQ for that situation:
+     http://austingroupbugs.net/view.php?id=293
+  */
 
   switch (errnum)
     {
+    case EILSEQ:
+    case EINVAL:
     case ENOENT:
     case ENOTDIR:
       return true;
@@ -402,7 +411,7 @@ fts_skip_tree (FTS *fts, FTSENT *ent)
 {
   fts_set (fts, ent, FTS_SKIP);
   /* Ensure that we do not process ENT a second time.  */
-  ent = fts_read (fts);
+  ignore_value (fts_read (fts));
 }
 
 /* Upon unlink failure, or when the user declines to remove ENT, mark

src/shuf.c

@@ -29,6 +29,7 @@
 #include "quotearg.h"
 #include "randint.h"
 #include "randperm.h"
+#include "read-file.h"
 #include "stdio--.h"
 #include "xstrtol.h"
 
@@ -147,52 +148,14 @@ read_input (FILE *in, char eolbyte, char ***pline)
 {
   char *p;
   char *buf = NULL;
+  size_t used;
   char *lim;
-  size_t alloc = 0;
-  size_t used = 0;
-  size_t next_alloc = (1 << 13) + 1;
-  size_t bytes_to_read;
-  size_t nread;
   char **line;
   size_t i;
   size_t n_lines;
-  int fread_errno;
-  struct stat instat;
 
-  if (fstat (fileno (in), &instat) == 0 && S_ISREG (instat.st_mode))
-    {
-      off_t file_size = instat.st_size;
-      off_t current_offset = ftello (in);
-      if (0 <= current_offset)
-        {
-          off_t remaining_size =
-            (current_offset < file_size ? file_size - current_offset : 0);
-          if (SIZE_MAX - 2 < remaining_size)
-            xalloc_die ();
-          next_alloc = remaining_size + 2;
-        }
-    }
-
-  do
-    {
-      if (alloc <= used + 1)
-        {
-          if (alloc == SIZE_MAX)
-            xalloc_die ();
-          alloc = next_alloc;
-          next_alloc = alloc * 2;
-          if (next_alloc < alloc)
-            next_alloc = SIZE_MAX;
-          buf = xrealloc (buf, alloc);
-        }
-
-      bytes_to_read = alloc - used - 1;
-      nread = fread (buf + used, sizeof (char), bytes_to_read, in);
-      used += nread;
-    }
-  while (nread == bytes_to_read);
-
-  fread_errno = errno;
+  if (!(buf = fread_file (in, &used)))
+    error (EXIT_FAILURE, errno, _("read error"));
 
   if (used && buf[used - 1] != eolbyte)
     buf[used++] = eolbyte;
@@ -209,7 +172,6 @@ read_input (FILE *in, char eolbyte, char ***pline)
   for (i = 1; i <= n_lines; i++)
     line[i] = p = next_line (p, eolbyte, lim - p);
 
-  errno = fread_errno;
   return n_lines;
 }
 
@@ -396,7 +358,7 @@ main (int argc, char **argv)
      doesn't have to worry about opening something other than
      stdin.  */
   if (! (echo || input_numbers_option_used (lo_input, hi_input))
-      && (ferror (stdin) || fclose (stdin) != 0))
+      && (fclose (stdin) != 0))
     error (EXIT_FAILURE, errno, _("read error"));
 
   permutation = randperm_new (randint_source, head_lines, n_lines);

src/sort.c

@@ -2214,7 +2214,9 @@ debug_key (struct line const *line, struct keyfield const *key)
 
           char *tighter_lim = beg;
 
-          if (key->month)
+          if (lim < beg)
+            tighter_lim = lim;
+          else if (key->month)
             getmonth (beg, &tighter_lim);
           else if (key->general_numeric)
             ignore_value (strtold (beg, &tighter_lim));
@@ -2341,10 +2343,10 @@ key_warnings (struct keyfield const *gkey, bool gkey_only)
           pn = stpcpy (stpcpy (pn, "-k "), umaxtostr (sword + 1, tmp));
           if (key->eword != SIZE_MAX)
             {
-              po = stpcpy (stpcpy (po, " -"), umaxtostr (eword + 1, tmp));
-              pn = stpcpy (stpcpy (pn, ","),
-                           umaxtostr (eword + 1
-                                      + (key->echar == SIZE_MAX), tmp));
+              stpcpy (stpcpy (po, " -"), umaxtostr (eword + 1, tmp));
+              stpcpy (stpcpy (pn, ","),
+                      umaxtostr (eword + 1
+                                 + (key->echar == SIZE_MAX), tmp));
             }
           error (0, 0, _("obsolescent key `%s' used; consider `%s' instead"),
                  obuf, nbuf);
@@ -3770,6 +3772,7 @@ sort (char *const *files, size_t nfiles, char const *output_file,
       size_t nthreads)
 {
   struct buffer buf;
+  IF_LINT (buf.buf = NULL);
   size_t ntemps = 0;
   bool output_file_created = false;
 

src/split.c

@@ -660,7 +660,7 @@ lines_rr (uintmax_t k, uintmax_t n, char *buf, size_t bufsize)
 {
   bool file_limit;
   size_t i_file;
-  of_t *files;
+  of_t *files IF_LINT (= NULL);
   uintmax_t line_no;
 
   if (k)

src/system.h

@@ -14,11 +14,15 @@
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
 
+/* Include this file _after_ system headers if possible.  */
+
 #include <alloca.h>
 
-/* Include sys/types.h before this file.  */
+/* Include <sys/types.h> before this file.
+   Note this doesn't warn if we're included
+   before all system headers.  */
 
-#if 2 <= __GLIBC__ && 2 <= __GLIBC_MINOR__
+#if 2 < __GLIBC__ || ( 2 == ___GLIBC__ && 2 <= __GLIBC_MINOR__ )
 # if ! defined _SYS_TYPES_H
 you must include <sys/types.h> before including this file
 # endif

src/tail.c

@@ -887,8 +887,11 @@ fremote (int fd, const char *name)
   int err = fstatfs (fd, &buf);
   if (err != 0)
     {
-      error (0, errno, _("cannot determine location of %s. "
-                         "reverting to polling"), quote (name));
+      /* On at least linux-2.6.38, fstatfs fails with ENOSYS when FD
+         is open on a pipe.  Treat that like a remote file.  */
+      if (errno != ENOSYS)
+        error (0, errno, _("cannot determine location of %s. "
+                           "reverting to polling"), quote (name));
     }
   else
     {

src/tee.c

@@ -183,10 +183,8 @@ tee_files (int nfiles, const char **files)
   while (1)
     {
       bytes_read = read (0, buffer, sizeof buffer);
-#ifdef EINTR
       if (bytes_read < 0 && errno == EINTR)
         continue;
-#endif
       if (bytes_read <= 0)
         break;
 

src/uniq.c

@@ -214,7 +214,7 @@ find_field (struct linebuffer const *line)
   size_t size = line->length - 1;
   size_t i = 0;
 
-  for (count = 0; count < skip_fields; count++)
+  for (count = 0; count < skip_fields && i < size; count++)
     {
       while (i < size && isblank (to_uchar (lp[i])))
         i++;
@@ -222,8 +222,7 @@ find_field (struct linebuffer const *line)
         i++;
     }
 
-  for (count = 0; count < skip_chars && i < size; count++)
-    i++;
+  i += MIN (skip_chars, size - i);
 
   return line->buffer + i;
 }

tests/Makefile.am

@@ -10,6 +10,7 @@ EXTRA_DIST =		\
   CuTmpdir.pm		\
   check.mk		\
   envvar-check		\
+  filefrag-extent-compare \
   init.cfg		\
   init.sh		\
   lang-default		\
@@ -25,6 +26,7 @@ root_tests =					\
   cp/special-bits				\
   cp/cp-mv-enotsup-xattr			\
   cp/capability					\
+  cp/sparse-fiemap                              \
   dd/skip-seek-past-dev				\
   install/install-C-root			\
   ls/capability					\
@@ -286,6 +288,7 @@ TESTS =						\
   misc/tty-eof					\
   misc/unexpand					\
   misc/uniq					\
+  misc/uniq-perf				\
   misc/xattr					\
   tail-2/wait					\
   chmod/c-option				\
@@ -317,6 +320,8 @@ TESTS =						\
   cp/dir-vs-file				\
   cp/existing-perm-race				\
   cp/fail-perm					\
+  cp/fiemap-perf                                \
+  cp/fiemap-2                                   \
   cp/file-perm-race				\
   cp/into-self					\
   cp/link					\
@@ -338,6 +343,7 @@ TESTS =						\
   cp/same-file					\
   cp/slink-2-slink				\
   cp/sparse					\
+  cp/sparse-to-pipe				\
   cp/special-f					\
   cp/src-base-dot				\
   cp/symlink-slash				\
@@ -370,6 +376,7 @@ TESTS =						\
   du/long-from-unreadable			\
   du/long-sloop					\
   du/max-depth					\
+  du/move-dir-while-traversing			\
   du/no-deref					\
   du/no-x					\
   du/one-file-system				\

tests/check.mk

@@ -20,8 +20,8 @@
 _v = TESTS
 _w = root_tests
 vc_exe_in_TESTS: Makefile
-	@rm -f t1 t2
-	@if test -d $(top_srcdir)/.git && test $(srcdir) = .; then	\
+	$(AM_V_GEN)rm -f t1 t2;						\
+	if test -d $(top_srcdir)/.git && test $(srcdir) = .; then	\
 	  { sed -n '/^$(_v) =[	 ]*\\$$/,/[^\]$$/p'			\
 		$(srcdir)/Makefile.am					\
 	    | sed 's/^  *//;/^\$$.*/d;/^$(_v) =/d';			\

tests/cp/backup-1

@@ -21,10 +21,8 @@
 print_ver_ cp
 
 suffix=.b
-file=b1.$$
+file=F
 file_backup="$file$suffix"
-temp_files="$file $file_backup"
-rm -f $temp_files
 
 echo test > $file || fail=1
 

tests/cp/fiemap-2

@@ -0,0 +1,54 @@
+#!/bin/sh
+# Exercise a few more corners of the fiemap-copying code.
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ cp
+
+# Require a fiemap-enabled FS.
+fiemap_capable_ . \
+  || skip_ "this file system lacks FIEMAP support"
+
+# Exercise the code that handles a file ending in a hole.
+printf x > k || framework_failure_
+dd bs=1k seek=128 of=k < /dev/null || framework_failure_
+
+# The first time through the outer loop, the input file, K, ends with a hole.
+# The second time through, we append a byte so that it does not.
+for append in no yes; do
+  test $append = yes && printf y >> k
+  for i in always never; do
+    cp --sparse=$i k k2 || fail=1
+    cmp k k2 || fail=1
+  done
+done
+
+# Ensure that --sparse=always can restore holes.
+rm -f k
+# Create a file starting with an "x", followed by 256K-1 0 bytes.
+printf x > k || framework_failure_
+dd bs=1k seek=1 of=k count=255 < /dev/zero || framework_failure_
+
+# cp should detect the all-zero blocks and convert some of them to holes.
+# How many it detects/converts currently depends on io_blksize.
+# Currently, on my F14/ext4 desktop, this K starts off with size 256KiB,
+# (note that the K in the preceding test starts off with size 4KiB).
+# cp from coreutils-8.9 with --sparse=always reduces the size to 32KiB.
+cp --sparse=always k k2 || fail=1
+test $(stat -c %b k2) -lt $(stat -c %b k) || fail=1
+
+Exit $fail

tests/cp/fiemap-perf

@@ -0,0 +1,36 @@
+#!/bin/sh
+# ensure that a sparse file is copied efficiently, by default
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ cp
+
+# Require a fiemap-enabled FS.
+fiemap_capable_ . \
+  || skip_ "this file system lacks FIEMAP support"
+
+# Create a large-but-sparse file.
+timeout 10 truncate -s1T f || framework_failure_
+
+# Nothing can read (much less write) that many bytes in so little time.
+timeout 10 cp f f2 || fail=1
+
+# Ensure that the sparse file copied through fiemap has the same size
+# in bytes as the original.
+test "$(stat --printf %s f)" = "$(stat --printf %s f2)" || fail=1
+
+Exit $fail

tests/cp/same-file

@@ -39,10 +39,7 @@ test $hard_link_to_symlink_does_the_deref = yes \
     && remove_these_sed='/^0 -[bf]*l .*sl1 ->/d' \
     || remove_these_sed='/^ELIDE NO TEST OUTPUT/d'
 
-actual=actual-$$
-expected=expected-$$
-
-exec 3>&1 1> $actual
+exec 3>&1 1> actual
 
 # FIXME: This should be bigger: like more than 8k
 contents=XYZ
@@ -130,7 +127,7 @@ for args in 'foo symlink' 'symlink foo' 'foo foo' 'sl1 sl2' 'foo hardlink'; do
   echo
 done
 
-cat <<\EOF | sed "$remove_these_sed" > $expected
+cat <<\EOF | sed "$remove_these_sed" > expected
 1 [cp: `foo' and `symlink' are the same file] (foo symlink -> foo)
 1 -d [cp: `foo' and `symlink' are the same file] (foo symlink -> foo)
 1 -f [cp: `foo' and `symlink' are the same file] (foo symlink -> foo)
@@ -218,6 +215,6 @@ EOF
 
 exec 1>&3 3>&-
 
-compare $expected $actual 1>&2 || fail=1
+compare expected actual 1>&2 || fail=1
 
 Exit $fail

tests/cp/sparse-fiemap

@@ -0,0 +1,104 @@
+#!/bin/sh
+# Test cp --sparse=always through fiemap copy
+
+# Copyright (C) 2010-2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ cp
+
+if fiemap_capable_ . ; then
+  : # Current dir is on a partition with working extents.  Good!
+else
+  # It's not;  we need to create one, hence we need root access.
+  require_root_
+
+  cwd=$PWD
+  cleanup_() { cd /; umount "$cwd/mnt"; }
+
+  skip=0
+  # Create an ext4 loopback file system
+  dd if=/dev/zero of=blob bs=32k count=1000 || skip=1
+  mkdir mnt
+  mkfs -t ext4 -F blob ||
+    skip_test_ "failed to create ext4 file system"
+  mount -oloop blob mnt   || skip=1
+  cd mnt                  || skip=1
+  echo test > f           || skip=1
+  test -s f               || skip=1
+
+  test $skip = 1 &&
+    skip_test_ "insufficient mount/ext4 support"
+fi
+
+# =================================================
+# Ensure that we exercise the FIEMAP-copying code enough
+# to provoke at least two iterations of the do...while loop
+# in which it calls ioctl (fd, FS_IOC_FIEMAP,...
+# This also verifies that non-trivial extents are preserved.
+
+$PERL -e 1 || skip_test_ 'skipping part of this test; you lack perl'
+
+# Extract logical block number and length pairs from filefrag -v output.
+# The initial sed is to remove the "eof" from the normally-empty "flags" field.
+# Similarly, remove flags values like "unknown,delalloc,eof".
+# That is required when that final extent has no number in the "expected" field.
+f()
+{
+  sed 's/ [a-z,][a-z,]*$//' $@ \
+    | awk '/^ *[0-9]/ {printf "%d %d ", $2 ,NF < 5 ? $NF : $5 } END {print ""}'
+}
+
+for i in $(seq 1 2 21); do
+  for j in 1 2 31 100; do
+    $PERL -e 'BEGIN { $n = '$i' * 1024; *F = *STDOUT }' \
+          -e 'for (1..'$j') { sysseek (*F, $n, 1)' \
+          -e '&& syswrite (*F, chr($_)x$n) or die "$!"}' > j1 || fail=1
+    # sync
+    cp --sparse=always j1 j2 || fail=1
+    # sync
+    # Technically we may need the 'sync' uses above, but
+    # uncommenting them makes this test take much longer.
+
+    cmp j1 j2 || fail=1
+    filefrag -v j1 | grep extent \
+      || skip_test_ 'skipping part of this test; you lack filefrag'
+
+    # Here is sample filefrag output:
+    #   $ perl -e 'BEGIN{$n=16*1024; *F=*STDOUT}' \
+    #          -e 'for (1..5) { sysseek(*F,$n,1)' \
+    #          -e '&& syswrite *F,"."x$n or die "$!"}' > j
+    #   $ filefrag -v j
+    #   File system type is: ef53
+    #   File size of j is 163840 (40 blocks, blocksize 4096)
+    #    ext logical physical expected length flags
+    #      0       4  6258884               4
+    #      1      12  6258892  6258887      4
+    #      2      20  6258900  6258895      4
+    #      3      28  6258908  6258903      4
+    #      4      36  6258916  6258911      4 eof
+    #   j: 6 extents found
+
+    # exclude the physical block numbers; they always differ
+    filefrag -v j1 > ff1 || fail=1
+    filefrag -v j2 > ff2 || fail=1
+    { f ff1; f ff2; } \
+      | $PERL $abs_top_srcdir/tests/filefrag-extent-compare \
+        || { fail=1; break; }
+  done
+  test $fail = 1 && break
+done
+
+Exit $fail

tests/cp/sparse-to-pipe

@@ -0,0 +1,31 @@
+#!/bin/sh
+# copy a sparse file to a pipe, to exercise some seldom-used parts of copy.c
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ cp
+
+require_sparse_support_
+
+mkfifo_or_skip_ pipe
+timeout 10 cat pipe > copy &
+
+truncate -s1M sparse || framework_failure_
+cp sparse pipe || fail=1
+cmp sparse copy || fail=1
+
+Exit $fail

tests/dd/misc

@@ -20,10 +20,10 @@
 . "${srcdir=.}/init.sh"; path_prepend_ ../src
 print_ver_ dd
 
-tmp_in=dd-in.$$
-tmp_in2=dd-in2.$$
-tmp_sym=dd-sym.$$
-tmp_out=dd-out.$$
+tmp_in=dd-in
+tmp_in2=dd-in2
+tmp_sym=dd-sym
+tmp_out=dd-out
 
 warn=0
 echo data > $tmp_in || framework_failure

tests/du/move-dir-while-traversing

@@ -0,0 +1,96 @@
+#!/bin/sh
+# Trigger a failed assertion in coreutils-8.9 and earlier.
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ du
+
+# We use a python-inotify script, so...
+python -m pyinotify -h > /dev/null \
+  || skip_ 'python inotify package not installed'
+
+# Move a directory "up" while du is processing its sub-directories.
+# While du is processing a hierarchy .../B/C/D/... this script
+# detects when du opens D/, and then moves C/ "up" one level
+# so that it is a sibling of B/.
+# Given the inherent race condition, we have to add enough "weight"
+# under D/ so that in most cases, the monitor performs the single
+# rename syscall before du finishes processing the subtree under D/.
+
+cat <<'EOF' > inotify-watch-for-dir-access.py
+#!/usr/bin/env python
+import pyinotify as pn
+import os,sys
+
+dir = sys.argv[1]
+dest_parent = os.path.dirname(os.path.dirname(dir))
+dest = os.path.join(dest_parent, os.path.basename(dir))
+
+class ProcessDir(pn.ProcessEvent):
+
+    def process_IN_OPEN(self, event):
+        os.rename(dir, dest)
+        sys.exit(0)
+
+    def process_default(self, event):
+        pass
+
+wm = pn.WatchManager()
+notifier = pn.Notifier(wm)
+wm.watch_transient_file(dir, pn.IN_OPEN, ProcessDir)
+sys.stdout.write('started\n')
+sys.stdout.flush()
+notifier.loop()
+EOF
+chmod a+x inotify-watch-for-dir-access.py
+
+t=T/U
+mkdir d2 || framework_failure
+long=d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z
+# One iteration of this loop creates a tree with which
+# du sometimes completes its traversal before the above rename.
+# Five iterations was not enough in 2 of 7 "make -j20 check" runs on a
+# 6/12-core system.  However, using "10", I saw no failure in 20 trials.
+# Using 10 iterations was not enough, either.
+# Using 30, I saw no failure in 200 trials.
+for i in $(seq 30); do
+  mkdir -p $t/3/a/b/c/$i/$long || framework_failure
+done
+
+# Prohibit suspension, which could otherwise cause a timeout-induced FP failure.
+trap '' TSTP
+
+timeout 6 ./inotify-watch-for-dir-access.py $t/3/a/b > start-msg &
+
+# Wait for the watcher to start...
+nonempty() { test -s start-msg || { sleep $1; return 1; }; }
+retry_delay_ nonempty .1 5
+
+# The above watches for an IN_OPEN event on $t/3/a/b,
+# and when it triggers, moves the parent, $t/3/a, up one level
+# so it's directly under $t.
+
+du -a $t d2 2> err
+# Before coreutils-8.10, du would abort.
+test $? = 1 || fail=1
+
+# check for the new diagnostic
+printf "du: fts_read failed: $t/3/a/b: No such file or directory\n" > exp \
+  || fail=1
+compare err exp || fail=1
+
+Exit $fail

tests/filefrag-extent-compare

@@ -0,0 +1,68 @@
+eval '(exit $?0)' && eval 'exec perl -wS "$0" ${1+"$@"}'
+  & eval 'exec perl -wS "$0" $argv:q'
+    if 0;
+# Determine whether two files have the same extents by comparing
+# the logical block numbers and lengths from filefrag -v for each.
+
+# Invoke like this:
+# This helper function, f, extracts logical block number and lengths.
+# f() { awk '/^ *[0-9]/ {printf "%d %d ",$2,NF<5?$NF:$5} END {print ""}'; }
+# { filefrag -v j1 | f; filefrag -v j2 | f; } | ./filefrag-extent-compare
+
+use warnings;
+use strict;
+(my $ME = $0) =~ s|.*/||;
+
+my @line = <>;
+my $n_lines = @line;
+$n_lines == 2
+  or die "$ME: expected exactly two input lines; got $n_lines\n";
+
+my @A = split ' ', $line[0];
+my @B = split ' ', $line[1];
+@A % 2 || @B % 2
+  and die "$ME: unexpected input: odd number of numbers; expected even\n";
+
+my @a;
+my @b;
+foreach my $i (0..@A/2-1) { $a[$i] = { L_BLK => $A[2*$i], LEN => $A[2*$i+1] } };
+foreach my $i (0..@B/2-1) { $b[$i] = { L_BLK => $B[2*$i], LEN => $B[2*$i+1] } };
+
+my $i = 0;
+my $j = 0;
+while (1)
+  {
+    !defined $a[$i] && !defined $b[$j]
+      and exit 0;
+    defined $a[$i] && defined $b[$j]
+      or die "\@a and \@b have different lengths, even after adjustment\n";
+    ($a[$i]->{L_BLK} == $b[$j]->{L_BLK}
+     && $a[$i]->{LEN} == $b[$j]->{LEN})
+      and next;
+    ($a[$i]->{LEN} < $b[$j]->{LEN}
+     && exists $a[$i+1] && $a[$i]->{LEN} + $a[$i+1]->{LEN} == $b[$j]->{LEN})
+      and ++$i, next;
+    exists $b[$j+1] && $a[$i]->{LEN} == $b[$i]->{LEN} + $b[$i+1]->{LEN}
+      and ++$j, next;
+    die "differing extent:\n"
+      . "  [$i]=$a[$i]->{L_BLK} $a[$i]->{LEN}\n"
+      . "  [$j]=$b[$j]->{L_BLK} $b[$j]->{LEN}\n"
+  }
+continue
+  {
+    ++$i;
+    ++$j;
+  }
+
+### Setup "GNU" style for perl-mode and cperl-mode.
+## Local Variables:
+## mode: perl
+## perl-indent-level: 2
+## perl-continued-statement-offset: 2
+## perl-continued-brace-offset: 0
+## perl-brace-offset: 0
+## perl-brace-imaginary-offset: 0
+## perl-label-offset: -2
+## perl-extra-newline-before-brace: t
+## perl-merge-trailing-else: nil
+## End:

tests/init.cfg

@@ -295,6 +295,15 @@ require_proc_pid_status_()
     kill $pid
 }
 
+# Return nonzero if the specified directory is on a file system for
+# which FIEMAP support exists, and the file system type is new enough
+# (unlike ext2 and ext3) that it is hard to find an instance *without*
+# FIEMAP support.
+fiemap_capable_()
+{
+  df -T -t btrfs -t xfs -t ext4 -t ocfs2 -t gfs2 "$@"
+}
+
 # Does the current (working-dir) file system support sparse files?
 require_sparse_support_()
 {

tests/misc/join

@@ -127,6 +127,26 @@ my @tv = (
 # From David Dyck
 ['9a', '', [" a 1\n b 2\n", " a Y\n b Z\n"], "a 1 Y\nb 2 Z\n", 0],
 
+# -o 'auto'
+['10a', '-a1 -a2 -e . -o auto',
+ ["a 1 2\nb 1\nd 1 2\n", "a 3 4\nb 3 4\nc 3 4\n"],
+ "a 1 2 3 4\nb 1 . 3 4\nc . . 3 4\nd 1 2 . .\n", 0],
+['10b', '-a1 -a2 -j3 -e . -o auto',
+ ["a 1 2\nb 1\nd 1 2\n", "a 3 4\nb 3 4\nc 3 4\n"],
+ "2 a 1 . .\n. b 1 . .\n2 d 1 . .\n4 . . a 3\n4 . . b 3\n4 . . c 3\n"],
+['10c', '-a1 -1 1 -2 4 -e. -o auto',
+ ["a 1 2\nb 1\nd 1 2\n", "a 3 4\nb 3 4\nc 3 4\n"],
+ "a 1 2 . . .\nb 1 . . . .\nd 1 2 . . .\n"],
+['10d', '-a2 -1 1 -2 4 -e. -o auto',
+ ["a 1 2\nb 1\nd 1 2\n", "a 3 4\nb 3 4\nc 3 4\n"],
+ ". . . a 3 4\n. . . b 3 4\n. . . c 3 4\n"],
+['10e', '-o auto',
+ ["a 1 2\nb 1 2 discard\n", "a 3 4\nb 3 4 discard\n"],
+ "a 1 2 3 4\nb 1 2 3 4\n"],
+['10f', '-t, -o auto',
+ ["a,1,,2\nb,1,2\n", "a,3,4\nb,3,4\n"],
+ "a,1,,2,3,4\nb,1,2,,3,4\n"],
+
 # From Tim Smithers: fixed in 1.22l
 ['trailing-sp', '-t: -1 1 -2 1', ["a:x \n", "a:y \n"], "a:x :y \n", 0],
 
@@ -189,7 +209,11 @@ my @tv = (
 
 # Before 6.10.143, this would mistakenly fail with the diagnostic:
 # join: File 1 is not in sorted order
-['chkodr-7', '-12', ["2 a\n1 b\n", ""], "", 0],
+['chkodr-7', '-12', ["2 a\n1 b\n", "2 c\n1 d"], "", 0],
+
+# After 8.9, join doesn't report disorder by default
+# when comparing against an empty input file.
+['chkodr-8', '', ["2 a\n1 b\n", ""], "", 0],
 
 # Test '--header' feature
 ['header-1', '--header',
@@ -219,6 +243,12 @@ my @tv = (
  [ "ID1 Name\n1 A\n2 B\n", "ID2 Color\n1 red\n"],
    "ID1 Name Color\n1 A red\n", 0],
 
+# '--header' doesn't check order of a header
+# even if there is no header in the second file
+['header-6', '--header -a1',
+ [ "ID1 Name\n1 A\n", ""],
+   "ID1 Name\n1 A\n", 0],
+
 );
 
 # Convert the above old-style test vectors to the newer

tests/misc/sort-debug-keys

@@ -234,6 +234,10 @@ _
 _
 >a
  _
+A>chr10
+     ^ no match for key
+B>chr1
+     ^ no match for key
 EOF
 
 (
@@ -275,6 +279,9 @@ env printf '1a\x002b\x00' | sort -s -n -z --debug
 
 # Check that \0 and \t intermix.
 printf '\0\ta\n' | sort -s -k2b,2 --debug | tr -d '\0'
+
+# Check that key end before key start is not underlined
+printf 'A\tchr10\nB\tchr1\n' | sort -s -k2.4b,2.3n --debug
 ) > out
 
 compare out exp || fail=1

tests/misc/uniq-perf

@@ -0,0 +1,25 @@
+#!/bin/sh
+# before coreutils-8.10, seq 100000|uniq -f 10000000000 would run for days
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ uniq
+
+seq 100 > in || fail=1
+timeout 1 uniq -f 10000000000 in || fail=1
+
+Exit $fail

tests/mv/i-3

@@ -23,6 +23,8 @@ require_controlling_input_terminal_
 skip_if_root_
 trap '' TTIN # Ignore SIGTTIN
 
+test "$(uname -s)" = FreeBSD && skip_ "known spurious failure on FreeBSD"
+
 touch f g h i || framework_failure
 chmod 0 g i || framework_failure
 

tests/mv/part-symlink

@@ -45,10 +45,7 @@ pwd_tmp=`pwd`
 # Exercise those four cases for each of
 # cp and mv, with lots of combinations of options.
 
-actual=actual-$$
-expected=expected-$$
-
-exec 1> $actual
+exec 1> actual
 
 # FIXME: This should be bigger: like more than 8k
 contents=XYZ
@@ -153,7 +150,7 @@ done
 test $fail = 1 &&
   { (exit 1); exit; }
 
-cat <<\EOF > $expected
+cat <<\EOF > expected
 1 cp loc_reg rem_sl
  [cp: `loc_reg' and `rem_sl' are the same file]
  (loc_reg) (rem_sl -> dir/loc_reg)
@@ -259,6 +256,6 @@ cat <<\EOF > $expected
 EOF
 
 # Redirect to stderr, since stdout is already taken.
-compare $expected $actual 1>&2 || fail=1
+compare expected actual 1>&2 || fail=1
 
 Exit $fail

tests/mv/to-symlink

@@ -24,10 +24,8 @@ cleanup_() { rm -rf "$other_partition_tmpdir"; }
 
 rem_file="$other_partition_tmpdir/file"
 rem_symlink="$other_partition_tmpdir/symlink"
-file=to-sym-$$
+file=to-sym
 
-
-rm -f $file || framework_failure
 echo local > $file || framework_failure
 echo remote > $rem_file || framework_failure
 ln -s $rem_file $rem_symlink || framework_failure

tests/touch/fail-diag

@@ -20,15 +20,11 @@
 print_ver_ touch
 skip_if_root_
 
-d1=no-$$
-dir=/$d1/such-dir
-# Ensure that $d1 doesn't already exist.
-ls -d $d1 2> /dev/null && framework_failure
+file=/no-such-dir/file
 
-
-touch $dir > out 2>&1 && fail=1
+touch $file > out 2>&1 && fail=1
 cat <<EOF > exp
-touch: cannot touch \`$dir': No such file or directory
+touch: cannot touch \`$file': No such file or directory
 EOF
 
 compare out exp || fail=1