devel/coreutils/coreutils-i18n.patch

diff -urNp coreutils-8.0-orig/lib/linebuffer.h coreutils-8.0/lib/linebuffer.h
--- coreutils-8.0-orig/lib/linebuffer.h	2009-10-06 10:59:48.000000000 +0200
+++ coreutils-8.0/lib/linebuffer.h	2009-10-07 10:07:16.000000000 +0200
@@ -21,6 +21,11 @@
 
 # include <stdio.h>
 
+/* Get mbstate_t.  */
+# if HAVE_WCHAR_H
+#  include <wchar.h>
+# endif
+
 /* A `struct linebuffer' holds a line of text. */
 
 struct linebuffer
@@ -28,6 +33,9 @@ struct linebuffer
   size_t size;			/* Allocated. */
   size_t length;		/* Used. */
   char *buffer;
+# if HAVE_WCHAR_H
+  mbstate_t state;
+# endif
 };
 
 /* Initialize linebuffer LINEBUFFER for use. */
diff -urNp coreutils-8.0-orig/src/cut.c coreutils-8.0/src/cut.c
--- coreutils-8.0-orig/src/cut.c	2009-09-23 10:25:44.000000000 +0200
+++ coreutils-8.0/src/cut.c	2009-10-07 10:07:16.000000000 +0200
@@ -28,6 +28,11 @@
 #include <assert.h>
 #include <getopt.h>
 #include <sys/types.h>
+
+/* Get mbstate_t, mbrtowc().  */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
 #include "system.h"
 
 #include "error.h"
@@ -36,6 +41,18 @@
 #include "quote.h"
 #include "xstrndup.h"
 
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+   installation; work around this configuration error.        */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# undef MB_LEN_MAX
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "cut"
 
@@ -71,6 +88,52 @@
     }							\
   while (0)
 
+/* Refill the buffer BUF to get a multibyte character. */
+#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM)                        \
+  do                                                                        \
+    {                                                                        \
+      if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM))        \
+        {                                                                \
+          memmove (BUF, BUFPOS, BUFLEN);                                \
+          BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \
+          BUFPOS = BUF;                                                        \
+        }                                                                \
+    }                                                                        \
+  while (0)
+
+/* Get wide character on BUFPOS. BUFPOS is not included after that.
+   If byte sequence is not valid as a character, CONVFAIL is 1. Otherwise 0. */ 
+#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \
+  do                                                                        \
+    {                                                                        \
+      mbstate_t state_bak;                                                \
+                                                                        \
+      if (BUFLEN < 1)                                                        \
+        {                                                                \
+          WC = WEOF;                                                        \
+          break;                                                        \
+        }                                                                \
+                                                                        \
+      /* Get a wide character. */                                        \
+      CONVFAIL = 0;                                                        \
+      state_bak = STATE;                                                \
+      MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE);        \
+                                                                        \
+      switch (MBLENGTH)                                                        \
+        {                                                                \
+        case (size_t)-1:                                                \
+        case (size_t)-2:                                                \
+          CONVFAIL++;                                                        \
+          STATE = state_bak;                                                \
+          /* Fall througn. */                                                \
+                                                                        \
+        case 0:                                                                \
+          MBLENGTH = 1;                                                        \
+          break;                                                        \
+        }                                                                \
+    }                                                                        \
+  while (0)
+
 struct range_pair
   {
     size_t lo;
@@ -89,7 +152,7 @@ static char *field_1_buffer;
 /* The number of bytes allocated for FIELD_1_BUFFER.  */
 static size_t field_1_bufsize;
 
-/* The largest field or byte index used as an endpoint of a closed
+/* The largest byte, character or field index used as an endpoint of a closed
    or degenerate range specification;  this doesn't include the starting
    index of right-open-ended ranges.  For example, with either range spec
    `2-5,9-', `2-3,5,9-' this variable would be set to 5.  */
@@ -101,10 +164,11 @@ static size_t eol_range_start;
 
 /* This is a bit vector.
    In byte mode, which bytes to output.
+   In character mode, which characters to output.
    In field mode, which DELIM-separated fields to output.
-   Both bytes and fields are numbered starting with 1,
+   Bytes, characters and fields are numbered starting with 1,
    so the zeroth bit of this array is unused.
-   A field or byte K has been selected if
+   A byte, character or field K has been selected if
    (K <= MAX_RANGE_ENDPOINT and is_printable_field(K))
     || (EOL_RANGE_START > 0 && K >= EOL_RANGE_START).  */
 static unsigned char *printable_field;
@@ -113,15 +177,25 @@ enum operating_mode
   {
     undefined_mode,
 
-    /* Output characters that are in the given bytes. */
+    /* Output bytes that are at the given positions. */
     byte_mode,
 
+    /* Output characters that are at the given positions. */
+    character_mode,
+
     /* Output the given delimeter-separated fields. */
     field_mode
   };
 
 static enum operating_mode operating_mode;
 
+/* If nonzero, when in byte mode, don't split multibyte characters.  */
+static int byte_mode_character_aware;
+
+/* If nonzero, the function for single byte locale is work
+   if this program runs on multibyte locale. */
+static int force_singlebyte_mode;
+
 /* If true do not output lines containing no delimeter characters.
    Otherwise, all such lines are printed.  This option is valid only
    with field mode.  */
@@ -133,6 +207,9 @@ static bool complement;
 
 /* The delimeter character for field mode. */
 static unsigned char delim;
+#if HAVE_WCHAR_H
+static wchar_t wcdelim;
+#endif
 
 /* True if the --output-delimiter=STRING option was specified.  */
 static bool output_delimiter_specified;
@@ -206,7 +283,7 @@ Mandatory arguments to long options are 
   -f, --fields=LIST       select only these fields;  also print any line\n\
                             that contains no delimiter character, unless\n\
                             the -s option is specified\n\
-  -n                      (ignored)\n\
+  -n                      with -b: don't split multibyte characters\n\
 "), stdout);
       fputs (_("\
       --complement        complement the set of selected bytes, characters\n\
@@ -365,7 +442,7 @@ set_fields (const char *fieldstr)
           in_digits = false;
           /* Starting a range. */
           if (dash_found)
-            FATAL_ERROR (_("invalid byte or field list"));
+            FATAL_ERROR (_("invalid byte, character or field list"));
           dash_found = true;
           fieldstr++;
 
@@ -389,14 +466,16 @@ set_fields (const char *fieldstr)
               if (!rhs_specified)
                 {
                   /* `n-'.  From `initial' to end of line. */
-                  eol_range_start = initial;
+                  if (eol_range_start == 0 ||
+                      (eol_range_start != 0 && eol_range_start > initial))
+                    eol_range_start = initial;
                   field_found = true;
                 }
               else
                 {
                   /* `m-n' or `-n' (1-n). */
                   if (value < initial)
-                    FATAL_ERROR (_("invalid decreasing range"));
+                    FATAL_ERROR (_("invalid byte, character or field list"));
 
                   /* Is there already a range going to end of line? */
                   if (eol_range_start != 0)
@@ -476,6 +555,9 @@ set_fields (const char *fieldstr)
               if (operating_mode == byte_mode)
                 error (0, 0,
                        _("byte offset %s is too large"), quote (bad_num));
+              else if (operating_mode == character_mode)
+                error (0, 0,
+                       _("character offset %s is too large"), quote (bad_num));
               else
                 error (0, 0,
                        _("field number %s is too large"), quote (bad_num));
@@ -486,7 +568,7 @@ set_fields (const char *fieldstr)
           fieldstr++;
         }
       else
-        FATAL_ERROR (_("invalid byte or field list"));
+        FATAL_ERROR (_("invalid byte, character or field list"));
     }
 
   max_range_endpoint = 0;
@@ -579,6 +661,63 @@ cut_bytes (FILE *stream)
     }
 }
 
+#if HAVE_MBRTOWC
+/* This function is in use for the following case.
+
+   1. Read from the stream STREAM, printing to standard output any selected
+   characters. 
+
+   2. Read from stream STREAM, printing to standard output any selected bytes,
+   without splitting multibyte characters.  */
+ 
+static void
+cut_characters_or_cut_bytes_no_split (FILE *stream)
+{
+  int idx;                /* number of bytes or characters in the line so far. */
+  char buf[MB_LEN_MAX + BUFSIZ];  /* For spooling a read byte sequence. */
+  char *bufpos;                /* Next read position of BUF. */
+  size_t buflen;        /* The length of the byte sequence in buf. */
+  wint_t wc;                /* A gotten wide character. */
+  size_t mblength;        /* The byte size of a multibyte character which shows
+                           as same character as WC. */
+  mbstate_t state;        /* State of the stream. */
+  int convfail;                /* 1, when conversion is failed. Otherwise 0. */
+
+  idx = 0;
+  buflen = 0;
+  bufpos = buf;
+  memset (&state, '\0', sizeof(mbstate_t));
+
+  while (1)
+    {
+      REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+      GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail);
+
+      if (wc == WEOF)
+        {
+          if (idx > 0)
+            putchar ('\n');
+          break;
+        }
+      else if (wc == L'\n')
+        {
+          putchar ('\n');
+          idx = 0;
+        }
+      else
+        {
+          idx += (operating_mode == byte_mode) ? mblength : 1;
+          if (print_kth (idx, NULL))
+            fwrite (bufpos, mblength, sizeof(char), stdout);
+        }
+
+      buflen -= mblength;
+      bufpos += mblength;
+    }
+}
+#endif
+                   
 /* Read from stream STREAM, printing to standard output any selected fields.  */
 
 static void
@@ -701,13 +840,192 @@ cut_fields (FILE *stream)
     }
 }
 
+#if HAVE_MBRTOWC
+static void
+cut_fields_mb (FILE *stream)
+{
+  int c;
+  unsigned int field_idx;
+  int found_any_selected_field;
+  int buffer_first_field;
+  int empty_input;
+  char buf[MB_LEN_MAX + BUFSIZ];  /* For spooling a read byte sequence. */
+  char *bufpos;                /* Next read position of BUF. */
+  size_t buflen;        /* The length of the byte sequence in buf. */
+  wint_t wc = 0;        /* A gotten wide character. */
+  size_t mblength;        /* The byte size of a multibyte character which shows
+                           as same character as WC. */
+  mbstate_t state;        /* State of the stream. */
+  int convfail;                /* 1, when conversion is failed. Otherwise 0. */
+
+  found_any_selected_field = 0;
+  field_idx = 1;
+  bufpos = buf;
+  buflen = 0;
+  memset (&state, '\0', sizeof(mbstate_t));
+
+  c = getc (stream);
+  empty_input = (c == EOF);
+  if (c != EOF)
+    ungetc (c, stream);
+  else
+    wc = WEOF;
+
+  /* To support the semantics of the -s flag, we may have to buffer
+     all of the first field to determine whether it is `delimited.'
+     But that is unnecessary if all non-delimited lines must be printed
+     and the first field has been selected, or if non-delimited lines
+     must be suppressed and the first field has *not* been selected.
+     That is because a non-delimited line has exactly one field.  */
+  buffer_first_field = (suppress_non_delimited ^ !print_kth (1, NULL));
+
+  while (1)
+    {
+      if (field_idx == 1 && buffer_first_field)
+        {
+          int len = 0;
+
+          while (1)
+            {
+              REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+              GET_NEXT_WC_FROM_BUFFER
+                (wc, bufpos, buflen, mblength, state, convfail);
+
+              if (wc == WEOF)
+                break;
+
+              field_1_buffer = xrealloc (field_1_buffer, len + mblength);
+              memcpy (field_1_buffer + len, bufpos, mblength);
+              len += mblength;
+              buflen -= mblength;
+              bufpos += mblength;
+
+              if (!convfail && (wc == L'\n' || wc == wcdelim))
+                break;
+            }
+
+          if (wc == WEOF)
+            break;
+
+          /* If the first field extends to the end of line (it is not
+             delimited) and we are printing all non-delimited lines,
+             print this one.  */
+          if (convfail || (!convfail && wc != wcdelim))
+            {
+              if (suppress_non_delimited)
+                {
+                  /* Empty.        */
+                }
+              else
+                {
+                  fwrite (field_1_buffer, sizeof (char), len, stdout);
+                  /* Make sure the output line is newline terminated.  */
+                  if (convfail || (!convfail && wc != L'\n'))
+                    putchar ('\n');
+                }
+              continue;
+            }
+
+          if (print_kth (1, NULL))
+            {
+              /* Print the field, but not the trailing delimiter.  */
+              fwrite (field_1_buffer, sizeof (char), len - 1, stdout);
+              found_any_selected_field = 1;
+            }
+          ++field_idx;
+        }
+
+      if (wc != WEOF)
+        {
+          if (print_kth (field_idx, NULL))
+            {
+              if (found_any_selected_field)
+                {
+                  fwrite (output_delimiter_string, sizeof (char),
+                          output_delimiter_length, stdout);
+                }
+              found_any_selected_field = 1;
+            }
+
+          while (1)
+            {
+              REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+              GET_NEXT_WC_FROM_BUFFER
+                (wc, bufpos, buflen, mblength, state, convfail);
+
+              if (wc == WEOF)
+                break;
+              else if (!convfail && (wc == wcdelim || wc == L'\n'))
+                {
+                  buflen -= mblength;
+                  bufpos += mblength;
+                  break;
+                }
+
+              if (print_kth (field_idx, NULL))
+                fwrite (bufpos, mblength, sizeof(char), stdout);
+
+              buflen -= mblength;
+              bufpos += mblength;
+            }
+        }
+
+      if ((!convfail || wc == L'\n') && buflen < 1)
+        wc = WEOF;
+
+      if (!convfail && wc == wcdelim)
+        ++field_idx;
+      else if (wc == WEOF || (!convfail && wc == L'\n'))
+        {
+          if (found_any_selected_field
+              || (!empty_input && !(suppress_non_delimited && field_idx == 1)))
+            putchar ('\n');
+          if (wc == WEOF)
+            break;
+          field_idx = 1;
+          found_any_selected_field = 0;
+        }
+    }
+}
+#endif
+
 static void
 cut_stream (FILE *stream)
 {
-  if (operating_mode == byte_mode)
-    cut_bytes (stream);
+#if HAVE_MBRTOWC
+  if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
+    {
+      switch (operating_mode)
+        {
+        case byte_mode:
+          if (byte_mode_character_aware)
+            cut_characters_or_cut_bytes_no_split (stream);
+          else
+            cut_bytes (stream);
+          break;
+
+        case character_mode:
+          cut_characters_or_cut_bytes_no_split (stream);
+          break;
+
+        case field_mode:
+          cut_fields_mb (stream);
+          break;
+
+        default:
+          abort ();
+        }
+    }
   else
-    cut_fields (stream);
+#endif
+    {
+      if (operating_mode == field_mode)
+        cut_fields (stream);
+      else
+        cut_bytes (stream);
+    }
 }
 
 /* Process file FILE to standard output.
@@ -757,6 +1075,8 @@ main (int argc, char **argv)
   bool ok;
   bool delim_specified = false;
   char *spec_list_string IF_LINT(= NULL);
+  char mbdelim[MB_LEN_MAX + 1];
+  size_t delimlen = 0;
 
   initialize_main (&argc, &argv);
   set_program_name (argv[0]);
@@ -779,7 +1099,6 @@ main (int argc, char **argv)
       switch (optc)
         {
         case 'b':
-        case 'c':
           /* Build the byte list. */
           if (operating_mode != undefined_mode)
             FATAL_ERROR (_("only one type of list may be specified"));
@@ -787,6 +1106,14 @@ main (int argc, char **argv)
           spec_list_string = optarg;
           break;
 
+        case 'c':
+          /* Build the character list. */
+          if (operating_mode != undefined_mode)
+            FATAL_ERROR (_("only one type of list may be specified"));
+          operating_mode = character_mode;
+          spec_list_string = optarg;
+          break;
+
         case 'f':
           /* Build the field list. */
           if (operating_mode != undefined_mode)
@@ -798,10 +1125,35 @@ main (int argc, char **argv)
         case 'd':
           /* New delimiter. */
           /* Interpret -d '' to mean `use the NUL byte as the delimiter.'  */
-          if (optarg[0] != '\0' && optarg[1] != '\0')
-            FATAL_ERROR (_("the delimiter must be a single character"));
-          delim = optarg[0];
-          delim_specified = true;
+            {
+#if HAVE_MBRTOWC
+              if(MB_CUR_MAX > 1)
+                {
+                  mbstate_t state;
+
+                  memset (&state, '\0', sizeof(mbstate_t));
+                  delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state);
+
+                  if (delimlen == (size_t)-1 || delimlen == (size_t)-2)
+                    ++force_singlebyte_mode;
+                  else
+                    {
+                      delimlen = (delimlen < 1) ? 1 : delimlen;
+                      if (wcdelim != L'\0' && *(optarg + delimlen) != '\0')
+                        FATAL_ERROR (_("the delimiter must be a single character"));
+                      memcpy (mbdelim, optarg, delimlen);
+                    }
+                }
+
+              if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
+#endif
+                {
+                  if (optarg[0] != '\0' && optarg[1] != '\0')
+                    FATAL_ERROR (_("the delimiter must be a single character"));
+                  delim = (unsigned char) optarg[0];
+                }
+            delim_specified = true;
+          }
           break;
 
         case OUTPUT_DELIMITER_OPTION:
@@ -814,6 +1166,7 @@ main (int argc, char **argv)
           break;
 
         case 'n':
+          byte_mode_character_aware = 1;
           break;
 
         case 's':
@@ -836,7 +1189,7 @@ main (int argc, char **argv)
   if (operating_mode == undefined_mode)
     FATAL_ERROR (_("you must specify a list of bytes, characters, or fields"));
 
-  if (delim != '\0' && operating_mode != field_mode)
+  if (delim_specified && operating_mode != field_mode)
     FATAL_ERROR (_("an input delimiter may be specified only\
  when operating on fields"));
 
@@ -863,15 +1216,34 @@ main (int argc, char **argv)
     }
 
   if (!delim_specified)
-    delim = '\t';
+    {
+      delim = '\t';
+#ifdef HAVE_MBRTOWC
+      wcdelim = L'\t';
+      mbdelim[0] = '\t';
+      mbdelim[1] = '\0';
+      delimlen = 1;
+#endif
+    }
 
   if (output_delimiter_string == NULL)
     {
-      static char dummy[2];
-      dummy[0] = delim;
-      dummy[1] = '\0';
-      output_delimiter_string = dummy;
-      output_delimiter_length = 1;
+#ifdef HAVE_MBRTOWC
+      if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
+        {
+          output_delimiter_string = xstrdup(mbdelim);
+          output_delimiter_length = delimlen;
+        }
+
+      if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
+#endif
+        {
+          static char dummy[2]; 
+          dummy[0] = delim;
+          dummy[1] = '\0';
+          output_delimiter_string = dummy;
+          output_delimiter_length = 1;
+        }
     }
 
   if (optind == argc)
diff -urNp coreutils-8.0-orig/src/expand.c coreutils-8.0/src/expand.c
--- coreutils-8.0-orig/src/expand.c	2009-09-29 15:27:54.000000000 +0200
+++ coreutils-8.0/src/expand.c	2009-10-07 10:07:16.000000000 +0200
@@ -37,11 +37,28 @@
 #include <stdio.h>
 #include <getopt.h>
 #include <sys/types.h>
+
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
 #include "system.h"
 #include "error.h"
 #include "quote.h"
 #include "xstrndup.h"
 
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+   installation; work around this configuration error.  */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "expand"
 
@@ -357,6 +374,142 @@ expand (void)
     }
 }
 
+#if HAVE_MBRTOWC
+static void
+expand_multibyte (void)
+{
+  FILE *fp;			/* Input strem. */
+  mbstate_t i_state;		/* Current shift state of the input stream. */
+  mbstate_t i_state_bak;	/* Back up the I_STATE. */
+  mbstate_t o_state;		/* Current shift state of the output stream. */
+  char buf[MB_LEN_MAX + BUFSIZ];  /* For spooling a read byte sequence. */
+  char *bufpos;			/* Next read position of BUF. */
+  size_t buflen = 0;		/* The length of the byte sequence in buf. */
+  wchar_t wc;			/* A gotten wide character. */
+  size_t mblength;		/* The byte size of a multibyte character
+				   which shows as same character as WC. */
+  int tab_index = 0;		/* Index in `tab_list' of next tabstop. */
+  int column = 0;		/* Column on screen of the next char. */
+  int next_tab_column;		/* Column the next tab stop is on. */
+  int convert = 1;		/* If nonzero, perform translations. */
+
+  fp = next_file ((FILE *) NULL);
+  if (fp == NULL)
+    return;
+
+  memset (&o_state, '\0', sizeof(mbstate_t));
+  memset (&i_state, '\0', sizeof(mbstate_t));
+
+  for (;;)
+    {
+      /* Refill the buffer BUF. */
+      if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp))
+	{
+	  memmove (buf, bufpos, buflen);
+	  buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp);
+	  bufpos = buf;
+	}
+
+      /* No character is left in BUF. */
+      if (buflen < 1)
+	{
+	  fp = next_file (fp);
+
+	  if (fp == NULL)
+	    break;		/* No more files. */
+	  else
+	    {
+	      memset (&i_state, '\0', sizeof(mbstate_t));
+	      continue;
+	    }
+	}
+
+      /* Get a wide character. */
+      i_state_bak = i_state;
+      mblength = mbrtowc (&wc, bufpos, buflen, &i_state);
+
+      switch (mblength)
+	{
+	case (size_t)-1:	/* illegal byte sequence. */
+	case (size_t)-2:
+	  mblength = 1;
+	  i_state = i_state_bak;
+	  if (convert)
+	    {
+	      ++column;
+	      if (convert_entire_line == 0)
+		convert = 0;
+	    }
+	  putchar (*bufpos);
+	  break;
+
+	case 0:		/* null. */
+	  mblength = 1;
+	  if (convert && convert_entire_line == 0)
+	    convert = 0;
+	  putchar ('\0');
+	  break;
+
+	default:
+	  if (wc == L'\n')   /* LF. */
+	    {
+	      tab_index = 0;
+	      column = 0;
+	      convert = 1;
+	      putchar ('\n');
+	    }
+	  else if (wc == L'\t' && convert)	/* Tab. */
+	    {
+	      if (tab_size == 0)
+		{
+		  /* Do not let tab_index == first_free_tab;
+		     stop when it is 1 less. */
+		  while (tab_index < first_free_tab - 1
+		      && column >= tab_list[tab_index])
+		    tab_index++;
+		  next_tab_column = tab_list[tab_index];
+		  if (tab_index < first_free_tab - 1)
+		    tab_index++;
+		  if (column >= next_tab_column)
+		    next_tab_column = column + 1;
+		}
+	      else
+		next_tab_column = column + tab_size - column % tab_size;
+
+	      while (column < next_tab_column)
+		{
+		  putchar (' ');
+		  ++column;
+		}
+	    }
+	  else  /* Others. */
+	    {
+	      if (convert)
+		{
+		  if (wc == L'\b')
+		    {
+		      if (column > 0)
+			--column;
+		    }
+		  else
+		    {
+		      int width;		/* The width of WC. */
+
+		      width = wcwidth (wc);
+		      column += (width > 0) ? width : 0;
+		      if (convert_entire_line == 0)
+			convert = 0;
+		    }
+		}
+	      fwrite (bufpos, sizeof(char), mblength, stdout);
+	    }
+	}
+      buflen -= mblength;
+      bufpos += mblength;
+    }
+}
+#endif
+
 int
 main (int argc, char **argv)
 {
@@ -421,7 +574,12 @@ main (int argc, char **argv)
 
   file_list = (optind < argc ? &argv[optind] : stdin_argv);
 
-  expand ();
+#if HAVE_MBRTOWC
+  if (MB_CUR_MAX > 1)
+    expand_multibyte ();
+  else
+#endif
+    expand ();
 
   if (have_read_stdin && fclose (stdin) != 0)
     error (EXIT_FAILURE, errno, "-");
diff -urNp coreutils-8.0-orig/src/fold.c coreutils-8.0/src/fold.c
--- coreutils-8.0-orig/src/fold.c	2009-09-23 10:25:44.000000000 +0200
+++ coreutils-8.0/src/fold.c	2009-10-07 10:07:16.000000000 +0200
@@ -22,11 +22,33 @@
 #include <getopt.h>
 #include <sys/types.h>
 
+/* Get mbstate_t, mbrtowc(), wcwidth().  */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get iswprint(), iswblank(), wcwidth().  */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
 #include "system.h"
 #include "error.h"
 #include "quote.h"
 #include "xstrtol.h"
 
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+      installation; work around this configuration error.  */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# undef MB_LEN_MAX
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
 #define TAB_WIDTH 8
 
 /* The official name of this program (e.g., no `g' prefix).  */
@@ -34,20 +56,41 @@
 
 #define AUTHORS proper_name ("David MacKenzie")
 
+#define FATAL_ERROR(Message)                                            \
+  do                                                                    \
+    {                                                                   \
+      error (0, 0, (Message));                                          \
+      usage (2);                                                        \
+    }                                                                   \
+  while (0)
+
+enum operating_mode
+{
+  /* Fold texts by columns that are at the given positions. */
+  column_mode,
+
+  /* Fold texts by bytes that are at the given positions. */
+  byte_mode,
+
+  /* Fold texts by characters that are at the given positions. */
+  character_mode,
+};
+
+/* The argument shows current mode. (Default: column_mode) */
+static enum operating_mode operating_mode;
+
 /* If nonzero, try to break on whitespace. */
 static bool break_spaces;
 
-/* If nonzero, count bytes, not column positions. */
-static bool count_bytes;
-
 /* If nonzero, at least one of the files we read was standard input. */
 static bool have_read_stdin;
 
-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::";
+static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::";
 
 static struct option const longopts[] =
 {
   {"bytes", no_argument, NULL, 'b'},
+  {"characters", no_argument, NULL, 'c'},
   {"spaces", no_argument, NULL, 's'},
   {"width", required_argument, NULL, 'w'},
   {GETOPT_HELP_OPTION_DECL},
@@ -77,6 +120,7 @@ Mandatory arguments to long options are 
 "), stdout);
       fputs (_("\
   -b, --bytes         count bytes rather than columns\n\
+  -c, --characters    count characters rather than columns\n\
   -s, --spaces        break at spaces\n\
   -w, --width=WIDTH   use WIDTH columns instead of 80\n\
 "), stdout);
@@ -94,7 +138,7 @@ Mandatory arguments to long options are 
 static size_t
 adjust_column (size_t column, char c)
 {
-  if (!count_bytes)
+  if (operating_mode != byte_mode)
     {
       if (c == '\b')
         {
@@ -117,30 +161,14 @@ adjust_column (size_t column, char c)
    to stdout, with maximum line length WIDTH.
    Return true if successful.  */
 
-static bool
-fold_file (char const *filename, size_t width)
+static void
+fold_text (FILE *istream, size_t width, int *saved_errno)
 {
-  FILE *istream;
   int c;
   size_t column = 0;		/* Screen column where next char will go. */
   size_t offset_out = 0;	/* Index in `line_out' for next char. */
   static char *line_out = NULL;
   static size_t allocated_out = 0;
-  int saved_errno;
-
-  if (STREQ (filename, "-"))
-    {
-      istream = stdin;
-      have_read_stdin = true;
-    }
-  else
-    istream = fopen (filename, "r");
-
-  if (istream == NULL)
-    {
-      error (0, errno, "%s", filename);
-      return false;
-    }
 
   while ((c = getc (istream)) != EOF)
     {
@@ -168,6 +196,15 @@ fold_file (char const *filename, size_t 
               bool found_blank = false;
               size_t logical_end = offset_out;
 
+              /* If LINE_OUT has no wide character,
+                 put a new wide character in LINE_OUT
+                 if column is bigger than width. */
+              if (offset_out == 0)
+                {
+                  line_out[offset_out++] = c;
+                  continue;
+                }
+
               /* Look for the last blank. */
               while (logical_end)
                 {
@@ -214,11 +251,222 @@ fold_file (char const *filename, size_t 
       line_out[offset_out++] = c;
     }
 
-  saved_errno = errno;
+  *saved_errno = errno;
 
   if (offset_out)
     fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
 
+}
+
+#if HAVE_MBRTOWC
+static void
+fold_multibyte_text (FILE *istream, size_t width, int *saved_errno)
+{
+  char buf[MB_LEN_MAX + BUFSIZ];  /* For spooling a read byte sequence. */
+  size_t buflen = 0;        /* The length of the byte sequence in buf. */
+  char *bufpos = NULL;         /* Next read position of BUF. */
+  wint_t wc;                /* A gotten wide character. */
+  size_t mblength;        /* The byte size of a multibyte character which shows
+                           as same character as WC. */
+  mbstate_t state, state_bak;        /* State of the stream. */
+  int convfail;                /* 1, when conversion is failed. Otherwise 0. */
+
+  static char *line_out = NULL;
+  size_t offset_out = 0;        /* Index in `line_out' for next char. */
+  static size_t allocated_out = 0;
+
+  int increment;
+  size_t column = 0;
+
+  size_t last_blank_pos;
+  size_t last_blank_column;
+  int is_blank_seen;
+  int last_blank_increment = 0;
+  int is_bs_following_last_blank;
+  size_t bs_following_last_blank_num;
+  int is_cr_after_last_blank;
+
+#define CLEAR_FLAGS                                \
+   do                                                \
+     {                                                \
+        last_blank_pos = 0;                        \
+        last_blank_column = 0;                        \
+        is_blank_seen = 0;                        \
+        is_bs_following_last_blank = 0;                \
+        bs_following_last_blank_num = 0;        \
+        is_cr_after_last_blank = 0;                \
+     }                                                \
+   while (0)
+
+#define START_NEW_LINE                        \
+   do                                        \
+     {                                        \
+      putchar ('\n');                        \
+      column = 0;                        \
+      offset_out = 0;                        \
+      CLEAR_FLAGS;                        \
+    }                                        \
+   while (0)
+
+  CLEAR_FLAGS;
+  memset (&state, '\0', sizeof(mbstate_t));
+
+  for (;; bufpos += mblength, buflen -= mblength)
+    {
+      if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream))
+        {
+          memmove (buf, bufpos, buflen);
+          buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream);
+          bufpos = buf;
+        }
+
+      if (buflen < 1)
+        break;
+
+      /* Get a wide character. */
+      convfail = 0;
+      state_bak = state;
+      mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state);
+
+      switch (mblength)
+        {
+        case (size_t)-1:
+        case (size_t)-2:
+          convfail++;
+          state = state_bak;
+          /* Fall through. */
+
+        case 0:
+          mblength = 1;
+          break;
+        }
+
+rescan:
+      if (operating_mode == byte_mode)                        /* byte mode */
+        increment = mblength;
+      else if (operating_mode == character_mode)        /* character mode */
+        increment = 1;
+      else                                                /* column mode */
+        {
+          if (convfail)
+            increment = 1;
+          else
+            {
+              switch (wc)
+                {
+                case L'\n':
+                  fwrite (line_out, sizeof(char), offset_out, stdout);
+                  START_NEW_LINE;
+                  continue;
+                  
+                case L'\b':
+                  increment = (column > 0) ? -1 : 0;
+                  break;
+
+                case L'\r':
+                  increment = -1 * column;
+                  break;
+
+                case L'\t':
+                  increment = 8 - column % 8;
+                  break;
+
+                default:
+                  increment = wcwidth (wc);
+                  increment = (increment < 0) ? 0 : increment;
+                }
+            }
+        }
+
+      if (column + increment > width && break_spaces && last_blank_pos)
+        {
+          fwrite (line_out, sizeof(char), last_blank_pos, stdout);
+          putchar ('\n');
+
+          offset_out = offset_out - last_blank_pos;
+          column = column - last_blank_column + ((is_cr_after_last_blank)
+              ? last_blank_increment : bs_following_last_blank_num);
+          memmove (line_out, line_out + last_blank_pos, offset_out);
+          CLEAR_FLAGS;
+          goto rescan;
+        }
+
+      if (column + increment > width && column != 0)
+        {
+          fwrite (line_out, sizeof(char), offset_out, stdout);
+          START_NEW_LINE;
+          goto rescan;
+        }
+
+      if (allocated_out < offset_out + mblength)
+        {
+          line_out = X2REALLOC (line_out, &allocated_out);
+        }
+
+      memcpy (line_out + offset_out, bufpos, mblength);
+      offset_out += mblength;
+      column += increment;
+
+      if (is_blank_seen && !convfail && wc == L'\r')
+        is_cr_after_last_blank = 1;
+
+      if (is_bs_following_last_blank && !convfail && wc == L'\b')
+        ++bs_following_last_blank_num;
+      else
+        is_bs_following_last_blank = 0;
+
+      if (break_spaces && !convfail && iswblank (wc))
+        {
+          last_blank_pos = offset_out;
+          last_blank_column = column;
+          is_blank_seen = 1;
+          last_blank_increment = increment;
+          is_bs_following_last_blank = 1;
+          bs_following_last_blank_num = 0;
+          is_cr_after_last_blank = 0;
+        }
+    }
+
+  *saved_errno = errno;
+
+  if (offset_out)
+    fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
+
+}
+#endif
+
+/* Fold file FILENAME, or standard input if FILENAME is "-",
+   to stdout, with maximum line length WIDTH.
+   Return 0 if successful, 1 if an error occurs. */
+
+static bool
+fold_file (char *filename, size_t width)
+{
+  FILE *istream;
+  int saved_errno;
+
+  if (STREQ (filename, "-"))
+    {
+      istream = stdin;
+      have_read_stdin = 1;
+    }
+  else
+    istream = fopen (filename, "r");
+
+  if (istream == NULL)
+    {
+      error (0, errno, "%s", filename);
+      return 1;
+    }
+
+  /* Define how ISTREAM is being folded. */
+#if HAVE_MBRTOWC
+  if (MB_CUR_MAX > 1)
+    fold_multibyte_text (istream, width, &saved_errno);
+  else
+#endif
+    fold_text (istream, width, &saved_errno);
+
   if (ferror (istream))
     {
       error (0, saved_errno, "%s", filename);
@@ -251,7 +499,8 @@ main (int argc, char **argv)
 
   atexit (close_stdout);
 
-  break_spaces = count_bytes = have_read_stdin = false;
+  operating_mode = column_mode;
+  break_spaces = have_read_stdin = false;
 
   while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1)
     {
@@ -260,7 +509,15 @@ main (int argc, char **argv)
       switch (optc)
         {
         case 'b':		/* Count bytes rather than columns. */
-          count_bytes = true;
+          if (operating_mode != column_mode)
+            FATAL_ERROR (_("only one way of folding may be specified"));
+          operating_mode = byte_mode;
+          break;
+
+        case 'c':
+          if (operating_mode != column_mode)
+            FATAL_ERROR (_("only one way of folding may be specified"));
+          operating_mode = character_mode;
           break;
 
         case 's':		/* Break at word boundaries. */
diff -urNp coreutils-8.0-orig/src/join.c coreutils-8.0/src/join.c
--- coreutils-8.0-orig/src/join.c	2009-09-23 10:25:44.000000000 +0200
+++ coreutils-8.0/src/join.c	2009-10-07 10:07:16.000000000 +0200
@@ -22,17 +22,31 @@
 #include <sys/types.h>
 #include <getopt.h>
 
+/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth().  */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get iswblank(), towupper.  */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
 #include "system.h"
 #include "error.h"
 #include "hard-locale.h"
 #include "linebuffer.h"
-#include "memcasecmp.h"
 #include "quote.h"
 #include "stdio--.h"
 #include "xmemcoll.h"
 #include "xstrtol.h"
 #include "argmatch.h"
 
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "join"
 
@@ -121,10 +135,12 @@ static struct outlist outlist_head;
 /* Last element in `outlist', where a new element can be added.  */
 static struct outlist *outlist_end = &outlist_head;
 
-/* Tab character separating fields.  If negative, fields are separated
-   by any nonempty string of blanks, otherwise by exactly one
-   tab character whose value (when cast to unsigned char) equals TAB.  */
-static int tab = -1;
+/* Tab character separating fields.  If NULL, fields are separated
+   by any nonempty string of blanks.  */
+static char *tab = NULL;
+
+/* The number of bytes used for tab. */
+static size_t tablen = 0;
 
 /* If nonzero, check that the input is correctly ordered. */
 static enum
@@ -239,10 +255,11 @@ xfields (struct line *line)
   if (ptr == lim)
     return;
 
-  if (0 <= tab)
+  if (tab != NULL)
     {
+      unsigned char t = tab[0];
       char *sep;
-      for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1)
+      for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1)
         extract_field (line, ptr, sep - ptr);
     }
   else
@@ -269,6 +286,148 @@ xfields (struct line *line)
   extract_field (line, ptr, lim - ptr);
 }
 
+#if HAVE_MBRTOWC
+static void
+xfields_multibyte (struct line *line)
+{
+  char *ptr = line->buf.buffer;
+  char const *lim = ptr + line->buf.length - 1;
+  wchar_t wc = 0;
+  size_t mblength = 1;
+  mbstate_t state, state_bak;
+
+  memset (&state, 0, sizeof (mbstate_t));
+
+  if (ptr >= lim)
+    return;
+
+  if (tab != NULL)
+    {
+      unsigned char t = tab[0];
+      char *sep = ptr;
+      for (; ptr < lim; ptr = sep + mblength)
+	{
+	  sep = ptr;
+	  while (sep < lim)
+	    {
+	      state_bak = state;
+	      mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
+
+	      if (mblength == (size_t)-1 || mblength == (size_t)-2)
+		{
+		  mblength = 1;
+		  state = state_bak;
+		}