Re: [Translation-i18n] Handling qt-format in gettext tools (was: Updating qt-format handling?)

[Bruno Haible]

Chusslove Illich wrote:
> Hm, indeed. I would only use this wording for unterminated directives:
> 
>   Directives cannot be escaped, hence '%' or '%L' not followed by a digit
>   is not an error, but a verbatim segment of the text.

Yes. Thanks for this spec update.

> > So, effectively we have two kinds of Qt format strings: [...] The
> > translator must not give a translation of the second kind for an msgid of
> > the first kind.
> 
> How about only insisting on exact match for the directives? E.g. if there is
> %1 in the msgid, translator cannot use any of %L1, %01, or %L01, and other
> ways around.

It is possible the other way around.

> The only change in the format parser for this is to consider
> %0n as directives 100-109, so that they are different from %n.

This doesn't seem right. What's needed is an additional boolean in the
format_parse result.

> > I would appreciate a patch that takes into account the finding for 1. + 2.
> > mentioned above.
> 
> One with the two tweaks above is attached (also a few extra format tests for
> make check).

Thanks. I'm using a different .c code than yours, but I can reuse most of
your test cases.

> > [: Bruno Haible :]
> > As you can see, I haven't found the time to do these changes in 4 months.
> 
> No problem, KDE4 has been anyway delayed until the end of the year :)

Can the KDE team work with a locally patched gettext-0.16.1, or is a new
gettext release needed soon?

Bruno


2007-09-09  Bruno Haible  <address@hidden>

        Add support for Qt 4 format strings.
        * gettext-tools/src/format-qt.c (struct spec): Increase args_used
        field size to 100. Add 'simple' field.
        (format_parse): Update for Qt 4 syntax. Remove error message when the
        same format argument is used more than once.
        (format_check): Add check: If the msgid is simple, the msgstr must be
        simple as well.
        * gettext-tools/tests/format-qt-1: Update for Qt 4 syntax.
        * gettext-tools/tests/format-qt-2: Likewise.
        Based on input by by Chusslove Illich <address@hidden>.
        Reported by Chusslove Illich <address@hidden>.

*** NEWS        2 Sep 2007 09:58:56 -0000       1.121
--- NEWS        10 Sep 2007 00:24:26 -0000
***************
*** 4,10 ****
    be updated all together.
  
  * Programming languages support:
!   Contexts (msgctxt) are now also supported for Java and C#.
  
  * Documentation:
    The "Users" chapter has been completely rewritten.
--- 4,11 ----
    be updated all together.
  
  * Programming languages support:
!   - Contexts (msgctxt) are now also supported for Java and C#.
!   - C# with Qt: The support for Qt format strings has been updated for Qt 4.
  
  * Documentation:
    The "Users" chapter has been completely rewritten.
*** gettext-tools/src/format-qt.c       22 Aug 2007 23:11:56 -0000      1.9
--- gettext-tools/src/format-qt.c       10 Sep 2007 00:24:26 -0000
***************
*** 31,52 ****
  #define _(str) gettext (str)
  
  /* Qt format strings are processed by QString::arg and are documented in
!    qt-3.0.5/doc/html/qstring.html.
!    A directive starts with '%' and is followed by a digit ('0' to '9').
!    Each %n must occur only once in the given string.
     The first .arg() invocation replaces the %n with the lowest numbered n,
     the next .arg() invocation then replaces the %n with the second-lowest
     numbered n, and so on.
!    (This is inherently buggy because a '%' in the first replacement confuses
!    the second .arg() invocation.)
     Although %0 is supported, usually %1 denotes the first argument, %2 the
     second argument etc.  */
  
  struct spec
  {
    unsigned int directives;
    unsigned int arg_count;
!   bool args_used[10];
  };
  
  
--- 31,67 ----
  #define _(str) gettext (str)
  
  /* Qt format strings are processed by QString::arg and are documented in
!    qt-4.3.0/doc/html/qstring.html.
!    A directive
!      - starts with '%',
!      - is optionally followed by 'L' (indicates locale-dependent processing),
!      - is followed by one or two digits ('0' to '9'). %0n is equivalent to %n.
!    An unterminated directive ('%' or '%L' not followed by a digit or at the
!    end) is not an error.
     The first .arg() invocation replaces the %n with the lowest numbered n,
     the next .arg() invocation then replaces the %n with the second-lowest
     numbered n, and so on.
!    This is inherently buggy because a '%' in the first replacement confuses
!    the second .arg() invocation.
!    To reduce this problem and introduce another one, there are also .arg()
!    methods that take up to 9 strings and perform the replacements in one 
swoop.
!    But this method works only on strings that contain no 'L' flags and only
!    single-digit argument designators.
     Although %0 is supported, usually %1 denotes the first argument, %2 the
     second argument etc.  */
  
  struct spec
  {
+   /* Number of format directives.  */
    unsigned int directives;
+ 
+   /* True if the string supports the multi-argument .arg() methods, i.e. if it
+      contains no 'L' flags and only single-digit argument designators.  */
+   bool simple;
+ 
+   /* Booleans telling which %nn was seen.  */
    unsigned int arg_count;
!   bool args_used[100];
  };
  
  
***************
*** 59,100 ****
    struct spec *result;
  
    spec.directives = 0;
    spec.arg_count = 0;
  
    for (; *format != '\0';)
      if (*format++ == '%')
!       if (*format >= '0' && *format <= '9')
!       {
!         /* A directive.  */
!         unsigned int number;
! 
!         FDI_SET (format - 1, FMTDIR_START);
!         spec.directives++;
! 
!         number = *format - '0';
! 
!         while (spec.arg_count <= number)
!           spec.args_used[spec.arg_count++] = false;
!         if (spec.args_used[number])
!           {
!             *invalid_reason =
!               xasprintf (_("Multiple references to %%%c."), *format);
!             FDI_SET (format, FMTDIR_ERROR);
!             goto bad_format;
!           }
!         spec.args_used[number] = true;
! 
!         FDI_SET (format, FMTDIR_END);
! 
!         format++;
!       }
  
    result = XMALLOC (struct spec);
    *result = spec;
    return result;
- 
-  bad_format:
-   return NULL;
  }
  
  static void
--- 74,124 ----
    struct spec *result;
  
    spec.directives = 0;
+   spec.simple = true;
    spec.arg_count = 0;
  
    for (; *format != '\0';)
      if (*format++ == '%')
!       {
!       const char *dir_start = format - 1;
!       bool locale_flag = false;
! 
!       if (*format == 'L')
!         {
!           locale_flag = true;
!           format++;
!         }
!       if (*format >= '0' && *format <= '9')
!         {
!           /* A directive.  */
!           unsigned int number;
! 
!           FDI_SET (dir_start, FMTDIR_START);
!           spec.directives++;
!           if (locale_flag)
!             spec.simple = false;
! 
!           number = *format - '0';
!           if (format[1] >= '0' && format[1] <= '9')
!             {
!               number = 10 * number + (format[1] - '0');
!               spec.simple = false;
!               format++;
!             }
! 
!           while (spec.arg_count <= number)
!             spec.args_used[spec.arg_count++] = false;
!           spec.args_used[number] = true;
! 
!           FDI_SET (format, FMTDIR_END);
! 
!           format++;
!         }
!       }
  
    result = XMALLOC (struct spec);
    *result = spec;
    return result;
  }
  
  static void
***************
*** 123,147 ****
    bool err = false;
    unsigned int i;
  
!   for (i = 0; i < spec1->arg_count || i < spec2->arg_count; i++)
      {
!       bool arg_used1 = (i < spec1->arg_count && spec1->args_used[i]);
!       bool arg_used2 = (i < spec2->arg_count && spec2->args_used[i]);
! 
!       /* The translator cannot omit a %n from the msgstr because that would
!        yield a "Argument missing" warning at runtime.  */
!       if (arg_used1 != arg_used2)
!       {
!         if (error_logger)
!           error_logger (arg_used1
!                         ? _("a format specification for argument %u doesn't 
exist in '%s'")
!                         : _("a format specification for argument %u, as in 
'%s', doesn't exist in 'msgid'"),
!                         i, pretty_msgstr);
!         err = true;
!         break;
!       }
      }
  
    return err;
  }
  
--- 147,180 ----
    bool err = false;
    unsigned int i;
  
!   if (spec1->simple && !spec2->simple)
      {
!       if (error_logger)
!       error_logger (_("'msgid' is a simple format string, but '%s' is not: it 
contains an 'L' flag or a double-digit argument number"),
!                     pretty_msgstr);
!       err = true;
      }
  
+   if (!err)
+     for (i = 0; i < spec1->arg_count || i < spec2->arg_count; i++)
+       {
+       bool arg_used1 = (i < spec1->arg_count && spec1->args_used[i]);
+       bool arg_used2 = (i < spec2->arg_count && spec2->args_used[i]);
+ 
+       /* The translator cannot omit a %n from the msgstr because that would
+          yield a "Argument missing" warning at runtime.  */
+       if (arg_used1 != arg_used2)
+         {
+           if (error_logger)
+             error_logger (arg_used1
+                           ? _("a format specification for argument %u doesn't 
exist in '%s'")
+                           : _("a format specification for argument %u, as in 
'%s', doesn't exist in 'msgid'"),
+                           i, pretty_msgstr);
+           err = true;
+           break;
+         }
+       }
+ 
    return err;
  }
  
*** gettext-tools/tests/format-qt-1     28 Oct 2003 16:10:35 -0000      1.1
--- gettext-tools/tests/format-qt-1     10 Sep 2007 00:24:26 -0000
***************
*** 13,28 ****
  "abc%1def"
  # Valid: one argument
  "abc%9def"
  # Valid: unterminated
  "abc%1def%"
  # Valid: non-digit
  "abc%1def%x"
  # Valid: zero
  "abc%1def%0"
  # Valid: permutation
  "abc%2def%1"
! # Invalid: multiple uses of same argument
  "abc%2def%1ghi%2"
  EOF
  
  : ${XGETTEXT=xgettext}
--- 13,40 ----
  "abc%1def"
  # Valid: one argument
  "abc%9def"
+ # Valid: one argument specified by two digits
+ "abc%09def"
+ # Valid: one argument specified by two digits
+ "abc%99def"
  # Valid: unterminated
  "abc%1def%"
+ # Valid: unterminated
+ "abc%1def%L"
  # Valid: non-digit
  "abc%1def%x"
  # Valid: zero
  "abc%1def%0"
+ # Valid: zero specified by two digits
+ "abc%1def%00"
  # Valid: permutation
  "abc%2def%1"
! # Valid: multiple uses of same argument
  "abc%2def%1ghi%2"
+ # Valid: an argument with locale-dependency flag
+ "abc%L1def"
+ # Valid: an argument with locale-dependency flag and two digits
+ "abc%L12def"
  EOF
  
  : ${XGETTEXT=xgettext}
*** gettext-tools/tests/format-qt-2     28 Oct 2003 16:10:35 -0000      1.1
--- gettext-tools/tests/format-qt-2     10 Sep 2007 00:24:26 -0000
***************
*** 10,21 ****
  # Valid: %% doesn't count
  msgid  "abc%%def"
  msgstr "xyz"
- # Invalid: invalid msgstr
- msgid  "abc%1def"
- msgstr "xyz%1%1"
  # Valid: same arguments
  msgid  "abc%2def"
  msgstr "xyz%2"
  # Valid: permutation
  msgid  "abc%3%1%2def"
  msgstr "xyz%2%1%3"
--- 10,36 ----
  # Valid: %% doesn't count
  msgid  "abc%%def"
  msgstr "xyz"
  # Valid: same arguments
  msgid  "abc%2def"
  msgstr "xyz%2"
+ # Valid: same arguments, msgstr may be simpler than msgid
+ msgid  "abc%L2def"
+ msgstr "xyz%2"
+ # Valid: same arguments, msgstr may be simpler than msgid
+ msgid  "abc%02def"
+ msgstr "xyz%2"
+ # Invalid: msgid is simple but msgstr is not
+ msgid  "abc%2def"
+ msgstr "xyz%L2"
+ # Invalid: msgid is simple but msgstr is not
+ msgid  "abc%2def"
+ msgstr "xyz%02"
+ # Valid: repetition of an argument in the translation
+ msgid  "abc%2def"
+ msgstr "xyz%2uvw%2"
+ # Valid: removing repeated argument in the translation
+ msgid  "abc%2def%2"
+ msgstr "xyz%2uvw"
  # Valid: permutation
  msgid  "abc%3%1%2def"
  msgstr "xyz%2%1%3"