[GNUnet-SVN] r4200 - in i18nHTML: . src src/admin

[grothoff]

Author: grothoff
Date: 2007-01-05 00:38:53 -0800 (Fri, 05 Jan 2007)
New Revision: 4200

Added:
   i18nHTML/src/admin/cleanup.php
   i18nHTML/src/admin/dig.php
Modified:
   i18nHTML/src/admin/commitTranslation.php
   i18nHTML/src/admin/index.php
   i18nHTML/src/admin/status.php
   i18nHTML/src/admin/tables.php
   i18nHTML/src/update0_3_0.php
   i18nHTML/todo
Log:
stuff

Added: i18nHTML/src/admin/cleanup.php
===================================================================
--- i18nHTML/src/admin/cleanup.php      2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/admin/cleanup.php      2007-01-05 08:38:53 UTC (rev 4200)
@@ -0,0 +1,28 @@
+<?php
+/*
+     (C) 2007 Christian Grothoff
+
+     This code is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 2, or (at your
+     option) any later version.
+
+     The code is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with the code; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+include_once("i18nHTML/i18nhtml.php");
+
+// delete empty translations
+$query = "DELETE FROM ${i18nHTMLsqlPrefix}map WHERE text=\"\"";
+mysql_query($query, $connection);
+$query = "DELETE FROM ${i18nHTMLsqlPrefix}map WHERE translation=\"\"";
+mysql_query($query, $connection);
+echo "Done.";
+?>
\ No newline at end of file

Modified: i18nHTML/src/admin/commitTranslation.php
===================================================================
--- i18nHTML/src/admin/commitTranslation.php    2007-01-04 23:38:59 UTC (rev 
4199)
+++ i18nHTML/src/admin/commitTranslation.php    2007-01-05 08:38:53 UTC (rev 
4200)
@@ -1,6 +1,6 @@
 <?php
 /*
-     (C) 2003, 2004, 2005 Christian Grothoff
+     (C) 2003, 2004, 2005, 2007 Christian Grothoff
 
      This code is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -24,7 +24,7 @@
   die();
 }
 if ($lang == $i18nHTMLsrcLang) {
-  W("Translating to " . $i18nHTMLsrcLang . " is not allowed.\n");
+  W("Translating to ${i18nHTMLsrcLang} is not allowed.\n");
   die();
 }
 if (strstr($lang, "@")) {
@@ -38,8 +38,7 @@
 
 $back            = $_REQUEST['back'];
 // check for identical translation
-$query = "SELECT ranking,translation FROM " . $i18nHTMLsqlPrefix . 
-         "map WHERE name=\"$text_sql\" AND lang=\"$lang\"";
+$query = "SELECT ranking,translation FROM ${i18nHTMLsqlPrefix}map WHERE 
name=\"$text_sql\" AND lang=\"$lang\"";
 
 $result = mysql_query($query, $connection);
 $num = 0;
@@ -78,12 +77,11 @@
       generateFooter();
       echo "</body></html>";
   } else {
-    $query = "INSERT INTO " . $i18nHTMLsqlPrefix . "map VALUES(\"$text_sql\", 
\"$lang\", " .
+    $query = "INSERT INTO ${i18nHTMLsqlPrefix}map VALUES(\"$text_sql\", 
\"$lang\", " .
              "\"$translation_sql\", $max+1, \"" . $_SERVER['REMOTE_ADDR'] . 
"\", $uid);";
     $result = mysql_query($query, $connection);
     if ($result) {
-      $query = "DELETE FROM " . $i18nHTMLsqlPrefix . 
-               "pending WHERE lang=\"$lang\" AND c=\"$text_sql\"";
+      $query = "DELETE FROM ${i18nHTMLsqlPrefix}pending WHERE lang=\"$lang\" 
AND c=\"$text_sql\"";
       mysql_query($query, $connection);
       header("Location: " . $back); /* Redirect browser */
     } else {

Added: i18nHTML/src/admin/dig.php
===================================================================
--- i18nHTML/src/admin/dig.php  2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/admin/dig.php  2007-01-05 08:38:53 UTC (rev 4200)
@@ -0,0 +1,148 @@
+<?php
+/*
+     (C) 2007 Christian Grothoff
+
+     This code is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 2, or (at your
+     option) any later version.
+
+     The code is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with the code; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+  /*
+   Ideas for additional diggers:
+   - punctuation mismatch (especially ".!?#")
+   - translation count of UID
+  */
+  /* 
+   TODO:
+   - actually allow deletion of translations (RC)
+   - filter translations of higher-level UIDs
+   - disallow page for 0-level UIDs (guard resources)
+  */
+
+include_once("i18nHTML/i18nhtml.php");
+include_once("login.php");
+$mode = $_REQUEST['mode'];
+DOCTYPE("HTML", "Transitional");
+echo "<html><head>\n";
+TITLE("i18nHTML administration: searching for bad translations");
+echo "</head><body>";
+BP();
+W("Using heuristic %s.", $mode);
+EP();
+
+$data = ARRAY();
+$query = "SELECT name,translation,uid FROM ${i18nHTMLsqlPrefix}map WHERE 
lang=\"${lang}\" ORDER BY name";
+$result = mysql_query($query, $connection);
+$num = 0;
+$alpha = 0;
+if ($result)
+  $num = mysql_numrows($result);
+while ($num-- > 0) {
+  $row = mysql_fetch_array($result);
+  $text = stripslashes($row["name"]);
+  $translation = stripslashes($row["translation"]);
+  $uid = $row["uid"];
+
+  $entry = "<tr><td><table border=0><tr><td>&quot;$text&quot;</td></tr>\n" . 
+           "<tr><td>&quot;$translation&quot;</td></tr></table></td>\n" . 
+           "<td>$uid</td>\n" . // todo: add link to delete account / etc
+           "<td></td></tr>";   // todo: add checkbox in form to delete 
translation
+  $score = 0;
+  switch($mode) {
+  case "wwcd": // weighted word-count distance
+    if ( ($text == "") || ($translation == "") ) {
+      $score = 999999;
+    } else {
+      $wc1 = count(explode($text, " "));
+      $wc2 = count(explode($translation, " "));
+      $dist = $wc1 - $wc2;
+      if ($wc1 == 0)
+       $wc1 = 1; // avoid division by zero
+      $score = intval(1000 * ($dist * $dist) / $wc1);
+    }
+    break;
+  case "wccd": // weighted character-count distance
+    $wc1 = strlen($text);
+    $wc2 = strlen($translation);
+    $dist = $wc1 - $wc2;
+    if ($wc1 == 0)
+      $wc1 = 1; // avoid division by zero
+    $score = intval(1000 * ($dist * $dist) / $wc1);
+    break;
+  case "wlmc": // weighted average word-length mismatch
+    if ( ($text == "") || ($translation == "") ) {
+      $score = 999999;
+    } else {
+      $e1 = explode($text, " ");
+      $e2 = explode($translation, " ");
+      $l1 = array_map("strlen", $e1);
+      $l2 = array_map("strlen", $e2);
+      $t1 = array_sum($l1);
+      $t2 = array_sum($l2);
+      $a1 = $t1 / count($e1);
+      $a2 = $t2 / count($e2);
+      $dist = $a1 - $a2;
+      $score = intval(1000 * ($dist * $dist));
+    }
+    break;
+  case "psmc": // percent sign mismatch counts
+    if ( ($text == "") || ($translation == "") ) {
+      $score = 999999;
+    } else {
+      $e1 = explode($text, "%");
+      $e2 = explode($translation, "%");
+      $total = 0;    
+      for ($pos=0;$pos<count($e1)-1;$pos++) {
+       $t1 = $e1[$pos];
+       if (! isset($e2[$pos])) {
+         $total++;
+         continue;
+       }
+       $t2 = $e2[$pos];
+       // first character after "%" should match
+       if ($t1[0] != $t2[0])
+         $total++;
+      }
+      $score = $total;
+    }
+    break;
+  case "alpha":
+  default:
+    $score = $alpha++; // alphabetical ordering
+    break;
+  }
+  if (isset($data[$score]))
+    $entries = $data[$score];
+  else
+    $entries = ARRAY();
+  $entries[] = $entry;
+  $data[$score] = $entries;
+}
+
+H2("Translations to ${lang}");
+echo "<table border='5'>\n";
+echo "<tr>";
+TH("Original and Translation");
+TH("Translator");
+TH("Delete");
+echo "</tr>\n";
+foreach ($data as $score => $entries) {
+  foreach ($entries as $entry) {
+    echo $entry;
+  }
+}
+echo "</table>";
+HR();
+generateFooter();
+echo "</body></html>";
+?>
\ No newline at end of file

Modified: i18nHTML/src/admin/index.php
===================================================================
--- i18nHTML/src/admin/index.php        2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/admin/index.php        2007-01-05 08:38:53 UTC (rev 4200)
@@ -32,12 +32,13 @@
 
 echo "<ul>\n";
 LILI("editor.php", "Go to mass translation");
-// search for suspicious translations
-// LILI("dig.php", "Search for suspicious translations");
 
-// allow admins to delete translations of lower-level
-// accounts
-// LILI("delete.php", "Administer translations");
+LILI("dig.php?mode=wwcd&xlang=$lang&f=", "Search for suspicious translations 
using weighted word-count distance");
+LILI("dig.php?mode=wccd&xlang=$lang&f=", "Search for suspicious translations 
using weighted character-count distance");
+LILI("dig.php?mode=wlmc&xlang=$lang&f=", "Search for suspicious translations 
using weighted word-length distance");
+LILI("dig.php?mode=psmc&xlang=$lang&f=", "Search for suspicious translations 
using percent sign mismatch counts");
+LILI("dig.php?mode=alpha&xlang=$lang&f=", "Search for suspicious translations 
using alphabetical ordering");
+
 echo "</ul>\n";
 
 H2("Account management");

Modified: i18nHTML/src/admin/status.php
===================================================================
--- i18nHTML/src/admin/status.php       2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/admin/status.php       2007-01-05 08:38:53 UTC (rev 4200)
@@ -35,13 +35,12 @@
 W("Language setting is %s.",
   $lang);
 P();
-$query = "SELECT count FROM " . $i18nHTMLsqlPrefix . 
-         "pending WHERE lang=\"" . $i18nHTMLsrcLang . "\"";
+$query = "SELECT count FROM ${i18nHTMLsqlPrefix}pending WHERE 
lang=\"${i18nHTMLsrcLang}\"";
 $result = mysql_query($query, $connection);
 $num = 0;
 if ($result)
   $num = mysql_numrows($result);
-W("Sentences in the original " . $i18nHTMLsrcLang . " text: %s",
+W("Sentences in the original ${i18nHTMLsrcLang} text: %s",
   $num);
 EP();
 if ($mode == 1) {  
@@ -55,7 +54,7 @@
   echo "</tr>\n";
 }
 $stats = ARRAY();
-$query = "SELECT name,ranking,translation,lang FROM " . $i18nHTMLsqlPrefix . 
"map";
+$query = "SELECT name,ranking,translation,lang FROM ${i18nHTMLsqlPrefix}map";
 $result = mysql_query($query, $connection);
 $num = 0;
 if ($result)
@@ -94,9 +93,6 @@
   printf("<tr><td>%s</td><td>%s</td></tr>\n", $a, $b);
 }
 echo "</table></center>\n";
-BP();
-intlink("index.php", "i18nHTML administration index");
-EP();
 HR();
 generateFooter();
 echo "</body></html>";

Modified: i18nHTML/src/admin/tables.php
===================================================================
--- i18nHTML/src/admin/tables.php       2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/admin/tables.php       2007-01-05 08:38:53 UTC (rev 4200)
@@ -46,23 +46,34 @@
 // setup root account
 $query="INSERT INTO ${i18nHTMLsqlPrefix}accounts VALUES (\"root\", \"" . 
mysql_real_escape_string(crypt($i18nHTMLsqlAdminPass, "salty")) . "\", 
\"Administrator\", \"$i18nHTMLsrcLang\", 9999, 1)";
        "INDEX(c(10)), INDEX(lang(4)), INDEX(count))";
-echo $query;
 mysql_query($query, $connection);
 
 // setup legacy account
 $query="INSERT INTO ${i18nHTMLsqlPrefix}accounts VALUES (\"legacy\", 
\"nologin\", \"Legacy\", \"$i18nHTMLsrcLang\", 0, 2)";
        "INDEX(c(10)), INDEX(lang(4)), INDEX(count))";
-echo $query;
 mysql_query($query, $connection);
 
 
 $query="CREATE TABLE IF NOT EXISTS ${i18nHTMLsqlPrefix}languages" .
-  " (lang TINYBLOB, dodef INT, INDEX(lang(10)))";
+  " (lang VARCHAR(64) PRIMARY KEY, dodef INT, INDEX(lang(10)))";
 $result = mysql_query($query, $connection);
 if (!$result) {
   die('Unable to initialize languages table.  Invalid query: ' . 
mysql_error());
 }
 
+// add existing languages to language table
+$query="SELECT DISTINCT lang FROM ${i18nHTMLsqlPrefix}map;";
+$result = mysql_query($query, $connection);
+$num = 0;
+if ($result)
+  $num = mysql_numrows($result);
+while ($num-- > 0) {
+  $row = mysql_fetch_array($result);
+  $lx = $row['lang'];
+  $query="INSERT INTO ${i18nHTMLsqlPrefix}languages VALUES (\"$lx\", 0)";
+  mysql_query($query, $connection);
+ }
+
 W("Tables created.");
 generateFooter();
 echo "</body></html>";

Modified: i18nHTML/src/update0_3_0.php
===================================================================
--- i18nHTML/src/update0_3_0.php        2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/src/update0_3_0.php        2007-01-05 08:38:53 UTC (rev 4200)
@@ -6,7 +6,7 @@
 
 // alter map, adding UID field
 # UID 2 == legacy
-$query="ALTER TABLE ${i18nHTMLsqlPrefix}map ADD uid DEFAULT 2 INT AFTER ip;";
+$query="ALTER TABLE ${i18nHTMLsqlPrefix}map ADD uid INT DEFAULT 2 AFTER ip;";
 $result = mysql_query($query, $connection);
 if (!$result) {
   die('Unable to initialize map table.  Invalid query: ' . mysql_error());

Modified: i18nHTML/todo
===================================================================
--- i18nHTML/todo       2007-01-04 23:38:59 UTC (rev 4199)
+++ i18nHTML/todo       2007-01-05 08:38:53 UTC (rev 4200)
@@ -3,11 +3,11 @@
 
 Translators:
 * allow high-level users to delete translations [HIGH]
-* bogus translation detection algorithm [HIGH]
+* test bogus translation detection algorithm [HIGH]
 
 Users:
 * change voting from GET to POST (status.php, translate.php) [HIGH]
-* expand on-line documentation [LOW]
+* expand on-line documentation [MED]
 
 Installation:
 * finish debian package:  [HIGH]