Re: Encoding changes for Python 3 (issue 571280043 by address@hidden)

[jonas . hahnfeld]

Reviewers: lemzwerg,

Message:
Closing as the change got smaller and smaller as I discovered more edge
cases where the current code matters with Python 2.x

Description:
Encoding changes for Python 3

This is required to eventually make the scripts work with Python 3.
The change happens to also work with Python 2.7 (and maybe also 2.6),
but most certainly not with Python 2.5 or older supported up to now.

Please review this at https://codereview.appspot.com/571280043/

Affected files (+16, -27 lines):
  M python/lilylib.py
  M python/midi.py
  M python/musicexp.py
  M python/musicxml.py
  M python/utilities.py
  M scripts/midi2ly.py
  M scripts/musicxml2ly.py


Index: python/lilylib.py
diff --git a/python/lilylib.py b/python/lilylib.py
index 
821d88ce4e68c40f00d2f597bb3637f41b16c52a..114e48877c28e988b5b100767feb1103505f0270
 100644
--- a/python/lilylib.py
+++ b/python/lilylib.py
@@ -17,6 +17,7 @@
 # along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
 
 import __main__
+import codecs
 import glob
 import os
 import re
@@ -25,6 +26,10 @@ import sys
 import optparse
 import time
 
+# Reload sys to get sys.setdefaultencoding back.
+reload(sys)
+sys.setdefaultencoding ('utf-8')
+
 ################################################################
 # Users of python modules should include this snippet
 # and customize variables below.
@@ -50,23 +55,9 @@ except:
         return p
 underscore = _
 
-# Urg, Python 2.4 does not define stderr/stdout encoding
-# Maybe guess encoding from LANG/LC_ALL/LC_CTYPE?
-
-reload (sys)
-sys.setdefaultencoding ('utf-8')
-import codecs
-sys.stdout = codecs.getwriter ('utf8') (sys.stdout)
-sys.stderr = codecs.getwriter ('utf8') (sys.stderr)
-
 def encoded_write(f, s):
     f.write (s.encode (f.encoding or 'utf-8', 'replace'))
 
-# ugh, Python 2.5 optparse requires Unicode strings in some argument
-# functions, and refuse them in some other places
-def display_encode (s):
-    return s.encode (sys.stderr.encoding or 'utf-8', 'replace')
-
 # Lilylib globals.
 program_version = '@TOPLEVEL_VERSION@'
 program_name = os.path.basename (sys.argv[0])
@@ -118,7 +109,7 @@ def is_verbose ():
     return is_loglevel ("DEBUG")
 
 def stderr_write (s):
-    encoded_write (sys.stderr, s)
+    sys.stderr.write (s)
 
 def print_logmessage (level, s, fullmessage = True, newline = True):
     if (is_loglevel (level)):
Index: python/midi.py
diff --git a/python/midi.py b/python/midi.py
index 
f4d17eae71ce918420658da50fa358bdad6d072e..1eca6c11a1921c2e37aac85be55e93e591bf0182
 100644
--- a/python/midi.py
+++ b/python/midi.py
@@ -176,19 +176,19 @@ def _parse_tracks (midi, pos, num_tracks, clocks_max):
     if num_tracks > 256:
         raise error('too many tracks: %d' % num_tracks)
     for i in xrange(num_tracks):
-        trackdata, pos = _parse_hunk (midi, pos, 'track', 'MTrk')
+        trackdata, pos = _parse_hunk (midi, pos, 'track', b'MTrk')
         yield list (_parse_track_body (trackdata, clocks_max))
     # if pos < len(midi):
     #     warn
 
 def parse_track (track, clocks_max=None):
-    track_body, end = _parse_hunk (track, 0, 'track', 'MTrk')
+    track_body, end = _parse_hunk (track, 0, 'track', b'MTrk')
     # if end < len(track):
     #     warn
     return list (_parse_track_body (track_body, clocks_max))
 
 def parse (midi, clocks_max=None):
-    header, first_track_pos = _parse_hunk(midi, 0, 'file', 'MThd')
+    header, first_track_pos = _parse_hunk(midi, 0, 'file', b'MThd')
     try:
         format, num_tracks, division = struct.unpack ('>3H', header[:6])
     except struct.error:
Index: python/musicexp.py
diff --git a/python/musicexp.py b/python/musicexp.py
index 
b3905af634d6ebef4fa9bc6fd52fad6132f9b401..7fc35eea3029d40c145a44fde1bac28cd88b16b4
 100644
--- a/python/musicexp.py
+++ b/python/musicexp.py
@@ -818,7 +818,7 @@ class Lyrics:
         for l in self.lyrics_syllables:
             lstr += l
         #lstr += "\n}"
-        return lstr.encode('utf-8')
+        return lstr
 
 class Header:
 
Index: python/musicxml.py
diff --git a/python/musicxml.py b/python/musicxml.py
index 
2a456f4c9e6172176f58c5de820207281091e365..4841358f492bcc93ea204e5b2393225ff7628d1d
 100644
--- a/python/musicxml.py
+++ b/python/musicxml.py
@@ -41,7 +41,7 @@ class Xml_node(object):
         if not self._children:
             return ''
 
-        return ''.join([c.get_text() for c in self._children]).encode('utf-8')
+        return ''.join([c.get_text() for c in self._children])
 
     def message(self, msg):
         ly.warning(msg)
Index: python/utilities.py
diff --git a/python/utilities.py b/python/utilities.py
index 
e124539d3bf2b6ecfe5e0c136d9a73ce2da72758..4513142fbb9983554d8585db85ea48e23538410e
 100644
--- a/python/utilities.py
+++ b/python/utilities.py
@@ -63,8 +63,6 @@ def hex_to_color(hex_val):
         return None
 
 def split_string_and_preserve_doublequoted_substrings(value):
-    if isinstance(value, unicode):
-        value = value.encode('utf-8')
     import shlex
     lex = shlex.shlex(value)
     lex.quotes = '"'
Index: scripts/midi2ly.py
diff --git a/scripts/midi2ly.py b/scripts/midi2ly.py
index 
96cca8dbd8f313c039bfe8b7474a7ca0d3be4e80..9bfabbe1e3a3688224a224a5a095eced50d62653
 100644
--- a/scripts/midi2ly.py
+++ b/scripts/midi2ly.py
@@ -1117,7 +1117,7 @@ def get_option_parser ():
     p.add_option ('-x', '--text-lyrics', help=_ ('treat every text as a 
lyric'),
            action='store_true')
 
-    p.add_option_group (ly.display_encode (_ ('Examples')),
+    p.add_option_group (_ ('Examples'),
               description = r'''
   $ midi2ly --key=-2:1 --duration-quant=32 --allow-tuplet=4*2/3 
--allow-tuplet=2*4/3 foo.midi
 ''')
Index: scripts/musicxml2ly.py
diff --git a/scripts/musicxml2ly.py b/scripts/musicxml2ly.py
index 
1a23236a8982ff83097b466a57c5f8a994c5279a..730863b4c1fc55cbd2d614f23dfbcb2e5fb14fb6
 100755
--- a/scripts/musicxml2ly.py
+++ b/scripts/musicxml2ly.py
@@ -1249,7 +1249,7 @@ def musicxml_dynamics_to_lily_event(dynentry):
               " = #(make-dynamic-script \"" + dynamicstext + "\")"
         needed_additional_definitions.append(dynamicsname)
     event = musicexp.DynamicsEvent()
-    event.type = dynamicsname.encode('utf-8')
+    event.type = dynamicsname
     return event
 
 # Convert single-color two-byte strings to numbers 0.0 - 1.0
@@ -3174,9 +3174,9 @@ def convert(filename, options):
     printer = musicexp.Output_printer()
     #ly.progress(_("Output to `%s'") % defs_ly_name, True)
     if (options.output_name == "-"):
-      printer.set_file(codecs.getwriter("utf-8")(sys.stdout))
+      printer.set_file(sys.stdout)
     else:
-      printer.set_file(codecs.open(output_ly_name, 'wb', encoding='utf-8'))
+      printer.set_file(open(output_ly_name, 'w'))
     print_ly_preamble(printer, filename)
     print_ly_additional_definitions(printer, filename)
     if score_information:
@@ -3256,7 +3256,7 @@ def main():
     conversion_settings.convert_stem_directions = 
options.convert_stem_directions
 
     # Allow the user to leave out the .xml or xml on the filename
-    basefilename = args[0].decode('utf-8')
+    basefilename = args[0]
     if basefilename == "-": # Read from stdin
         filename = "-"
     else: