[Emacs-diffs] trunk r117636: Avoid undefined behavior with signed left shift.

[Paul Eggert]

------------------------------------------------------------
revno: 117636
revision-id: address@hidden
parent: address@hidden
committer: Paul Eggert <address@hidden>
branch nick: trunk
timestamp: Sun 2014-08-03 05:34:44 -0700
message:
  Avoid undefined behavior with signed left shift.
  
  Caught by 'gcc -fsanitize=undefined'.
  * dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
  * dispnew.c (line_hash_code, scrolling):
  * scroll.c (calculate_scrolling, calculate_direct_scrolling):
  Use 'unsigned', not 'int', for line hashes.
  (scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
modified:
  src/ChangeLog                  changelog-20091113204419-o5vbwnq5f7feedwu-1438
  src/dispextern.h               
dispextern.h-20091113204419-o5vbwnq5f7feedwu-218
  src/dispnew.c                  dispnew.c-20091113204419-o5vbwnq5f7feedwu-258
  src/scroll.c                   scroll.c-20091113204419-o5vbwnq5f7feedwu-173

=== modified file 'src/ChangeLog'
--- a/src/ChangeLog     2014-08-02 17:12:11 +0000
+++ b/src/ChangeLog     2014-08-03 12:34:44 +0000
@@ -1,3 +1,13 @@
+2014-08-03  Paul Eggert  <address@hidden>
+
+       Avoid undefined behavior with signed left shift.
+       Caught by 'gcc -fsanitize=undefined'.
+       * dispextern.h, scroll.c (scrolling_max_lines_saved, scrolling_1):
+       * dispnew.c (line_hash_code, scrolling):
+       * scroll.c (calculate_scrolling, calculate_direct_scrolling):
+       Use 'unsigned', not 'int', for line hashes.
+       (scrolling_max_lines_saved): Avoid mystery constants for hash sizes.
+
 2014-08-02  Paul Eggert  <address@hidden>
 
        Make compare-strings more compatible with old behavior (Bug#17903).

=== modified file 'src/dispextern.h'
--- a/src/dispextern.h  2014-07-27 13:21:30 +0000
+++ b/src/dispextern.h  2014-08-03 12:34:44 +0000
@@ -3507,13 +3507,13 @@
 
 /* Defined in scroll.c */
 
-extern int scrolling_max_lines_saved (int, int, int *, int *, int *);
+extern int scrolling_max_lines_saved (int, int, unsigned *, unsigned *, int *);
 extern void do_line_insertion_deletion_costs (struct frame *, const char *,
                                               const char *, const char *,
                                              const char *, const char *,
                                              const char *, int);
-void scrolling_1 (struct frame *, int, int, int, int *, int *, int *,
-                  int *, int);
+void scrolling_1 (struct frame *, int, int, int, int *, int *, unsigned *,
+                  unsigned *, int);
 
 /* Defined in frame.c */
 

=== modified file 'src/dispnew.c'
--- a/src/dispnew.c     2014-07-28 09:39:09 +0000
+++ b/src/dispnew.c     2014-08-03 12:34:44 +0000
@@ -1108,10 +1108,10 @@
 /* Return a hash code for glyph row ROW, which may
    be from current or desired matrix of frame F.  */
 
-static int
+static unsigned
 line_hash_code (struct frame *f, struct glyph_row *row)
 {
-  int hash = 0;
+  unsigned hash = 0;
 
   if (row->enabled_p)
     {
@@ -4647,8 +4647,8 @@
   int unchanged_at_top, unchanged_at_bottom;
   int window_size;
   int changed_lines;
-  int *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
-  int *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *old_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
+  unsigned *new_hash = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   int *old_draw_cost = alloca (FRAME_TOTAL_LINES (frame) * sizeof (int));
   register int i;

=== modified file 'src/scroll.c'
--- a/src/scroll.c      2014-01-01 07:43:34 +0000
+++ b/src/scroll.c      2014-08-03 12:34:44 +0000
@@ -90,7 +90,7 @@
                     /* matrix is of size window_size + 1 on each side.  */
                     struct matrix_elt *matrix,
                     int window_size, int lines_below,
-                    int *draw_cost, int *old_hash, int *new_hash,
+                    int *draw_cost, unsigned *old_hash, unsigned *new_hash,
                     int free_at_end)
 {
   register int i, j;
@@ -427,7 +427,7 @@
                            struct matrix_elt *matrix,
                            int window_size, int lines_below,
                            int *draw_cost, int *old_draw_cost,
-                           int *old_hash, int *new_hash,
+                           unsigned *old_hash, unsigned *new_hash,
                            int free_at_end)
 {
   register int i, j;
@@ -794,7 +794,7 @@
 void
 scrolling_1 (struct frame *frame, int window_size, int unchanged_at_top,
             int unchanged_at_bottom, int *draw_cost, int *old_draw_cost,
-            int *old_hash, int *new_hash, int free_at_end)
+            unsigned *old_hash, unsigned *new_hash, int free_at_end)
 {
   struct matrix_elt *matrix
     = alloca ((window_size + 1) * (window_size + 1) * sizeof *matrix);
@@ -829,12 +829,14 @@
 
 int
 scrolling_max_lines_saved (int start, int end,
-                           int *oldhash, int *newhash,
+                           unsigned *oldhash, unsigned *newhash,
                            int *cost)
 {
-  struct { int hash; int count; } lines[01000];
-  register int i, h;
-  register int matchcount = 0;
+  enum { LOG2_NLINES = 9 };
+  enum { NLINES = 1 << LOG2_NLINES };
+  struct { unsigned hash; int count; } lines[NLINES];
+  int i, h;
+  int matchcount = 0;
   int avg_length = 0;
   int threshold;
 
@@ -855,7 +857,7 @@
     {
       if (cost[i] > threshold)
        {
-         h = newhash[i] & 0777;
+         h = newhash[i] & (NLINES - 1);
          lines[h].hash = newhash[i];
          lines[h].count++;
        }
@@ -865,7 +867,7 @@
      matches between old lines and new.  */
   for (i = start; i < end; i++)
     {
-      h = oldhash[i] & 0777;
+      h = oldhash[i] & (NLINES - 1);
       if (oldhash[i] == lines[h].hash)
        {
          matchcount++;