[PATCH] globsort: handle int overflow in cmp functions

[Grisha Levit]

The current cmp implementation for size and blocks subtracts the two
values and returns the difference as an int. This subtraction can
overflow, and the returned int can end up having the wrong sign.

This also makes the qsort comparison function non-transitive. (Some
interesting discussion on that at [1]).

[1]: https://www.qualys.com/2024/01/30/qsort.txt
---
 pathexp.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pathexp.c b/pathexp.c
index f62efafb..9335f8d5 100644
--- a/pathexp.c
+++ b/pathexp.c
@@ -747,7 +747,9 @@ globsort_namecmp (char **s1, char **s2)
 static int
 globsort_sizecmp (struct globsort_t *g1, struct globsort_t *g2)
 {
-  return ((glob_sorttype < SORT_REVERSE) ? g1->st.size - g2->st.size : 
g2->st.size - g1->st.size);
+  return (glob_sorttype < SORT_REVERSE)
+        ? (g1->st.size > g2->st.size) - (g1->st.size < g2->st.size)
+        : (g2->st.size > g1->st.size) - (g1->st.size < g2->st.size);
 }
 
 static int
@@ -779,7 +781,9 @@ globsort_timecmp (struct globsort_t *g1, struct globsort_t 
*g2)
 static int
 globsort_blockscmp (struct globsort_t *g1, struct globsort_t *g2)
 {
-  return ((glob_sorttype < SORT_REVERSE) ? g1->st.blocks - g2->st.blocks : 
g2->st.blocks - g1->st.blocks);
+  return (glob_sorttype < SORT_REVERSE)
+        ? (g1->st.blocks > g2->st.blocks) - (g1->st.size < g2->st.blocks)
+        : (g2->st.blocks > g1->st.blocks) - (g1->st.size < g2->st.blocks);
 }
 
 static struct globsort_t *
-- 
2.45.1