Fix for test "malloc_usable_size: expected 7 but got 11"

[BZ #17581] The checking chain of unused chunks was terminated by a hash of
    the block pointer, which was sometimes confused with the chunk length byte.
    We now avoid using a length byte equal to the magic byte.

3 files changed, 55 insertions, 32 deletions

diff --git a/ChangeLog b/ChangeLog
index 07f478a227..4268eb75ad 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2015-05-19  James Lemke  <jwlemke@codesourcery.com>
+
+	[BZ #17581]
+	* malloc/hooks.c
+	(magicbyte): Convert to a function and avoid returning 0x01.
+	(mem2mem_check): Avoid using a length byte equal to the magic byte.
+	(mem2chunk_check): Fix unsigned comparisons to zero.
+	Hoist defs of sz and magic.
+
 2015-05-19  Richard Henderson  <rth@redhat.com>
 
 	* soft-fp/op-common.h (_FP_FROM_INT): Don't write to R.
diff --git a/NEWS b/NEWS
index 877ea34809..7d74b8e7b8 100644
--- a/NEWS
+++ b/NEWS
@@ -11,14 +11,14 @@ Version 2.22
 
   4719, 6792, 13064, 14094, 14841, 14906, 15319, 15467, 15790, 15969, 16159,
   16339, 16351, 16352, 16512, 16560, 16704, 16783, 16850, 17090, 17195,
-  17269, 17523, 17542, 17569, 17588, 17596, 17620, 17621, 17628, 17631,
-  17692, 17711, 17715, 17776, 17779, 17792, 17836, 17912, 17916, 17930,
-  17932, 17944, 17949, 17964, 17965, 17967, 17969, 17978, 17987, 17991,
-  17996, 17998, 17999, 18007, 18019, 18020, 18029, 18030, 18032, 18036,
-  18038, 18039, 18042, 18043, 18046, 18047, 18068, 18080, 18093, 18100,
-  18104, 18110, 18111, 18125, 18128, 18138, 18185, 18196, 18197, 18206,
-  18210, 18211, 18217, 18220, 18221, 18247, 18287, 18319, 18333, 18346,
-  18397, 18409, 18418.
+  17269, 17523, 17542, 17569, 17581, 17588, 17596, 17620, 17621, 17628,
+  17631, 17692, 17711, 17715, 17776, 17779, 17792, 17836, 17912, 17916,
+  17930, 17932, 17944, 17949, 17964, 17965, 17967, 17969, 17978, 17987,
+  17991, 17996, 17998, 17999, 18007, 18019, 18020, 18029, 18030, 18032,
+  18036, 18038, 18039, 18042, 18043, 18046, 18047, 18068, 18080, 18093,
+  18100, 18104, 18110, 18111, 18125, 18128, 18138, 18185, 18196, 18197,
+  18206, 18210, 18211, 18217, 18220, 18221, 18247, 18287, 18319, 18333,
+  18346, 18397, 18409, 18418.
 
 * Cache information can be queried via sysconf() function on s390 e.g. with
   _SC_LEVEL1_ICACHE_SIZE as argument.
diff --git a/malloc/hooks.c b/malloc/hooks.c
index 9303fe5bd7..ec79f0a908 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -88,11 +88,22 @@ __malloc_check_init (void)
    overruns.  The goal here is to avoid obscure crashes due to invalid
    usage, unlike in the MALLOC_DEBUG code. */
 
-#define MAGICBYTE(p) ((((size_t) p >> 3) ^ ((size_t) p >> 11)) & 0xFF)
+static unsigned char
+magicbyte (const void *p)
+{
+  unsigned char magic;
+
+  magic = (((uintptr_t) p >> 3) ^ ((uintptr_t) p >> 11)) & 0xFF;
+  /* Do not return 1.  See the comment in mem2mem_check().  */
+  if (magic == 1)
+    ++magic;
+  return magic;
+}
+
 
-/* Visualize the chunk as being partitioned into blocks of 256 bytes from the
-   highest address of the chunk, downwards.  The beginning of each block tells
-   us the size of the previous block, up to the actual size of the requested
+/* Visualize the chunk as being partitioned into blocks of 255 bytes from the
+   highest address of the chunk, downwards.  The end of each block tells
+   us the size of that block, up to the actual size of the requested
    memory.  Our magic byte is right at the end of the requested size, so we
    must reach it with this iteration, otherwise we have witnessed a memory
    corruption.  */
@@ -101,7 +112,7 @@ malloc_check_get_size (mchunkptr p)
 {
   size_t size;
   unsigned char c;
-  unsigned char magic = MAGICBYTE (p);
+  unsigned char magic = magicbyte (p);
 
   assert (using_malloc_checking == 1);
 
@@ -123,32 +134,36 @@ malloc_check_get_size (mchunkptr p)
 }
 
 /* Instrument a chunk with overrun detector byte(s) and convert it
-   into a user pointer with requested size sz. */
+   into a user pointer with requested size req_sz. */
 
 static void *
 internal_function
-mem2mem_check (void *ptr, size_t sz)
+mem2mem_check (void *ptr, size_t req_sz)
 {
   mchunkptr p;
   unsigned char *m_ptr = ptr;
-  size_t i;
+  size_t max_sz, block_sz, i;
+  unsigned char magic;
 
   if (!ptr)
     return ptr;
 
   p = mem2chunk (ptr);
-  for (i = chunksize (p) - (chunk_is_mmapped (p) ? 2 * SIZE_SZ + 1 : SIZE_SZ + 1);
-       i > sz;
-       i -= 0xFF)
+  magic = magicbyte (p);
+  max_sz = chunksize (p) - 2 * SIZE_SZ;
+  if (!chunk_is_mmapped (p))
+    max_sz += SIZE_SZ;
+  for (i = max_sz - 1; i > req_sz; i -= block_sz)
     {
-      if (i - sz < 0x100)
-        {
-          m_ptr[i] = (unsigned char) (i - sz);
-          break;
-        }
-      m_ptr[i] = 0xFF;
+      block_sz = MIN (i - req_sz, 0xff);
+      /* Don't allow the magic byte to appear in the chain of length bytes.
+         For the following to work, magicbyte cannot return 0x01.  */
+      if (block_sz == magic)
+        --block_sz;
+
+      m_ptr[i] = block_sz;
     }
-  m_ptr[sz] = MAGICBYTE (p);
+  m_ptr[req_sz] = magic;
   return (void *) m_ptr;
 }
 
@@ -167,11 +182,12 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
     return NULL;
 
   p = mem2chunk (mem);
+  sz = chunksize (p);
+  magic = magicbyte (p);
   if (!chunk_is_mmapped (p))
     {
       /* Must be a chunk in conventional heap memory. */
       int contig = contiguous (&main_arena);
-      sz = chunksize (p);
       if ((contig &&
            ((char *) p < mp_.sbrk_base ||
             ((char *) p + sz) >= (mp_.sbrk_base + main_arena.system_mem))) ||
@@ -181,10 +197,9 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
                                next_chunk (prev_chunk (p)) != p)))
         return NULL;
 
-      magic = MAGICBYTE (p);
       for (sz += SIZE_SZ - 1; (c = ((unsigned char *) p)[sz]) != magic; sz -= c)
         {
-          if (c <= 0 || sz < (c + 2 * SIZE_SZ))
+          if (c == 0 || sz < (c + 2 * SIZE_SZ))
             return NULL;
         }
     }
@@ -202,13 +217,12 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
            offset < 0x2000) ||
           !chunk_is_mmapped (p) || (p->size & PREV_INUSE) ||
           ((((unsigned long) p - p->prev_size) & page_mask) != 0) ||
-          ((sz = chunksize (p)), ((p->prev_size + sz) & page_mask) != 0))
+          ((p->prev_size + sz) & page_mask) != 0)
         return NULL;
 
-      magic = MAGICBYTE (p);
       for (sz -= 1; (c = ((unsigned char *) p)[sz]) != magic; sz -= c)
         {
-          if (c <= 0 || sz < (c + 2 * SIZE_SZ))
+          if (c == 0 || sz < (c + 2 * SIZE_SZ))
             return NULL;
         }
     }