libio: Multiple fixes for open_{w}memstram (BZ#18241 and BZ#20181)

This patches fixes multiples issues on open_{w}memstream reported on both
BZ#18241 and BZ#20181:

  - failed fseek does not set errno.
  - negative offset in fseek fails even when resulting position is
    a valid one.
  - a flush after write if the current write position is not at the
    end of the stream currupt data.

The main fix is on seek operation for memstream (_IO_{w}str_seekoff), where
both _IO_read_ptr and _IO_read_end pointer are updated if a write operation
has occured (similar to default file operations).  Also, to calculate the
offset on both read and write pointers, a temporary value is instead of
updating the argument supplied value.  Negative offset are valid if resulting
internal pointer is within the range of _IO_{read,write}_base and
_IO_{read,write}_end.

Also POSIX states that a null or wide null shall be appended to the current
buffer iff a write moves the position to a value larger than the current
lenght.  Current implementation appends a null or wide null regardless
of this condition.  This patch fixes it by removing the 'else' condition
on _IO_{w}mem_sync.

Checked on x86_64.

	[BZ #18241]
	[BZ #20181]
	* libio/Makefile (test): Add tst-memstream3 and tst-wmemstream3.
	* libio/memstream.c (_IO_mem_sync): Only append a null byte if
	write position is at the end the buffer.
	* libio/wmemstream.c (_IO_wmem_sync): Likewise.
	* libio/strops.c (_IO_str_switch_to_get_mode): New function.
	(_IO_str_seekoff): Set correct offset from negative displacement and
	set EINVAL for invalid ones.
	* libio/wstrops.c (enlarge_userbuf): Use correct function to calculate
	buffer length.
	(_IO_wstr_switch_to_get_mode): New function.
	(_IO_wstr_seekoff): Set correct offset from negative displacement and
	set EINVAL for invalid ones.
	* libio/tst-memstream3.c: New file.
	* libio/tst-wmemstream3.c: Likewise.
	* manual/examples/memstrm.c: Remove warning when priting size_t.

1 files changed, 55 insertions, 26 deletions

diff --git a/libio/strops.c b/libio/strops.c
index 2ba3704dd2..1bb8a77932 100644
--- a/libio/strops.c
+++ b/libio/strops.c
@@ -230,6 +230,21 @@ enlarge_userbuf (_IO_FILE *fp, _IO_off64_t offset, int reading)
   return 0;
 }
 
+static void
+_IO_str_switch_to_get_mode (_IO_FILE *fp)
+{
+  if (_IO_in_backup (fp))
+    fp->_IO_read_base = fp->_IO_backup_base;
+  else
+    {
+      fp->_IO_read_base = fp->_IO_buf_base;
+      if (fp->_IO_write_ptr > fp->_IO_read_end)
+        fp->_IO_read_end = fp->_IO_write_ptr;
+    }
+  fp->_IO_read_ptr = fp->_IO_read_end = fp->_IO_write_ptr;
+
+  fp->_flags &= ~_IO_CURRENTLY_PUTTING;
+}
 
 _IO_off64_t
 _IO_str_seekoff (_IO_FILE *fp, _IO_off64_t offset, int dir, int mode)
@@ -239,14 +254,14 @@ _IO_str_seekoff (_IO_FILE *fp, _IO_off64_t offset, int dir, int mode)
   if (mode == 0 && (fp->_flags & _IO_TIED_PUT_GET))
     mode = (fp->_flags & _IO_CURRENTLY_PUTTING ? _IOS_OUTPUT : _IOS_INPUT);
 
+  bool was_writing = (fp->_IO_write_ptr > fp->_IO_write_base
+		     || _IO_in_put_mode (fp));
+  if (was_writing)
+    _IO_str_switch_to_get_mode (fp);
+
   if (mode == 0)
     {
-      /* Don't move any pointers. But there is no clear indication what
-	 mode FP is in. Let's guess. */
-      if (fp->_IO_file_flags & _IO_NO_WRITES)
-        new_pos = fp->_IO_read_ptr - fp->_IO_read_base;
-      else
-        new_pos = fp->_IO_write_ptr - fp->_IO_write_base;
+      new_pos = fp->_IO_read_ptr - fp->_IO_read_base;
     }
   else
     {
@@ -256,48 +271,62 @@ _IO_str_seekoff (_IO_FILE *fp, _IO_off64_t offset, int dir, int mode)
       /* Move the get pointer, if requested. */
       if (mode & _IOS_INPUT)
 	{
+	  _IO_ssize_t base;
 	  switch (dir)
 	    {
-	    case _IO_seek_end:
-	      offset += cur_size;
+	    case _IO_seek_set:
+	      base = 0;
 	      break;
 	    case _IO_seek_cur:
-	      offset += fp->_IO_read_ptr - fp->_IO_read_base;
+	      base = fp->_IO_read_ptr - fp->_IO_read_base;
 	      break;
-	    default: /* case _IO_seek_set: */
+	    default: /* case _IO_seek_end: */
+	      base = cur_size;
 	      break;
 	    }
-	  if (offset < 0)
-	    return EOF;
-	  if ((_IO_ssize_t) offset > cur_size
-	      && enlarge_userbuf (fp, offset, 1) != 0)
+	  _IO_ssize_t maxval = SSIZE_MAX - base;
+	  if (offset < -base || offset > maxval)
+	    {
+	      __set_errno (EINVAL);
+	      return EOF;
+	    }
+	  base += offset;
+	  if (base > cur_size
+	      && enlarge_userbuf (fp, base, 1) != 0)
 	    return EOF;
-	  fp->_IO_read_ptr = fp->_IO_read_base + offset;
+	  fp->_IO_read_ptr = fp->_IO_read_base + base;
 	  fp->_IO_read_end = fp->_IO_read_base + cur_size;
-	  new_pos = offset;
+	  new_pos = base;
 	}
 
       /* Move the put pointer, if requested. */
       if (mode & _IOS_OUTPUT)
 	{
+	  _IO_ssize_t base;
 	  switch (dir)
 	    {
-	    case _IO_seek_end:
-	      offset += cur_size;
+	    case _IO_seek_set:
+	      base = 0;
 	      break;
 	    case _IO_seek_cur:
-	      offset += fp->_IO_write_ptr - fp->_IO_write_base;
+	      base = fp->_IO_write_ptr - fp->_IO_write_base;
 	      break;
-	    default: /* case _IO_seek_set: */
+	    default: /* case _IO_seek_end: */
+	      base = cur_size;
 	      break;
 	    }
-	  if (offset < 0)
-	    return EOF;
-	  if ((_IO_ssize_t) offset > cur_size
-	      && enlarge_userbuf (fp, offset, 0) != 0)
+	  _IO_ssize_t maxval = SSIZE_MAX - base;
+	  if (offset < -base || offset > maxval)
+	    {
+	      __set_errno (EINVAL);
+	      return EOF;
+	    }
+	  base += offset;
+	  if (base > cur_size
+	      && enlarge_userbuf (fp, base, 0) != 0)
 	    return EOF;
-	  fp->_IO_write_ptr = fp->_IO_write_base + offset;
-	  new_pos = offset;
+	  fp->_IO_write_ptr = fp->_IO_write_base + base;
+	  new_pos = base;
 	}
     }
   return new_pos;