about summary refs log tree commit diff
path: root/sysdeps/tile/memset.c
blob: db3769b15f82e6471356c6d05c591ea727f3a802 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
/* Copyright (C) 2011-2018 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Chris Metcalf <cmetcalf@tilera.com>, 2011.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library.  If not, see
   <http://www.gnu.org/licenses/>.  */

#include <arch/chip.h>
#include <string.h>
#include <stdint.h>
#include "string-endian.h"

void * inhibit_loop_to_libcall
__memset (void *s, int c, size_t n)
{
  uint64_t *out64;
  int n64, to_align64;
  uint64_t v64;
  uint8_t *out8 = s;

  /* Experimentation shows that a trivial tight loop is a win up until
     around a size of 20, where writing a word at a time starts to win.  */
#define BYTE_CUTOFF 20

#if BYTE_CUTOFF < 7
  /* This must be at least at least this big, or some code later
     on doesn't work.  */
# error "BYTE_CUTOFF is too small."
#endif

  if (n < BYTE_CUTOFF)
    {
      /* Strangely, this turns out to be the tightest way to write
         this loop.  */
      if (n != 0)
        {
          do
            {
              /* Strangely, combining these into one line performs worse.  */
              *out8 = c;
              out8++;
            }
          while (--n != 0);
        }

      return s;
    }

  /* Align 'out8'. We know n >= 7 so this won't write past the end.  */
  while (((uintptr_t) out8 & 7) != 0)
    {
      *out8++ = c;
      --n;
    }

  /* Align 'n'. */
  while (n & 7)
    out8[--n] = c;

  out64 = (uint64_t *) out8;
  n64 = n >> 3;

  /* Tile input byte out to 64 bits. */
  v64 = copy_byte(c);

  /* This must be at least 8 or the following loop doesn't work. */
#define CACHE_LINE_SIZE_IN_DOUBLEWORDS (CHIP_L2_LINE_SIZE() / 8)

  /* Determine how many words we need to emit before the 'out32'
     pointer becomes aligned modulo the cache line size.  */
  to_align64 = (-((uintptr_t) out64 >> 3)) &
    (CACHE_LINE_SIZE_IN_DOUBLEWORDS - 1);

  /* Only bother aligning and using wh64 if there is at least
     one full cache line to process.  This check also prevents
     overrunning the end of the buffer with alignment words.  */
  if (to_align64 <= n64 - CACHE_LINE_SIZE_IN_DOUBLEWORDS)
    {
      int lines_left;

      /* Align out64 mod the cache line size so we can use wh64. */
      n64 -= to_align64;
      for (; to_align64 != 0; to_align64--)
        {
          *out64 = v64;
          out64++;
        }

      /* Use unsigned divide to turn this into a right shift. */
      lines_left = (unsigned) n64 / CACHE_LINE_SIZE_IN_DOUBLEWORDS;

      do
        {
          /* Only wh64 a few lines at a time, so we don't exceed the
             maximum number of victim lines.  */
          int x = ((lines_left < CHIP_MAX_OUTSTANDING_VICTIMS ()) ? lines_left
                   : CHIP_MAX_OUTSTANDING_VICTIMS ());
          uint64_t *wh = out64;
          int i = x;
          int j;

          lines_left -= x;

          do
            {
              __insn_wh64 (wh);
              wh += CACHE_LINE_SIZE_IN_DOUBLEWORDS;
            }
          while (--i);

          for (j = x * (CACHE_LINE_SIZE_IN_DOUBLEWORDS / 4); j != 0; j--)
            {
              *out64++ = v64;
              *out64++ = v64;
              *out64++ = v64;
              *out64++ = v64;
            }
        }
      while (lines_left != 0);

      /* We processed all full lines above, so only this many
         words remain to be processed.  */
      n64 &= CACHE_LINE_SIZE_IN_DOUBLEWORDS - 1;
    }

  /* Now handle any leftover values. */
  if (n64 != 0)
    {
      do
        {
          *out64 = v64;
          out64++;
        }
      while (--n64 != 0);
    }

  return s;
}
weak_alias (__memset, memset)
libc_hidden_builtin_def (memset)