1// Copyright 2018 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef V8_MEMCOPY_H_
6#define V8_MEMCOPY_H_
7
8#include <stdint.h>
9#include <stdlib.h>
10#include <string.h>
11
12#include "src/base/logging.h"
13#include "src/base/macros.h"
14
15namespace v8 {
16namespace internal {
17
18typedef uintptr_t Address;
19
20// ----------------------------------------------------------------------------
21// Generated memcpy/memmove for ia32, arm, and mips.
22
23void init_memcopy_functions();
24
25#if defined(V8_TARGET_ARCH_IA32)
26// Limit below which the extra overhead of the MemCopy function is likely
27// to outweigh the benefits of faster copying.
28const size_t kMinComplexMemCopy = 64;
29
30// Copy memory area. No restrictions.
31V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size);
32typedef void (*MemMoveFunction)(void* dest, const void* src, size_t size);
33
34// Keep the distinction of "move" vs. "copy" for the benefit of other
35// architectures.
36V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
37 MemMove(dest, src, size);
38}
39#elif defined(V8_HOST_ARCH_ARM)
40typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
41 size_t size);
42V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
43V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
44 size_t chars) {
45 memcpy(dest, src, chars);
46}
47// For values < 16, the assembler function is slower than the inlined C code.
48const size_t kMinComplexMemCopy = 16;
49V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
50 (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
51 reinterpret_cast<const uint8_t*>(src), size);
52}
53V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
54 size_t size) {
55 memmove(dest, src, size);
56}
57
58typedef void (*MemCopyUint16Uint8Function)(uint16_t* dest, const uint8_t* src,
59 size_t size);
60extern MemCopyUint16Uint8Function memcopy_uint16_uint8_function;
61void MemCopyUint16Uint8Wrapper(uint16_t* dest, const uint8_t* src,
62 size_t chars);
63// For values < 12, the assembler function is slower than the inlined C code.
64const int kMinComplexConvertMemCopy = 12;
65V8_INLINE void MemCopyUint16Uint8(uint16_t* dest, const uint8_t* src,
66 size_t size) {
67 (*memcopy_uint16_uint8_function)(dest, src, size);
68}
69#elif defined(V8_HOST_ARCH_MIPS)
70typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
71 size_t size);
72V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
73V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
74 size_t chars) {
75 memcpy(dest, src, chars);
76}
77// For values < 16, the assembler function is slower than the inlined C code.
78const size_t kMinComplexMemCopy = 16;
79V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
80 (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
81 reinterpret_cast<const uint8_t*>(src), size);
82}
83V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
84 size_t size) {
85 memmove(dest, src, size);
86}
87#else
88// Copy memory area to disjoint memory area.
89V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
90 memcpy(dest, src, size);
91}
92V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
93 size_t size) {
94 memmove(dest, src, size);
95}
96const size_t kMinComplexMemCopy = 8;
97#endif // V8_TARGET_ARCH_IA32
98
99// Copies words from |src| to |dst|. The data spans must not overlap.
100// |src| and |dst| must be TWord-size aligned.
101template <size_t kBlockCopyLimit, typename T>
102inline void CopyImpl(T* dst_ptr, const T* src_ptr, size_t count) {
103 constexpr int kTWordSize = sizeof(T);
104#ifdef DEBUG
105 Address dst = reinterpret_cast<Address>(dst_ptr);
106 Address src = reinterpret_cast<Address>(src_ptr);
107 DCHECK(IsAligned(dst, kTWordSize));
108 DCHECK(IsAligned(src, kTWordSize));
109 DCHECK(((src <= dst) && ((src + count * kTWordSize) <= dst)) ||
110 ((dst <= src) && ((dst + count * kTWordSize) <= src)));
111#endif
112
113 // Use block copying MemCopy if the segment we're copying is
114 // enough to justify the extra call/setup overhead.
115 if (count < kBlockCopyLimit) {
116 do {
117 count--;
118 *dst_ptr++ = *src_ptr++;
119 } while (count > 0);
120 } else {
121 MemCopy(dst_ptr, src_ptr, count * kTWordSize);
122 }
123}
124
125// Copies kSystemPointerSize-sized words from |src| to |dst|. The data spans
126// must not overlap. |src| and |dst| must be kSystemPointerSize-aligned.
127inline void CopyWords(Address dst, const Address src, size_t num_words) {
128 static const size_t kBlockCopyLimit = 16;
129 CopyImpl<kBlockCopyLimit>(reinterpret_cast<Address*>(dst),
130 reinterpret_cast<const Address*>(src), num_words);
131}
132
133// Copies data from |src| to |dst|. The data spans must not overlap.
134template <typename T>
135inline void CopyBytes(T* dst, const T* src, size_t num_bytes) {
136 STATIC_ASSERT(sizeof(T) == 1);
137 if (num_bytes == 0) return;
138 CopyImpl<kMinComplexMemCopy>(dst, src, num_bytes);
139}
140
141inline void MemsetInt32(int32_t* dest, int32_t value, size_t counter) {
142#if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
143#define STOS "stosl"
144#endif
145
146#if defined(MEMORY_SANITIZER)
147 // MemorySanitizer does not understand inline assembly.
148#undef STOS
149#endif
150
151#if defined(__GNUC__) && defined(STOS)
152 asm volatile(
153 "cld;"
154 "rep ; " STOS
155 : "+&c"(counter), "+&D"(dest)
156 : "a"(value)
157 : "memory", "cc");
158#else
159 for (size_t i = 0; i < counter; i++) {
160 dest[i] = value;
161 }
162#endif
163
164#undef STOS
165}
166
167inline void MemsetPointer(Address* dest, Address value, size_t counter) {
168#if V8_HOST_ARCH_IA32
169#define STOS "stosl"
170#elif V8_HOST_ARCH_X64
171#define STOS "stosq"
172#endif
173
174#if defined(MEMORY_SANITIZER)
175 // MemorySanitizer does not understand inline assembly.
176#undef STOS
177#endif
178
179#if defined(__GNUC__) && defined(STOS)
180 asm volatile(
181 "cld;"
182 "rep ; " STOS
183 : "+&c"(counter), "+&D"(dest)
184 : "a"(value)
185 : "memory", "cc");
186#else
187 for (size_t i = 0; i < counter; i++) {
188 dest[i] = value;
189 }
190#endif
191
192#undef STOS
193}
194
195template <typename T, typename U>
196inline void MemsetPointer(T** dest, U* value, size_t counter) {
197#ifdef DEBUG
198 T* a = nullptr;
199 U* b = nullptr;
200 a = b; // Fake assignment to check assignability.
201 USE(a);
202#endif // DEBUG
203 MemsetPointer(reinterpret_cast<Address*>(dest),
204 reinterpret_cast<Address>(value), counter);
205}
206
207template <typename sourcechar, typename sinkchar>
208V8_INLINE static void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src,
209 size_t chars);
210#if defined(V8_HOST_ARCH_ARM)
211V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
212 size_t chars);
213V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src,
214 size_t chars);
215V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
216 size_t chars);
217#elif defined(V8_HOST_ARCH_MIPS)
218V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
219 size_t chars);
220V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
221 size_t chars);
222#elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
223V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
224 size_t chars);
225V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
226 size_t chars);
227#endif
228
229// Copy from 8bit/16bit chars to 8bit/16bit chars.
230template <typename sourcechar, typename sinkchar>
231V8_INLINE void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars);
232
233template <typename sourcechar, typename sinkchar>
234void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars) {
235 DCHECK_LE(sizeof(sourcechar), 2);
236 DCHECK_LE(sizeof(sinkchar), 2);
237 if (sizeof(sinkchar) == 1) {
238 if (sizeof(sourcechar) == 1) {
239 CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
240 reinterpret_cast<const uint8_t*>(src), chars);
241 } else {
242 CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
243 reinterpret_cast<const uint16_t*>(src), chars);
244 }
245 } else {
246 if (sizeof(sourcechar) == 1) {
247 CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
248 reinterpret_cast<const uint8_t*>(src), chars);
249 } else {
250 CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
251 reinterpret_cast<const uint16_t*>(src), chars);
252 }
253 }
254}
255
256template <typename sourcechar, typename sinkchar>
257void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src, size_t chars) {
258 sinkchar* limit = dest + chars;
259 if ((sizeof(*dest) == sizeof(*src)) &&
260 (chars >= kMinComplexMemCopy / sizeof(*dest))) {
261 MemCopy(dest, src, chars * sizeof(*dest));
262 } else {
263 while (dest < limit) *dest++ = static_cast<sinkchar>(*src++);
264 }
265}
266
267#if defined(V8_HOST_ARCH_ARM)
268void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
269 switch (static_cast<unsigned>(chars)) {
270 case 0:
271 break;
272 case 1:
273 *dest = *src;
274 break;
275 case 2:
276 memcpy(dest, src, 2);
277 break;
278 case 3:
279 memcpy(dest, src, 3);
280 break;
281 case 4:
282 memcpy(dest, src, 4);
283 break;
284 case 5:
285 memcpy(dest, src, 5);
286 break;
287 case 6:
288 memcpy(dest, src, 6);
289 break;
290 case 7:
291 memcpy(dest, src, 7);
292 break;
293 case 8:
294 memcpy(dest, src, 8);
295 break;
296 case 9:
297 memcpy(dest, src, 9);
298 break;
299 case 10:
300 memcpy(dest, src, 10);
301 break;
302 case 11:
303 memcpy(dest, src, 11);
304 break;
305 case 12:
306 memcpy(dest, src, 12);
307 break;
308 case 13:
309 memcpy(dest, src, 13);
310 break;
311 case 14:
312 memcpy(dest, src, 14);
313 break;
314 case 15:
315 memcpy(dest, src, 15);
316 break;
317 default:
318 MemCopy(dest, src, chars);
319 break;
320 }
321}
322
323void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src, size_t chars) {
324 if (chars >= static_cast<size_t>(kMinComplexConvertMemCopy)) {
325 MemCopyUint16Uint8(dest, src, chars);
326 } else {
327 MemCopyUint16Uint8Wrapper(dest, src, chars);
328 }
329}
330
331void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
332 switch (static_cast<unsigned>(chars)) {
333 case 0:
334 break;
335 case 1:
336 *dest = *src;
337 break;
338 case 2:
339 memcpy(dest, src, 4);
340 break;
341 case 3:
342 memcpy(dest, src, 6);
343 break;
344 case 4:
345 memcpy(dest, src, 8);
346 break;
347 case 5:
348 memcpy(dest, src, 10);
349 break;
350 case 6:
351 memcpy(dest, src, 12);
352 break;
353 case 7:
354 memcpy(dest, src, 14);
355 break;
356 default:
357 MemCopy(dest, src, chars * sizeof(*dest));
358 break;
359 }
360}
361
362#elif defined(V8_HOST_ARCH_MIPS)
363void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
364 if (chars < kMinComplexMemCopy) {
365 memcpy(dest, src, chars);
366 } else {
367 MemCopy(dest, src, chars);
368 }
369}
370
371void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
372 if (chars < kMinComplexMemCopy) {
373 memcpy(dest, src, chars * sizeof(*dest));
374 } else {
375 MemCopy(dest, src, chars * sizeof(*dest));
376 }
377}
378#elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
379#define CASE(n) \
380 case n: \
381 memcpy(dest, src, n); \
382 break
383void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
384 switch (static_cast<unsigned>(chars)) {
385 case 0:
386 break;
387 case 1:
388 *dest = *src;
389 break;
390 CASE(2);
391 CASE(3);
392 CASE(4);
393 CASE(5);
394 CASE(6);
395 CASE(7);
396 CASE(8);
397 CASE(9);
398 CASE(10);
399 CASE(11);
400 CASE(12);
401 CASE(13);
402 CASE(14);
403 CASE(15);
404 CASE(16);
405 CASE(17);
406 CASE(18);
407 CASE(19);
408 CASE(20);
409 CASE(21);
410 CASE(22);
411 CASE(23);
412 CASE(24);
413 CASE(25);
414 CASE(26);
415 CASE(27);
416 CASE(28);
417 CASE(29);
418 CASE(30);
419 CASE(31);
420 CASE(32);
421 CASE(33);
422 CASE(34);
423 CASE(35);
424 CASE(36);
425 CASE(37);
426 CASE(38);
427 CASE(39);
428 CASE(40);
429 CASE(41);
430 CASE(42);
431 CASE(43);
432 CASE(44);
433 CASE(45);
434 CASE(46);
435 CASE(47);
436 CASE(48);
437 CASE(49);
438 CASE(50);
439 CASE(51);
440 CASE(52);
441 CASE(53);
442 CASE(54);
443 CASE(55);
444 CASE(56);
445 CASE(57);
446 CASE(58);
447 CASE(59);
448 CASE(60);
449 CASE(61);
450 CASE(62);
451 CASE(63);
452 CASE(64);
453 default:
454 memcpy(dest, src, chars);
455 break;
456 }
457}
458#undef CASE
459
460#define CASE(n) \
461 case n: \
462 memcpy(dest, src, n * 2); \
463 break
464void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
465 switch (static_cast<unsigned>(chars)) {
466 case 0:
467 break;
468 case 1:
469 *dest = *src;
470 break;
471 CASE(2);
472 CASE(3);
473 CASE(4);
474 CASE(5);
475 CASE(6);
476 CASE(7);
477 CASE(8);
478 CASE(9);
479 CASE(10);
480 CASE(11);
481 CASE(12);
482 CASE(13);
483 CASE(14);
484 CASE(15);
485 CASE(16);
486 CASE(17);
487 CASE(18);
488 CASE(19);
489 CASE(20);
490 CASE(21);
491 CASE(22);
492 CASE(23);
493 CASE(24);
494 CASE(25);
495 CASE(26);
496 CASE(27);
497 CASE(28);
498 CASE(29);
499 CASE(30);
500 CASE(31);
501 CASE(32);
502 default:
503 memcpy(dest, src, chars * 2);
504 break;
505 }
506}
507#undef CASE
508#endif
509
510} // namespace internal
511} // namespace v8
512
513#endif // V8_MEMCOPY_H_
514