/* strcpy with SSSE3
   Copyright (C) 2009 Free Software Foundation, Inc.
   Contributed by Intel Corporation.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, write to the Free
   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   02111-1307 USA.  */

#include <sysdep.h>
#include <init-arch.h>

#if !defined (USE_AS_STPCPY) && !defined (USE_AS_STRNCPY)
# ifndef STRCPY
#  define STRCPY strcpy
# endif
#endif

#ifdef USE_AS_STPCPY
# ifdef USE_AS_STRNCPY
#  define STRCPY_SSSE3	__stpncpy_ssse3
#  define STRCPY_SSE2	__stpncpy_sse2
#  define __GI_STRCPY	__GI_stpncpy
# else
#  define STRCPY_SSSE3	__stpcpy_ssse3
#  define STRCPY_SSE2	__stpcpy_sse2
#  define __GI_STRCPY	__GI_stpcpy
#  define __GI___STRCPY	__GI___stpcpy
# endif
#else
# ifdef USE_AS_STRNCPY
#  define STRCPY_SSSE3	__strncpy_ssse3
#  define STRCPY_SSE2	__strncpy_sse2
#  define __GI_STRCPY	__GI_strncpy
# else
#  define STRCPY_SSSE3	__strcpy_ssse3
#  define STRCPY_SSE2	__strcpy_sse2
#  define __GI_STRCPY	__GI_strcpy
# endif
#endif

#ifndef LABEL
#define LABEL(l) L(l)
#endif

/* Define multiple versions only for the definition in libc.  */
#ifndef NOT_IN_libc
	.text
ENTRY(STRCPY)
	.type	STRCPY, @gnu_indirect_function
	cmpl	$0, __cpu_features+KIND_OFFSET(%rip)
	jne	1f
	call	__init_cpu_features
1:	leaq	STRCPY_SSE2(%rip), %rax
	testl	$bit_SSSE3, __cpu_features+CPUID_OFFSET+index_SSSE3(%rip)
	jz	2f
	leaq	STRCPY_SSSE3(%rip), %rax
2:	ret
END(STRCPY)

	.section .text.ssse3,"ax",@progbits
STRCPY_SSSE3:
	cfi_startproc
	CALL_MCOUNT

/*
 * This implementation uses SSE to copy up to 16 bytes at a time.
 */
#ifdef USE_AS_STRNCPY
	test    %rdx, %rdx
	jz      LABEL(strncpy_exitz)
	mov     %rdx, %r8
#else
	xor	%edx, %edx
#endif
	mov	%esi, %ecx
	and	$0xfffffffffffffff0, %rsi	/*force rsi 16 byte align*/
	and	$15, %ecx
	mov	%rdi, %rax			/*store return parameter*/


	pxor	%xmm0, %xmm0			/* clear %xmm0 */
	pcmpeqb	(%rsi), %xmm0			/* compare 16 bytes in (%rsi) and %xmm0 for equality, try to find null char*/
	pmovmskb %xmm0, %edx			/* move each byte mask of %xmm0 to edx*/
	shr	%cl, %edx			/* get real bits left in edx*/
	test	%edx, %edx			/* edx must be 0 if there is no null char from rsi+%rcx */
	jnz	LABEL(less16bytes)

#ifdef USE_AS_STRNCPY
	lea	-16(%r8,%rcx), %r11
	cmp	$0, %r11
	jle	LABEL(less16bytes)		/* if r8 + rcx <= 16, branch to less16bytes.  */
#endif

	mov	%rcx, %r9
	or	%edi, %ecx
	and	$15, %ecx
	lea	-16(%r9), %r10
	jz	LABEL(ashr_0)			/* ecx must be 0 if offset of rsi and rdi is 16 byte align*/

	neg	%r10				/* store the rest in rsi aligned 16 bytes for unaligned_exit*/

	pxor	%xmm0, %xmm0			/* clear %xmm0, may be polluted by unaligned operation*/
	pcmpeqb	16(%rsi), %xmm0			/* compare 16 bytes in (%rsi) and %xmm0 for equality, try to find null char*/
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(less32bytes)
	/*
	* at least 16 byte available to fill destination rdi
	*/
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(less32bytes_strncpy_truncation)
#endif
	mov	(%rsi, %r9), %rdx
	mov	%rdx, (%rdi)
	mov	8(%rsi, %r9), %rdx
	mov	%rdx, 8(%rdi)

	/*
	* so far destatination rdi may be aligned by 16, re-calculate rsi to jump
	* crossponding case
	* rcx is offset of rsi
	* rax is offset of rdi
	*/

	and	$0xfffffffffffffff0, %rdi	/* force rdi 16 byte align */
	mov	%rax, %rdx			/* rax store orignal rdi */
	xor	%rdi, %rdx			/* equal to and $15, %rdx */
#ifdef USE_AS_STRNCPY
	add     %rdx, %r8
#endif

	add	$16, %rdi			/* next 16 bytes for rdi */
	sub	%rdx, %r9

	lea	16(%r9, %rsi), %rsi		/*re-calculate rsi by (16 - rdx)+ rcx */
	mov	%esi, %ecx			/*store offset of rsi */
	and	$0xfffffffffffffff0, %rsi	/* force rsi 16 byte align */

	and	$15, %ecx			/* ecx must be 0 if rdx is equal to rcx*/
	jz	LABEL(ashr_0)

	lea	-16(%rcx), %r10
	mov	%rcx, %r9
	neg	%r10
	lea	LABEL(unaligned_table)(%rip), %r11
	movslq  (%r11, %rcx,4), %rcx
	lea	(%r11, %rcx), %rcx
	jmp	*%rcx

 /*
 * The following cases will be handled by ashr_0 & ashr_0_start
 *  rcx(offset of rsi)  rax(offset of rdi)  relative offset  corresponding case
 *	0		    0		  0		 ashr_0
 *	n(1~15)	     n(1~15)	   0		 ashr_0_start
 *
 */
	.p2align 5
LABEL(ashr_0):
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_aligned)
#endif
	movdqa  (%rsi), %xmm1	   /* fetch first 16 bytes from rsi */
	movdqa  %xmm1, (%rdi)	   /* store first 16 bytes into rdi */
	add     $16, %rsi
	add     $16, %rdi
	pcmpeqb  (%rsi), %xmm0		   /* compare 16 bytes in (%rsi) and %xmm0 for equality, try to find null char */
	pmovmskb  %xmm0, %edx		   /* move each byte mask of %xmm0 to edx*/

	test    %edx, %edx		  /* edx must be 0 if there is no null char in rsi*/
	jnz	LABEL(aligned_16bytes)

LABEL(ashr_0_loop):
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_aligned)
#endif
	movdqa  (%rsi, %rcx), %xmm1
	movdqa  %xmm1, (%rdi, %rcx)
	add	$16, %rcx
	pcmpeqb  (%rsi, %rcx), %xmm0
	pmovmskb  %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(aligned_exit)

#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_aligned)
#endif
	movdqa  (%rsi, %rcx), %xmm1
	movdqa  %xmm1, (%rdi, %rcx)
	add	$16, %rcx
	pcmpeqb  (%rsi, %rcx), %xmm0
	pmovmskb  %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(aligned_exit)

#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_aligned)
#endif
	movdqa  (%rsi, %rcx), %xmm1
	movdqa  %xmm1, (%rdi, %rcx)
	add	$16, %rcx
	pcmpeqb  (%rsi, %rcx), %xmm0
	pmovmskb  %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(aligned_exit)

#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_aligned)
#endif
	movdqa  (%rsi, %rcx), %xmm1
	movdqa  %xmm1, (%rdi, %rcx)
	add	$16, %rcx
	pcmpeqb  (%rsi, %rcx), %xmm0
	pmovmskb  %xmm0, %edx
	test	%edx, %edx
	jz	LABEL(ashr_0_loop)

	jmp	LABEL(aligned_exit)
        .p2align 4

/*
 * The following cases will be handled by ashr_15
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(15)		n - 15		15((16 - (n -15) + n)%16	 ashr_15
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_15):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_15_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $15, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $15, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_15_use_ssse3)

/*
 * The following cases will be handled by ashr_14
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(14~15)		n - 14		14((16 - (n -14) + n)%16	 ashr_14
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_14):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_14_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $14, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $14, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_14_use_ssse3)

/*
 * The following cases will be handled by ashr_13
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(13~15)		n - 13		13((16 - (n -13) + n)%16	 ashr_13
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_13):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_13_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $13, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $13, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_13_use_ssse3)

/*
 * The following cases will be handled by ashr_12
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(12~15)		n - 12		12((16 - (n -12) + n)%16	 ashr_12
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_12):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_12_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $12, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $12, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_12_use_ssse3)

/*
 * The following cases will be handled by ashr_11
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(11~15)		n - 11		11((16 - (n -11) + n)%16	 ashr_11
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_11):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_11_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $11, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $11, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_11_use_ssse3)

/*
 * The following cases will be handled by ashr_10
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(10~15)		n - 10		10((16 - (n -10) + n)%16	 ashr_10
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_10):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_10_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $10, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $10, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_10_use_ssse3)

/*
 * The following cases will be handled by ashr_9
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(9~15)		n - 9		9((16 - (n -9) + n)%16	 ashr_9
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_9):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_9_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $9, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $9, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_9_use_ssse3)

/*
 * The following cases will be handled by ashr_8
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(8~15)		n - 8		8((16 - (n -8) + n)%16	 ashr_8
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_8):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_8_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $8, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $8, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_8_use_ssse3)

/*
 * The following cases will be handled by ashr_7
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(7~15)		n - 7		7((16 - (n -7) + n)%16	 ashr_7
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_7):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	.p2align 4

LABEL(ashr_7_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $7, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $7, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_7_use_ssse3)

/*
 * The following cases will be handled by ashr_6
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(6~15)		n - 6		6((16 - (n -6) + n)%16	 ashr_6
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_6):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_6_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $6, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $6, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_6_use_ssse3)

 /*
 * The following cases will be handled by ashr_5
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(5~15)		n - 5		5((16 - (n -5) + n)%16	 ashr_5
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_5):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_5_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $5, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $5, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_5_use_ssse3)

/*
 *
 * The following cases will be handled by ashr_4
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(4~15)		n - 4		4((16 - (n -4) + n)%16	 ashr_4
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_4):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_4_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $4, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $4, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_4_use_ssse3)

/*
 *
 * The following cases will be handled by ashr_3
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(3~15)		n - 3		3((16 - (n -3) + n)%16	 ashr_3
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_3):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_3_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $3, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $3, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_3_use_ssse3)

/*
 *
 * The following cases will be handled by ashr_2
 *  rcx(offset of rsi)  rax(offset of rdi)	relative offset  	  corresponding case
 *      n(2~15)		n - 2		2((16 - (n -2) + n)%16	 ashr_2
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_2):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_2_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $2, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $2, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_2_use_ssse3)

/*
 *
 * The following cases will be handled by ashr_1
 *  rcx(offset of rsi)  rax(offset of rdi)  relative offset  		corresponding case
 *	n(1~15)		n - 1	   	1 ((16 - (n -1) + n)%16	 ashr_1
 *
 * Based on above operation , start from  (%r9 + rsi) to the left of this cache bank, there is no null byte
 */
	.p2align 4
LABEL(ashr_1):
	xor	%ecx, %ecx				/*clear ecx */
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	.p2align 4
LABEL(ashr_1_use_ssse3):
	movdqa	16(%rsi, %rcx), %xmm3
	pcmpeqb	%xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif

	palignr $1, (%rsi, %rcx), %xmm3
	movdqa	%xmm3, (%rdi, %rcx)
	add	$16, %rcx
#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif

	movdqa  16(%rsi, %rcx), %xmm3
	pcmpeqb %xmm3, %xmm0
	pmovmskb %xmm0, %edx
	test	%edx, %edx
	jnz	LABEL(unaligned_exit)
#ifdef USE_AS_STRNCPY
	sub	$16, %r8
	jbe	LABEL(strncpy_truncation_unaligned)
#endif
	palignr $1, (%rsi, %rcx), %xmm3
	movdqa  %xmm3, (%rdi, %rcx)
	add	$16, %rcx

#ifdef USE_AS_STRNCPY
	cmp	%r10, %r8
	jbe	LABEL(unaligned_exit)
#endif
	jmp	LABEL(ashr_1_use_ssse3)

	.p2align 4
LABEL(less32bytes):
	xor	%ecx, %ecx
LABEL(unaligned_exit):
	add	%r9, %rsi		/* r9 stores original offset of rsi*/
	mov	%rcx, %r9
	mov	%r10, %rcx
	shl	%cl, %edx		/* after shl, calculate the exact number to be filled*/
	mov	%r9, %rcx
	.p2align 4
LABEL(aligned_exit):
	add	%rcx, %rdi		/*locate exact address for rdi */
LABEL(less16bytes):
	add	%rcx, %rsi		/*locate exact address for rsi */
LABEL(aligned_16bytes):
#ifdef USE_AS_STRNCPY
	mov     $1, %r9d
	lea     -1(%r8), %rcx
	shl     %cl, %r9d
	cmp     $32, %r8
	ja      LABEL(strncpy_tail)
	or      %r9d, %edx
LABEL(strncpy_tail):
#endif
	bsf	%rdx, %rcx		/*If a least significant 1 bit in %rdx is found, its bit index is stored in %rcx*/
	lea	LABEL(tail_table)(%rip), %r11
	movslq	(%r11, %rcx,4), %rcx
	lea	(%r11, %rcx), %rcx
	jmp	*%rcx

#ifdef USE_AS_STRNCPY
	.p2align 4
LABEL(less32bytes_strncpy_truncation):
	xor     %ecx, %ecx
LABEL(strncpy_truncation_unaligned):
	add      %r9, %rsi
LABEL(strncpy_truncation_aligned):
	add      %rcx, %rdi
	add      %rcx, %rsi
	add     $16, %r8
	lea     -1(%r8), %rcx
	lea     LABEL(tail_table)(%rip), %r11
	movslq  (%r11, %rcx,4), %rcx
	lea     (%r11, %rcx), %rcx
	jmp     *%rcx
	.p2align 4
LABEL(strncpy_exitz):
	mov     %rdi, %rax
	ret
#endif

#ifdef USE_AS_STRNCPY
	.p2align 4
LABEL(strncpy_fill_tail):
	mov	%rax, %rdx
	movzx	%cl, %rax
	mov	%r8, %rcx
	add	%rax, %rdi
	xor	%eax, %eax
	shr	$3, %ecx
	jz	LABEL(strncpy_fill_less_8)

	rep	stosq
LABEL(strncpy_fill_less_8):
	mov	%r8, %rcx
	and	$7, %ecx
	jz	LABEL(strncpy_fill_return)
LABEL(strncpy_fill_less_7):
	sub	$1, %ecx
	mov	%al, (%rdi, %rcx)
	jnz	LABEL(strncpy_fill_less_7)
LABEL(strncpy_fill_return):
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rdx)
	sbb	$-1, %rdx
#endif
	mov	%rdx, %rax
	ret
#endif
	.p2align 4
LABEL(tail_0):
	mov	(%rsi), %cl
	mov	%cl, (%rdi)
#ifdef USE_AS_STPCPY
	mov	%rdi, %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$1, %cl
	sub	$1, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_1):
	mov	(%rsi), %cx
	mov	%cx, (%rdi)
#ifdef USE_AS_STPCPY
	lea	1(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$2, %cl
	sub	$2, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_2):
	mov	(%rsi), %cx
	mov	%cx, (%rdi)
	mov	1(%rsi), %cx
	mov	%cx, 1(%rdi)
#ifdef USE_AS_STPCPY
	lea	2(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$3, %cl
	sub	$3, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_3):
	mov	(%rsi), %ecx
	mov	%ecx, (%rdi)
#ifdef USE_AS_STPCPY
	lea	3(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$4, %cl
	sub	$4, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_4):
	mov	(%rsi), %ecx
	mov	%ecx, (%rdi)
	mov	1(%rsi), %edx
	mov	%edx, 1(%rdi)
#ifdef USE_AS_STPCPY
	lea	4(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$5, %cl
	sub	$5, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_5):
	mov	(%rsi), %ecx
	mov	%ecx, (%rdi)
	mov	2(%rsi), %edx
	mov	%edx, 2(%rdi)
#ifdef USE_AS_STPCPY
	lea	5(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$6, %cl
	sub	$6, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_6):
	mov	(%rsi), %ecx
	mov	%ecx, (%rdi)
	mov	3(%rsi), %edx
	mov	%edx,3(%rdi)
#ifdef USE_AS_STPCPY
	lea	6(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$7, %cl
	sub	$7, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_7):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
#ifdef USE_AS_STPCPY
	lea	7(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$8, %cl
	sub	$8, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_8):

	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	5(%rsi), %edx
	mov	%edx, 5(%rdi)
#ifdef USE_AS_STPCPY
	lea	8(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$9, %cl
	sub	$9, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_9):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	6(%rsi), %edx
	mov	%edx, 6(%rdi)
#ifdef USE_AS_STPCPY
	lea	9(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$10, %cl
	sub	$10, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_10):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	7(%rsi), %edx
	mov	%edx, 7(%rdi)
#ifdef USE_AS_STPCPY
	lea	10(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$11, %cl
	sub	$11, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_11):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %edx
	mov	%edx, 8(%rdi)
#ifdef USE_AS_STPCPY
	lea	11(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$12, %cl
	sub	$12, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_12):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	5(%rsi), %rcx
	mov	%rcx, 5(%rdi)
#ifdef USE_AS_STPCPY
	lea	12(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$13, %cl
	sub	$13, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_13):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	6(%rsi), %rcx
	mov	%rcx, 6(%rdi)
#ifdef USE_AS_STPCPY
	lea	13(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$14, %cl
	sub	$14, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_14):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	7(%rsi), %rcx
	mov	%rcx, 7(%rdi)
#ifdef USE_AS_STPCPY
	lea	14(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$15, %cl
	sub	$15, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

LABEL(tail_15):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
#ifdef USE_AS_STPCPY
	lea	15(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$16, %cl
	sub	$16, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif

	ret

	.p2align 4
LABEL(tail_16):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %cl
	mov	%cl, 16(%rdi)
#ifdef USE_AS_STPCPY
	lea	16(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$17, %cl
	sub	$17, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_17):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %cx
	mov	%cx, 16(%rdi)
#ifdef USE_AS_STPCPY
	lea	17(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$18, %cl
	sub	$18, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_18):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	15(%rsi), %ecx
	mov	%ecx,15(%rdi)
#ifdef USE_AS_STPCPY
	lea	18(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$19, %cl
	sub	$19, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_19):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %ecx
	mov	%ecx, 16(%rdi)
#ifdef USE_AS_STPCPY
	lea	19(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$20, %cl
	sub	$20, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_20):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	13(%rsi), %rcx
	mov	%rcx, 13(%rdi)
#ifdef USE_AS_STPCPY
	lea	20(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$21, %cl
	sub	$21, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_21):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	14(%rsi), %rcx
	mov	%rcx, 14(%rdi)
#ifdef USE_AS_STPCPY
	lea	21(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$22, %cl
	sub	$22, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_22):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	15(%rsi), %rcx
	mov	%rcx, 15(%rdi)
#ifdef USE_AS_STPCPY
	lea	22(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$23, %cl
	sub	$23, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_23):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
#ifdef USE_AS_STPCPY
	lea	23(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$24, %cl
	sub	$24, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif

	ret

	.p2align 4
LABEL(tail_24):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	21(%rsi), %edx
	mov	%edx, 21(%rdi)
#ifdef USE_AS_STPCPY
	lea	24(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$25, %cl
	sub	$25, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_25):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	22(%rsi), %edx
	mov	%edx, 22(%rdi)
#ifdef USE_AS_STPCPY
	lea	25(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$26, %cl
	sub	$26, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_26):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	23(%rsi), %edx
	mov	%edx, 23(%rdi)
#ifdef USE_AS_STPCPY
	lea	26(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$27, %cl
	sub	$27, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_27):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	24(%rsi), %edx
	mov	%edx, 24(%rdi)
#ifdef USE_AS_STPCPY
	lea	27(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$28, %cl
	sub	$28, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	.p2align 4
LABEL(tail_28):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	21(%rsi), %rdx
	mov	%rdx, 21(%rdi)
#ifdef USE_AS_STPCPY
	lea	28(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$29, %cl
	sub	$29, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif

	ret

	.p2align 4
LABEL(tail_29):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	22(%rsi), %rdx
	mov	%rdx, 22(%rdi)
#ifdef USE_AS_STPCPY
	lea	29(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$30, %cl
	sub	$30, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif

	ret


	.p2align 4
LABEL(tail_30):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	23(%rsi), %rdx
	mov	%rdx, 23(%rdi)
#ifdef USE_AS_STPCPY
	lea	30(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$31, %cl
	sub	$31, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret

	.p2align 4
LABEL(tail_31):
	mov	(%rsi), %rcx
	mov	%rcx, (%rdi)
	mov	8(%rsi), %rdx
	mov	%rdx, 8(%rdi)
	mov	16(%rsi), %rcx
	mov	%rcx, 16(%rdi)
	mov	24(%rsi), %rdx
	mov	%rdx, 24(%rdi)
#ifdef USE_AS_STPCPY
	lea	31(%rdi), %rax
#endif
#ifdef USE_AS_STRNCPY
	mov	$32, %cl
	sub	$32, %r8
	jnz	LABEL(strncpy_fill_tail)
#ifdef USE_AS_STPCPY
	cmpb	$1, (%rax)
	sbb	$-1, %rax
#endif
#endif
	ret
	cfi_endproc
	.size	STRCPY_SSSE3, .-STRCPY_SSSE3

	.p2align 4
	.section .rodata.ssse3,"a",@progbits
LABEL(tail_table):
	.int	LABEL(tail_0) - LABEL(tail_table)
	.int	LABEL(tail_1) - LABEL(tail_table)
	.int	LABEL(tail_2) - LABEL(tail_table)
	.int	LABEL(tail_3) - LABEL(tail_table)
	.int	LABEL(tail_4) - LABEL(tail_table)
	.int	LABEL(tail_5) - LABEL(tail_table)
	.int	LABEL(tail_6) - LABEL(tail_table)
	.int	LABEL(tail_7) - LABEL(tail_table)
	.int	LABEL(tail_8) - LABEL(tail_table)
	.int	LABEL(tail_9) - LABEL(tail_table)
	.int	LABEL(tail_10) - LABEL(tail_table)
	.int	LABEL(tail_11) - LABEL(tail_table)
	.int	LABEL(tail_12) - LABEL(tail_table)
	.int	LABEL(tail_13) - LABEL(tail_table)
	.int	LABEL(tail_14) - LABEL(tail_table)
	.int	LABEL(tail_15) - LABEL(tail_table)
	.int	LABEL(tail_16) - LABEL(tail_table)
	.int	LABEL(tail_17) - LABEL(tail_table)
	.int	LABEL(tail_18) - LABEL(tail_table)
	.int	LABEL(tail_19) - LABEL(tail_table)
	.int	LABEL(tail_20) - LABEL(tail_table)
	.int	LABEL(tail_21) - LABEL(tail_table)
	.int	LABEL(tail_22) - LABEL(tail_table)
	.int	LABEL(tail_23) - LABEL(tail_table)
	.int	LABEL(tail_24) - LABEL(tail_table)
	.int	LABEL(tail_25) - LABEL(tail_table)
	.int	LABEL(tail_26) - LABEL(tail_table)
	.int	LABEL(tail_27) - LABEL(tail_table)
	.int	LABEL(tail_28) - LABEL(tail_table)
	.int	LABEL(tail_29) - LABEL(tail_table)
	.int	LABEL(tail_30) - LABEL(tail_table)
	.int	LABEL(tail_31) - LABEL(tail_table)

	.p2align 4
LABEL(unaligned_table):
	.int	LABEL(ashr_0) - LABEL(unaligned_table)
	.int	LABEL(ashr_1) - LABEL(unaligned_table)
	.int	LABEL(ashr_2) - LABEL(unaligned_table)
	.int	LABEL(ashr_3) - LABEL(unaligned_table)
	.int	LABEL(ashr_4) - LABEL(unaligned_table)
	.int	LABEL(ashr_5) - LABEL(unaligned_table)
	.int	LABEL(ashr_6) - LABEL(unaligned_table)
	.int	LABEL(ashr_7) - LABEL(unaligned_table)
	.int	LABEL(ashr_8) - LABEL(unaligned_table)
	.int	LABEL(ashr_9) - LABEL(unaligned_table)
	.int	LABEL(ashr_10) - LABEL(unaligned_table)
	.int	LABEL(ashr_11) - LABEL(unaligned_table)
	.int	LABEL(ashr_12) - LABEL(unaligned_table)
	.int	LABEL(ashr_13) - LABEL(unaligned_table)
	.int	LABEL(ashr_14) - LABEL(unaligned_table)
	.int	LABEL(ashr_15) - LABEL(unaligned_table)

# undef ENTRY
# define ENTRY(name) \
	.type STRCPY_SSE2, @function; \
	.align 16; \
	STRCPY_SSE2: cfi_startproc; \
	CALL_MCOUNT
# undef END
# define END(name) \
	cfi_endproc; .size STRCPY_SSE2, .-STRCPY_SSE2
# undef libc_hidden_builtin_def
/* It doesn't make sense to send libc-internal strcpy calls through a PLT.
   The speedup we get from using SSSE3 instruction is likely eaten away
   by the indirect call in the PLT.  */
# define libc_hidden_builtin_def(name) \
	.globl __GI_STRCPY; __GI_STRCPY = STRCPY_SSE2
# undef libc_hidden_def
# define libc_hidden_def(name) \
	.globl __GI___STRCPY; __GI___STRCPY = STRCPY_SSE2
#endif

#ifndef USE_AS_STRNCPY
#include "../strcpy.S"
#endif