libavcodec/x86/hevc_idct.asm - manifest_repos/ffmpeg - Git at Google

 ;*******************************************************************************
 ;* SIMD-optimized IDCT functions for HEVC decoding
 ;* Copyright (c) 2014 Pierre-Edouard LEPERE
 ;* Copyright (c) 2014 James Almer
 ;* Copyright (c) 2016 Alexandra Hájková
 ;*
 ;* This file is part of FFmpeg.
 ;*
 ;* FFmpeg is free software; you can redistribute it and/or
 ;* modify it under the terms of the GNU Lesser General Public
 ;* License as published by the Free Software Foundation; either
 ;* version 2.1 of the License, or (at your option) any later version.
 ;*
 ;* FFmpeg is distributed in the hope that it will be useful,
 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 ;* Lesser General Public License for more details.
 ;*
 ;* You should have received a copy of the GNU Lesser General Public
 ;* License along with FFmpeg; if not, write to the Free Software
 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 ;******************************************************************************

 %include "libavutil/x86/x86util.asm"

 SECTION_RODATA

 pd_64: times 4 dd 64
 pd_2048: times 4 dd 2048
 pd_512: times 4 dd 512

 ; 4x4 transform coeffs
 cextern pw_64
 pw_64_m64: times 4 dw 64, -64
 pw_83_36: times 4 dw 83, 36
 pw_36_m83: times 4 dw 36, -83

 ; 8x8 transform coeffs
 pw_89_75: times 4 dw 89, 75
 pw_50_18: times 4 dw 50, 18

 pw_75_m18: times 4 dw 75, -18
 pw_m89_m50: times 4 dw -89, -50

 pw_50_m89: times 4 dw 50, -89
 pw_18_75: times 4 dw 18, 75

 pw_18_m50: times 4 dw 18, -50
 pw_75_m89: times 4 dw 75, -89

 ; 16x16 transformation coeffs
 trans_coeffs16: times 4 dw 90, 87
 times 4 dw 80, 70
 times 4 dw 57, 43
 times 4 dw 25, 9

 times 4 dw 87, 57
 times 4 dw 9, -43
 times 4 dw -80, -90
 times 4 dw -70, -25

 times 4 dw 80, 9
 times 4 dw -70, -87
 times 4 dw -25, 57
 times 4 dw 90, 43

 times 4 dw 70, -43
 times 4 dw -87, 9
 times 4 dw 90, 25
 times 4 dw -80, -57

 times 4 dw 57, -80
 times 4 dw -25, 90
 times 4 dw -9, -87
 times 4 dw 43, 70

 times 4 dw 43, -90
 times 4 dw 57, 25
 times 4 dw -87, 70
 times 4 dw 9, -80

 times 4 dw 25, -70
 times 4 dw 90, -80
 times 4 dw 43, 9
 times 4 dw -57, 87

 times 4 dw 9, -25
 times 4 dw 43, -57
 times 4 dw 70, -80
 times 4 dw 87, -90

 ; 32x32 transform coeffs
 trans_coeff32: times 8 dw 90
 times 4 dw 88, 85
 times 4 dw 82, 78
 times 4 dw 73, 67
 times 4 dw 61, 54
 times 4 dw 46, 38
 times 4 dw 31, 22
 times 4 dw 13, 4

 times 4 dw 90, 82
 times 4 dw 67, 46
 times 4 dw 22, -4
 times 4 dw -31, -54
 times 4 dw -73, -85
 times 4 dw -90, -88
 times 4 dw -78, -61
 times 4 dw -38, -13

 times 4 dw 88, 67
 times 4 dw 31, -13
 times 4 dw -54, -82
 times 4 dw -90, -78
 times 4 dw -46, -4
 times 4 dw 38, 73
 times 4 dw 90, 85
 times 4 dw 61, 22

 times 4 dw 85, 46
 times 4 dw -13, -67
 times 4 dw -90, -73
 times 4 dw -22, 38
 times 4 dw 82, 88
 times 4 dw 54, -4
 times 4 dw -61, -90
 times 4 dw -78, -31

 times 4 dw 82, 22
 times 4 dw -54, -90
 times 4 dw -61, 13
 times 4 dw 78, 85
 times 4 dw 31, -46
 times 4 dw -90, -67
 times 4 dw 4, 73
 times 4 dw 88, 38

 times 4 dw 78, -4
 times 4 dw -82, -73
 times 4 dw 13, 85
 times 4 dw 67, -22
 times 4 dw -88, -61
 times 4 dw 31, 90
 times 4 dw 54, -38
 times 4 dw -90, -46

 times 4 dw 73, -31
 times 4 dw -90, -22
 times 4 dw 78, 67
 times 4 dw -38, -90
 times 4 dw -13, 82
 times 4 dw 61, -46
 times 4 dw -88, -4
 times 4 dw 85, 54

 times 4 dw 67, -54
 times 4 dw -78, 38
 times 4 dw 85, -22
 times 4 dw -90, 4
 times 4 dw 90, 13
 times 4 dw -88, -31
 times 4 dw 82, 46
 times 4 dw -73, -61

 times 4 dw 61, -73
 times 4 dw -46, 82
 times 4 dw 31, -88
 times 4 dw -13, 90
 times 4 dw -4, -90
 times 4 dw 22, 85
 times 4 dw -38, -78
 times 4 dw 54, 67

 times 4 dw 54, -85
 times 4 dw -4, 88
 times 4 dw -46, -61
 times 4 dw 82, 13
 times 4 dw -90, 38
 times 4 dw 67, -78
 times 4 dw -22, 90
 times 4 dw -31, -73

 times 4 dw 46, -90
 times 4 dw 38, 54
 times 4 dw -90, 31
 times 4 dw 61, -88
 times 4 dw 22, 67
 times 4 dw -85, 13
 times 4 dw 73, -82
 times 4 dw 4, 78

 times 4 dw 38, -88
 times 4 dw 73, -4
 times 4 dw -67, 90
 times 4 dw -46, -31
 times 4 dw 85, -78
 times 4 dw 13, 61
 times 4 dw -90, 54
 times 4 dw 22, -82

 times 4 dw 31, -78
 times 4 dw 90, -61
 times 4 dw 4, 54
 times 4 dw -88, 82
 times 4 dw -38, -22
 times 4 dw 73, -90
 times 4 dw 67, -13
 times 4 dw -46, 85

 times 4 dw 22, -61
 times 4 dw 85, -90
 times 4 dw 73, -38
 times 4 dw -4, 46
 times 4 dw -78, 90
 times 4 dw -82, 54
 times 4 dw -13, -31
 times 4 dw 67, -88

 times 4 dw 13, -38
 times 4 dw 61, -78
 times 4 dw 88, -90
 times 4 dw 85, -73
 times 4 dw 54, -31
 times 4 dw 4, 22
 times 4 dw -46, 67
 times 4 dw -82, 90

 times 4 dw 4, -13
 times 4 dw 22, -31
 times 4 dw 38, -46
 times 4 dw 54, -61
 times 4 dw 67, -73
 times 4 dw 78, -82
 times 4 dw 85, -88
 times 4 dw 90, -90

 SECTION .text

 ; void ff_hevc_idct_HxW_dc_{8,10}_<opt>(int16_t *coeffs)
 ; %1 = HxW
 ; %2 = number of loops
 ; %3 = bitdepth
 %macro IDCT_DC 3
 cglobal hevc_idct_%1x%1_dc_%3, 1, 2, 1, coeff, tmp
     movsx             tmpd, word [coeffq]
     add               tmpd, (1 << (14 - %3)) + 1
     sar               tmpd, (15 - %3)
     movd               xm0, tmpd
     SPLATW              m0, xm0
     DEFINE_ARGS coeff, cnt
     mov               cntd, %2
 .loop:
     mova [coeffq+mmsize*0], m0
     mova [coeffq+mmsize*1], m0
     mova [coeffq+mmsize*2], m0
     mova [coeffq+mmsize*3], m0
     add  coeffq, mmsize*8
     mova [coeffq+mmsize*-4], m0
     mova [coeffq+mmsize*-3], m0
     mova [coeffq+mmsize*-2], m0
     mova [coeffq+mmsize*-1], m0
     dec  cntd
     jg  .loop
     RET
 %endmacro

 ; %1 = HxW
 ; %2 = bitdepth
 %macro IDCT_DC_NL 2 ; No loop
 cglobal hevc_idct_%1x%1_dc_%2, 1, 2, 1, coeff, tmp
     movsx             tmpd, word [coeffq]
     add               tmpd, (1 << (14 - %2)) + 1
     sar               tmpd, (15 - %2)
     movd                m0, tmpd
     SPLATW              m0, xm0
     mova [coeffq+mmsize*0], m0
     mova [coeffq+mmsize*1], m0
     mova [coeffq+mmsize*2], m0
     mova [coeffq+mmsize*3], m0
 %if mmsize == 16
     mova [coeffq+mmsize*4], m0
     mova [coeffq+mmsize*5], m0
     mova [coeffq+mmsize*6], m0
     mova [coeffq+mmsize*7], m0
 %endif
     RET
 %endmacro

 ; IDCT 4x4, expects input in m0, m1
 ; %1 - shift
 ; %2 - 1/0 - SCALE and Transpose or not
 ; %3 - 1/0 add constant or not
 %macro TR_4x4 3
     ; interleaves src0 with src2 to m0
     ;         and src1 with scr3 to m2
     ; src0: 00 01 02 03     m0: 00 20 01 21 02 22 03 23
     ; src1: 10 11 12 13 -->
     ; src2: 20 21 22 23     m1: 10 30 11 31 12 32 13 33
     ; src3: 30 31 32 33

     SBUTTERFLY wd, 0, 1, 2

     pmaddwd m2, m0, [pw_64]    ; e0
     pmaddwd m3, m1, [pw_83_36] ; o0
     pmaddwd m0, [pw_64_m64]    ; e1
     pmaddwd m1, [pw_36_m83]    ; o1

 %if %3 == 1
     %assign %%add 1 << (%1 - 1)
     mova  m4, [pd_ %+ %%add]
     paddd m2, m4
     paddd m0, m4
 %endif

     SUMSUB_BADC d, 3, 2, 1, 0, 4

 %if %2 == 1
     psrad m3, %1 ; e0 + o0
     psrad m1, %1 ; e1 + o1
     psrad m2, %1 ; e0 - o0
     psrad m0, %1 ; e1 - o1
     ;clip16
     packssdw m3, m1
     packssdw m0, m2
     ; Transpose
     SBUTTERFLY wd, 3, 0, 1
     SBUTTERFLY wd, 3, 0, 1
     SWAP 3, 1, 0
 %else
     SWAP 3, 2, 0
 %endif
 %endmacro

 %macro DEFINE_BIAS 1
     %assign shift (20 - %1)
     %assign c_add (1 << (shift - 1))
     %define arr_add pd_ %+ c_add
 %endmacro

 ; %1 - bit_depth
 ; %2 - register add constant
 ; is loaded to
 ; shift = 20 - bit_depth
 %macro LOAD_BIAS 2
     DEFINE_BIAS %1
     mova %2, [arr_add]
 %endmacro

 ; %1, %2 - registers to load packed 16 bit values to
 ; %3, %4, %5, %6 - vertical offsets
 ; %7 - horizontal offset
 %macro LOAD_BLOCK 7
     movq   %1, [r0 + %3 + %7]
     movhps %1, [r0 + %5 + %7]
     movq   %2, [r0 + %4 + %7]
     movhps %2, [r0 + %6 + %7]
 %endmacro

 ; void ff_hevc_idct_4x4__{8,10}_<opt>(int16_t *coeffs, int col_limit)
 ; %1 = bitdepth
 %macro IDCT_4x4 1
 cglobal hevc_idct_4x4_%1, 1, 1, 5, coeffs
     mova m0, [coeffsq]
     mova m1, [coeffsq + 16]

     TR_4x4 7, 1, 1
     TR_4x4 20 - %1, 1, 1

     mova [coeffsq],      m0
     mova [coeffsq + 16], m1
     RET
 %endmacro

 ; scale, pack (clip16) and store the residuals     0 e8[0] + o8[0] --> + %1
 ; 4 at one time (4 columns)                        1 e8[1] + o8[1]
 ; from %5: e8/16 + o8/16, with %1 offset                  ...
 ; and  %3: e8/16 - o8/16, with %2 offset           6 e8[1] - o8[1]
 ; %4 - shift                                       7 e8[0] - o8[0] --> + %2
 %macro STORE_8 7
     psrad    %5, %4
     psrad    %3, %4
     packssdw %5, %3
     movq     [coeffsq + %1], %5
     movhps   [coeffsq + %2], %5
 %endmacro

 ; %1 - horizontal offset
 ; %2 - shift
 ; %3, %4 - transform coeffs
 ; %5 - vertical offset for e8 + o8
 ; %6 - vertical offset for e8 - o8
 ; %7 - register with e8 inside
 ; %8 - block_size
 ; %9 - register to store e8 +o8
 ; %10 - register to store e8 - o8
 %macro E8_O8 10
     pmaddwd m6, m4, %3
     pmaddwd m7, m5, %4

     paddd m6, m7
     paddd m7, m6, %7 ; o8 + e8
     psubd %7, m6     ; e8 - o8
 %if %8 == 8
     STORE_8 %5 + %1, %6 + %1, %7, %2, m7, 0, 0
 %else
     SWAP m7, %9
     SWAP %7, %10
 %endif
 %endmacro

 ; 8x4 residuals are processed and stored
 ; %1 - horizontal offset
 ; %2 - shift
 ; %3 - offset of the even row
 ; %4 - step: 1 for 8x8, 2 for 16x16, 4 for 32x32
 ; %5 - offset of the odd row
 ; %6 - block size
 ; %7 - 1/0 add a constant in TR_4x4 or not
 ; I want to add a constant for 8x8 transform but not for 16x16 and 32x32
 %macro TR_8x4 7
     ; load 4 columns of even rows
     LOAD_BLOCK  m0, m1, 0, 2 * %4 * %3, %4 * %3, 3 * %4 * %3, %1

     TR_4x4 %2, 0, %7 ; e8: m0, m1, m2, m3, for 4 columns only

     ; load 4 columns of odd rows
     LOAD_BLOCK m4, m5, %4 * %5, 3 * %4 * %5, 5 * %4 * %5, 7 * %4 * %5, %1

     ; 00 01 02 03
     ; 10 11 12 13      m4: 10 30 11 31 12 32 13 33

     ; ...        -- >
     ;                  m5: 50 70 51 71 52 72 53 73
     ; 70 71 72 73
     SBUTTERFLY wd, 4, 5, 6

     E8_O8 %1, %2, [pw_89_75],  [pw_50_18],   0,      %5 * 7, m0, %6, m8, m15
     E8_O8 %1, %2, [pw_75_m18], [pw_m89_m50], %5,     %5 * 6, m1, %6, m9, m14
     E8_O8 %1, %2, [pw_50_m89], [pw_18_75],   %5 * 2, %5 * 5, m2, %6, m10, m13
     E8_O8 %1, %2, [pw_18_m50], [pw_75_m89],  %5 * 3, %5 * 4, m3, %6, m11, m12
 %endmacro

 %macro STORE_PACKED 7
     movq   [r0 + %3 + %7], %1
     movhps [r0 + %4 + %7], %1
     movq   [r0 + %5 + %7], %2
     movhps [r0 + %6 + %7], %2
 %endmacro

 ; transpose 4x4 block packed
 ; in %1 and %2 registers
 ; %3 - temporary register
 %macro TRANSPOSE_4x4 3
     SBUTTERFLY wd, %1, %2, %3
     SBUTTERFLY dq, %1, %2, %3
 %endmacro

 ; %1 - horizontal offset of the block i
 ; %2 - vertical offset of the block i
 ; %3 - width in bytes
 ; %4 - vertical offset for the block j
 ; %5 - horizontal offset for the block j
 %macro SWAP_BLOCKS 5
     ; M_j
     LOAD_BLOCK m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
     TRANSPOSE_4x4 4, 5, 6

     ; M_i
     LOAD_BLOCK m6, m7, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

     STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

     ; transpose and store M_i
     SWAP m6, m4
     SWAP m7, m5
     TRANSPOSE_4x4 4, 5, 6
     STORE_PACKED m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
 %endmacro

 ; %1 - horizontal offset
 ; %2 - vertical offset of the block
 ; %3 - width in bytes
 %macro TRANSPOSE_BLOCK 3
     LOAD_BLOCK m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
     TRANSPOSE_4x4 4, 5, 6
     STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
 %endmacro

 %macro TRANSPOSE_8x8 0
 cglobal hevc_idct_transpose_8x8, 0, 0, 0
     ; M1 M2 ^T = M1^t M3^t
     ; M3 M4      M2^t M4^t

     ; M1 4x4 block
     TRANSPOSE_BLOCK 0, 0, 16

     ; M2 and M3
     SWAP_BLOCKS 0, 64, 16, 0, 8

     ; M4
     TRANSPOSE_BLOCK 8, 64, 16

     ret
 %endmacro

 ; void ff_hevc_idct_8x8_{8,10}_<opt>(int16_t *coeffs, int col_limit)
 ; %1 = bitdepth
 %macro IDCT_8x8 1
 cglobal hevc_idct_8x8_%1, 1, 1, 8, coeffs
     TR_8x4 0, 7, 32, 1, 16, 8, 1
     TR_8x4 8, 7, 32, 1, 16, 8, 1

     call hevc_idct_transpose_8x8_ %+ cpuname

     DEFINE_BIAS %1
     TR_8x4 0, shift, 32, 1, 16, 8, 1
     TR_8x4 8, shift, 32, 1, 16, 8, 1

     TAIL_CALL hevc_idct_transpose_8x8_ %+ cpuname, 1
 %endmacro

 ; store intermedite e32 coeffs on stack
 ; as 16x4 matrix
 ; from m10: e8 + o8, with %6 offset
 ; and  %3:  e8 - o8, with %7 offset
 ; %4 - shift, unused here
 %macro STORE_16 7
     mova [rsp + %6], %5
     mova [rsp + %7], %3
 %endmacro

 ; %1, %2 - transform constants
 ; %3, %4 - regs with interleaved coeffs
 ; %5 - 1/0 SWAP or add
 ; %6, %7 - registers for intermidiate sums
 ; %8 - accumulator register
 %macro ADD_ROWS 8
     pmaddwd %6, %3, %1
     pmaddwd %7, %4, %2
     paddd   %6, %7
 %if %5 == 1
     SWAP %6, %8
 %else
     paddd %8, %6
 %endif
 %endmacro

 ; %1 - transform coeffs
 ; %2, %3 offsets for storing e+o/e-o back to coeffsq
 ; %4 - shift
 ; %5 - add
 ; %6 - block_size
 ; %7 - register with e16
 ; %8, %9 - stack offsets for storing e+o/e-o
 %macro E16_O16 9
     ADD_ROWS [%1],          [%1 +     16], m0, m1, 1, m5, m6, m7
     ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m5, m6, m7

 %if %6 == 8
     paddd %7, %5
 %endif

     paddd m4, m7, %7 ; o16 + e16
     psubd %7, m7     ; e16 - o16
     STORE_%6 %2, %3, %7, %4, m4, %8, %9
 %endmacro

 %macro TR_16x4 10
     ; produce 8x4 matrix of e16 coeffs
     ; for 4 first rows and store it on stack (128 bytes)
     TR_8x4 %1, 7, %4, %5, %6, %8, 0

     ; load 8 even rows
     LOAD_BLOCK m0, m1, %9 * %6, %9 * 3 * %6, %9 * 5 * %6, %9 * 7 * %6, %1
     LOAD_BLOCK m2, m3, %9 * 9 * %6, %9 * 11 * %6, %9 * 13 * %6, %9 * 15 * %6, %1

     SBUTTERFLY wd, 0, 1, 4
     SBUTTERFLY wd, 2, 3, 4

     E16_O16 trans_coeffs16,               0 + %1, 15 * %6 + %1, %2, %3, %7, m8,       0, 15 * 16
     mova m8, %3
     E16_O16 trans_coeffs16 +     64,     %6 + %1, 14 * %6 + %1, %2, m8, %7, m9,      16, 14 * 16
     E16_O16 trans_coeffs16 + 2 * 64, 2 * %6 + %1, 13 * %6 + %1, %2, m8, %7, m10, 2 * 16, 13 * 16
     E16_O16 trans_coeffs16 + 3 * 64, 3 * %6 + %1, 12 * %6 + %1, %2, m8, %7, m11, 3 * 16, 12 * 16
     E16_O16 trans_coeffs16 + 4 * 64, 4 * %6 + %1, 11 * %6 + %1, %2, m8, %7, m12, 4 * 16, 11 * 16
     E16_O16 trans_coeffs16 + 5 * 64, 5 * %6 + %1, 10 * %6 + %1, %2, m8, %7, m13, 5 * 16, 10 * 16
     E16_O16 trans_coeffs16 + 6 * 64, 6 * %6 + %1,  9 * %6 + %1, %2, m8, %7, m14, 6 * 16,  9 * 16
     E16_O16 trans_coeffs16 + 7 * 64, 7 * %6 + %1,  8 * %6 + %1, %2, m8, %7, m15, 7 * 16,  8 * 16
 %endmacro

 %macro TRANSPOSE_16x16 0
 cglobal hevc_idct_transpose_16x16, 0, 0, 0
 ; M1  M2  M3  M4 ^T      m1 m5 m9  m13   M_i^T = m_i
 ; M5  M6  M7  M8    -->  m2 m6 m10 m14
 ; M9  M10 M11 M12        m3 m7 m11 m15
 ; M13 M14 M15 M16        m4 m8 m12 m16

     ; M1 4x4 block
     TRANSPOSE_BLOCK 0, 0, 32

     ; M5, M2
     SWAP_BLOCKS 0, 128, 32, 0, 8
     ; M9, M3
     SWAP_BLOCKS 0, 256, 32, 0, 16
     ; M13, M4
     SWAP_BLOCKS 0, 384, 32, 0, 24

     ;M6
     TRANSPOSE_BLOCK 8, 128, 32

     ; M10, M7
     SWAP_BLOCKS 8, 256, 32, 128, 16
     ; M14, M8
     SWAP_BLOCKS 8, 384, 32, 128, 24

     ;M11
     TRANSPOSE_BLOCK 16, 256, 32

     ; M15, M12
     SWAP_BLOCKS 16, 384, 32, 256, 24

     ;M16
     TRANSPOSE_BLOCK 24, 384, 32

     ret
 %endmacro

 ; void ff_hevc_idct_16x16_{8,10}_<opt>(int16_t *coeffs, int col_limit)
 ; %1 = bitdepth
 %macro IDCT_16x16 1
 cglobal hevc_idct_16x16_%1, 1, 2, 16, coeffs
     mov r1d, 3
 .loop16:
     TR_16x4 8 * r1, 7, [pd_64], 64, 2, 32, 8, 16, 1, 0
     dec r1d
     jge .loop16

     call hevc_idct_transpose_16x16_ %+ cpuname

     DEFINE_BIAS %1
     mov r1d, 3
 .loop16_2:
     TR_16x4 8 * r1, shift, [arr_add], 64, 2, 32, 8, 16, 1, 1
     dec r1d
     jge .loop16_2

     TAIL_CALL hevc_idct_transpose_16x16_ %+ cpuname, 1
 %endmacro

 ; scale, pack (clip16) and store the residuals     0 e32[0] + o32[0] --> %1
 ; 4 at one time (4 columns)                        1 e32[1] + o32[1]
 ; %1 - address to store e32 + o32
 ; %2 - address to store e32 - e32
 ; %5 - reg with e32 + o32                                  ...
 ; %3 - reg with e32 - o32                          30 e32[1] - o32[1]
 ; %4 - shift                                       31 e32[0] - o32[0] --> %2
 %macro STORE_32 5
     psrad    %5, %4
     psrad    %3, %4
     packssdw %5, %3
     movq     [%1], %5
     movhps   [%2], %5
 %endmacro

 ; %1 - transform coeffs
 ; %2 - stack offset for e32
 ; %2, %3 offsets for storing e+o/e-o back to coeffsq
 ; %4 - shift
 ; %5 - stack offset of e32
 %macro E32_O32 5
     ADD_ROWS [%1],          [%1 +     16], m0, m1, 1, m8, m9, m10
     ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m8, m9, m10
     ADD_ROWS [%1 + 4 * 16], [%1 + 5 * 16], m4, m5, 0, m8, m9, m10
     ADD_ROWS [%1 + 6 * 16], [%1 + 7 * 16], m6, m7, 0, m8, m9, m10

     paddd m11, m14, [rsp + %5]
     paddd m12, m10, m11 ; o32 + e32
     psubd m11, m10      ; e32 - o32
     STORE_32 %2, %3, m11, %4, m12
 %endmacro

 ; %1 - horizontal offset
 ; %2 - bitdepth
 %macro TR_32x4 3
     TR_16x4 %1, 7, [pd_64], 128, 4, 64, 16, 16, 2, 0

     LOAD_BLOCK m0, m1,      64,  3 * 64,  5 * 64,  7 * 64, %1
     LOAD_BLOCK m2, m3,  9 * 64, 11 * 64, 13 * 64, 15 * 64, %1
     LOAD_BLOCK m4, m5, 17 * 64, 19 * 64, 21 * 64, 23 * 64, %1
     LOAD_BLOCK m6, m7, 25 * 64, 27 * 64, 29 * 64, 31 * 64, %1

     SBUTTERFLY wd, 0, 1, 8
     SBUTTERFLY wd, 2, 3, 8
     SBUTTERFLY wd, 4, 5, 8
     SBUTTERFLY wd, 6, 7, 8

 %if %3 == 1
     %assign shift 7
     mova m14, [pd_64]
 %else
     LOAD_BIAS %2, m14
 %endif

     lea r2, [trans_coeff32 + 15 * 128]
     lea r3, [coeffsq + %1]
     lea r4, [r3 + 16 * 64]
     mov r5d, 15 * 16
 %%loop:
     E32_O32 r2, r3 + r5 * 4, r4, shift, r5
     sub r2, 128
     add r4, 64
     sub r5d, 16
     jge %%loop
 %endmacro

 %macro TRANSPOSE_32x32 0
 cglobal hevc_idct_transpose_32x32, 0, 0, 0
     ; M0  M1 ... M7
     ; M8         M15
     ;
     ; ...
     ;
     ; M56        M63

     TRANSPOSE_BLOCK 0, 0, 64 ; M1
     mov r1d, 7
     mov r2d, 7 * 256
 .loop_transpose:
     SWAP_BLOCKS 0, r2, 64, 0, r1 * 8
     sub r2d, 256
     dec r1d
     jg .loop_transpose

     TRANSPOSE_BLOCK 8, 256, 64 ; M9
     mov r1d, 6
     mov r2d, 512
     mov r3d, 16
 .loop_transpose2:
     SWAP_BLOCKS 8, r2, 64, 256, r3
     add r3d, 8
     add r2d, 256
     dec r1d
     jg .loop_transpose2

     TRANSPOSE_BLOCK 2 * 8, 2 * 256, 64 ; M9
     mov r1d, 5
     mov r2d, 768
     mov r3d, 24
 .loop_transpose3:
     SWAP_BLOCKS 2 * 8, r2, 64, 2 * 256, r3
     add r3d, 8
     add r2d, 256
     dec r1d
     jg .loop_transpose3

     TRANSPOSE_BLOCK 3 * 8, 3 * 256, 64 ; M27
     mov r1d, 4
     mov r2d, 1024
     mov r3d, 32
 .loop_transpose4:
     SWAP_BLOCKS 3 * 8, r2, 64, 3 * 256, r3
     add r3d, 8
     add r2d, 256
     dec r1d
     jg .loop_transpose4

     TRANSPOSE_BLOCK 4 * 8, 4 * 256, 64 ; M36
     mov r1d, 3
     mov r2d, 1280
     mov r3d, 40
 .loop_transpose5:
     SWAP_BLOCKS 4 * 8, r2, 64, 4 * 256, r3
     add r3d, 8
     add r2d, 256
     dec r1d
     jg .loop_transpose5

     TRANSPOSE_BLOCK 5 * 8, 5 * 256, 64 ; M45
     SWAP_BLOCKS 5 * 8, 6 * 256, 64, 5 * 256, 6 * 8
     SWAP_BLOCKS 5 * 8, 7 * 256, 64, 5 * 256, 7 * 8

     TRANSPOSE_BLOCK 6 * 8, 6 * 256, 64 ; M54
     SWAP_BLOCKS 6 * 8, 7 * 256, 64, 6 * 256, 7 * 8

     TRANSPOSE_BLOCK 7 * 8, 7 * 256, 64 ; M63

     ret
 %endmacro

 ; void ff_hevc_idct_32x32_{8,10}_<opt>(int16_t *coeffs, int col_limit)
 ; %1 = bitdepth
 %macro IDCT_32x32 1
 cglobal hevc_idct_32x32_%1, 1, 6, 16, 256, coeffs
     mov r1d, 7
 .loop32:
     TR_32x4 8 * r1, %1, 1
     dec r1d
     jge .loop32

     call hevc_idct_transpose_32x32_ %+ cpuname

     mov r1d, 7
 .loop32_2:
     TR_32x4 8 * r1, %1, 0
     dec r1d
     jge .loop32_2

     TAIL_CALL hevc_idct_transpose_32x32_ %+ cpuname, 1
 %endmacro

 %macro INIT_IDCT_DC 1
 INIT_MMX mmxext
 IDCT_DC_NL  4,      %1
 IDCT_DC     8,  2,  %1

 INIT_XMM sse2
 IDCT_DC_NL  8,      %1
 IDCT_DC    16,  4,  %1
 IDCT_DC    32, 16,  %1

 %if HAVE_AVX2_EXTERNAL
     INIT_YMM avx2
     IDCT_DC    16,  2,  %1
     IDCT_DC    32,  8,  %1
 %endif ;HAVE_AVX2_EXTERNAL
 %endmacro

 %macro INIT_IDCT 2
 INIT_XMM %2
 %if %1 == 8
     TRANSPOSE_8x8
     %if ARCH_X86_64
         TRANSPOSE_16x16
         TRANSPOSE_32x32
     %endif
 %endif
 %if ARCH_X86_64
     IDCT_32x32 %1
     IDCT_16x16 %1
 %endif
 IDCT_8x8 %1
 IDCT_4x4 %1
 %endmacro

 INIT_IDCT_DC 8
 INIT_IDCT_DC 10
 INIT_IDCT_DC 12
 INIT_IDCT 8, sse2
 INIT_IDCT 8, avx
 INIT_IDCT 10, sse2
 INIT_IDCT 10, avx
 ;INIT_IDCT 12, sse2
 ;INIT_IDCT 12, avx
	;*******************************************************************************
	;* SIMD-optimized IDCT functions for HEVC decoding
	;* Copyright (c) 2014 Pierre-Edouard LEPERE
	;* Copyright (c) 2014 James Almer
	;* Copyright (c) 2016 Alexandra Hájková
	;*
	;* This file is part of FFmpeg.
	;*
	;* FFmpeg is free software; you can redistribute it and/or
	;* modify it under the terms of the GNU Lesser General Public
	;* License as published by the Free Software Foundation; either
	;* version 2.1 of the License, or (at your option) any later version.
	;*
	;* FFmpeg is distributed in the hope that it will be useful,
	;* but WITHOUT ANY WARRANTY; without even the implied warranty of
	;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	;* Lesser General Public License for more details.
	;*
	;* You should have received a copy of the GNU Lesser General Public
	;* License along with FFmpeg; if not, write to the Free Software
	;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	;******************************************************************************

	%include "libavutil/x86/x86util.asm"

	SECTION_RODATA

	pd_64: times 4 dd 64
	pd_2048: times 4 dd 2048
	pd_512: times 4 dd 512

	; 4x4 transform coeffs
	cextern pw_64
	pw_64_m64: times 4 dw 64, -64
	pw_83_36: times 4 dw 83, 36
	pw_36_m83: times 4 dw 36, -83

	; 8x8 transform coeffs
	pw_89_75: times 4 dw 89, 75
	pw_50_18: times 4 dw 50, 18

	pw_75_m18: times 4 dw 75, -18
	pw_m89_m50: times 4 dw -89, -50

	pw_50_m89: times 4 dw 50, -89
	pw_18_75: times 4 dw 18, 75

	pw_18_m50: times 4 dw 18, -50
	pw_75_m89: times 4 dw 75, -89

	; 16x16 transformation coeffs
	trans_coeffs16: times 4 dw 90, 87
	times 4 dw 80, 70
	times 4 dw 57, 43
	times 4 dw 25, 9

	times 4 dw 87, 57
	times 4 dw 9, -43
	times 4 dw -80, -90
	times 4 dw -70, -25

	times 4 dw 80, 9
	times 4 dw -70, -87
	times 4 dw -25, 57
	times 4 dw 90, 43

	times 4 dw 70, -43
	times 4 dw -87, 9
	times 4 dw 90, 25
	times 4 dw -80, -57

	times 4 dw 57, -80
	times 4 dw -25, 90
	times 4 dw -9, -87
	times 4 dw 43, 70

	times 4 dw 43, -90
	times 4 dw 57, 25
	times 4 dw -87, 70
	times 4 dw 9, -80

	times 4 dw 25, -70
	times 4 dw 90, -80
	times 4 dw 43, 9
	times 4 dw -57, 87

	times 4 dw 9, -25
	times 4 dw 43, -57
	times 4 dw 70, -80
	times 4 dw 87, -90

	; 32x32 transform coeffs
	trans_coeff32: times 8 dw 90
	times 4 dw 88, 85
	times 4 dw 82, 78
	times 4 dw 73, 67
	times 4 dw 61, 54
	times 4 dw 46, 38
	times 4 dw 31, 22
	times 4 dw 13, 4

	times 4 dw 90, 82
	times 4 dw 67, 46
	times 4 dw 22, -4
	times 4 dw -31, -54
	times 4 dw -73, -85
	times 4 dw -90, -88
	times 4 dw -78, -61
	times 4 dw -38, -13

	times 4 dw 88, 67
	times 4 dw 31, -13
	times 4 dw -54, -82
	times 4 dw -90, -78
	times 4 dw -46, -4
	times 4 dw 38, 73
	times 4 dw 90, 85
	times 4 dw 61, 22

	times 4 dw 85, 46
	times 4 dw -13, -67
	times 4 dw -90, -73
	times 4 dw -22, 38
	times 4 dw 82, 88
	times 4 dw 54, -4
	times 4 dw -61, -90
	times 4 dw -78, -31

	times 4 dw 82, 22
	times 4 dw -54, -90
	times 4 dw -61, 13
	times 4 dw 78, 85
	times 4 dw 31, -46
	times 4 dw -90, -67
	times 4 dw 4, 73
	times 4 dw 88, 38

	times 4 dw 78, -4
	times 4 dw -82, -73
	times 4 dw 13, 85
	times 4 dw 67, -22
	times 4 dw -88, -61
	times 4 dw 31, 90
	times 4 dw 54, -38
	times 4 dw -90, -46

	times 4 dw 73, -31
	times 4 dw -90, -22
	times 4 dw 78, 67
	times 4 dw -38, -90
	times 4 dw -13, 82
	times 4 dw 61, -46
	times 4 dw -88, -4
	times 4 dw 85, 54

	times 4 dw 67, -54
	times 4 dw -78, 38
	times 4 dw 85, -22
	times 4 dw -90, 4
	times 4 dw 90, 13
	times 4 dw -88, -31
	times 4 dw 82, 46
	times 4 dw -73, -61

	times 4 dw 61, -73
	times 4 dw -46, 82
	times 4 dw 31, -88
	times 4 dw -13, 90
	times 4 dw -4, -90
	times 4 dw 22, 85
	times 4 dw -38, -78
	times 4 dw 54, 67

	times 4 dw 54, -85
	times 4 dw -4, 88
	times 4 dw -46, -61
	times 4 dw 82, 13
	times 4 dw -90, 38
	times 4 dw 67, -78
	times 4 dw -22, 90
	times 4 dw -31, -73

	times 4 dw 46, -90
	times 4 dw 38, 54
	times 4 dw -90, 31
	times 4 dw 61, -88
	times 4 dw 22, 67
	times 4 dw -85, 13
	times 4 dw 73, -82
	times 4 dw 4, 78

	times 4 dw 38, -88
	times 4 dw 73, -4
	times 4 dw -67, 90
	times 4 dw -46, -31
	times 4 dw 85, -78
	times 4 dw 13, 61
	times 4 dw -90, 54
	times 4 dw 22, -82

	times 4 dw 31, -78
	times 4 dw 90, -61
	times 4 dw 4, 54
	times 4 dw -88, 82
	times 4 dw -38, -22
	times 4 dw 73, -90
	times 4 dw 67, -13
	times 4 dw -46, 85

	times 4 dw 22, -61
	times 4 dw 85, -90
	times 4 dw 73, -38
	times 4 dw -4, 46
	times 4 dw -78, 90
	times 4 dw -82, 54
	times 4 dw -13, -31
	times 4 dw 67, -88

	times 4 dw 13, -38
	times 4 dw 61, -78
	times 4 dw 88, -90
	times 4 dw 85, -73
	times 4 dw 54, -31
	times 4 dw 4, 22
	times 4 dw -46, 67
	times 4 dw -82, 90

	times 4 dw 4, -13
	times 4 dw 22, -31
	times 4 dw 38, -46
	times 4 dw 54, -61
	times 4 dw 67, -73
	times 4 dw 78, -82
	times 4 dw 85, -88
	times 4 dw 90, -90

	SECTION .text

	; void ff_hevc_idct_HxW_dc_{8,10}_<opt>(int16_t *coeffs)
	; %1 = HxW
	; %2 = number of loops
	; %3 = bitdepth
	%macro IDCT_DC 3
	cglobal hevc_idct_%1x%1_dc_%3, 1, 2, 1, coeff, tmp
	movsx tmpd, word [coeffq]
	add tmpd, (1 << (14 - %3)) + 1
	sar tmpd, (15 - %3)
	movd xm0, tmpd
	SPLATW m0, xm0
	DEFINE_ARGS coeff, cnt
	mov cntd, %2
	.loop:
	mova [coeffq+mmsize*0], m0
	mova [coeffq+mmsize*1], m0
	mova [coeffq+mmsize*2], m0
	mova [coeffq+mmsize*3], m0
	add coeffq, mmsize*8
	mova [coeffq+mmsize*-4], m0
	mova [coeffq+mmsize*-3], m0
	mova [coeffq+mmsize*-2], m0
	mova [coeffq+mmsize*-1], m0
	dec cntd
	jg .loop
	RET
	%endmacro

	; %1 = HxW
	; %2 = bitdepth
	%macro IDCT_DC_NL 2 ; No loop
	cglobal hevc_idct_%1x%1_dc_%2, 1, 2, 1, coeff, tmp
	movsx tmpd, word [coeffq]
	add tmpd, (1 << (14 - %2)) + 1
	sar tmpd, (15 - %2)
	movd m0, tmpd
	SPLATW m0, xm0
	mova [coeffq+mmsize*0], m0
	mova [coeffq+mmsize*1], m0
	mova [coeffq+mmsize*2], m0
	mova [coeffq+mmsize*3], m0
	%if mmsize == 16
	mova [coeffq+mmsize*4], m0
	mova [coeffq+mmsize*5], m0
	mova [coeffq+mmsize*6], m0
	mova [coeffq+mmsize*7], m0
	%endif
	RET
	%endmacro

	; IDCT 4x4, expects input in m0, m1
	; %1 - shift
	; %2 - 1/0 - SCALE and Transpose or not
	; %3 - 1/0 add constant or not
	%macro TR_4x4 3
	; interleaves src0 with src2 to m0
	; and src1 with scr3 to m2
	; src0: 00 01 02 03 m0: 00 20 01 21 02 22 03 23
	; src1: 10 11 12 13 -->
	; src2: 20 21 22 23 m1: 10 30 11 31 12 32 13 33
	; src3: 30 31 32 33

	SBUTTERFLY wd, 0, 1, 2

	pmaddwd m2, m0, [pw_64] ; e0
	pmaddwd m3, m1, [pw_83_36] ; o0
	pmaddwd m0, [pw_64_m64] ; e1
	pmaddwd m1, [pw_36_m83] ; o1

	%if %3 == 1
	%assign %%add 1 << (%1 - 1)
	mova m4, [pd_ %+ %%add]
	paddd m2, m4
	paddd m0, m4
	%endif

	SUMSUB_BADC d, 3, 2, 1, 0, 4

	%if %2 == 1
	psrad m3, %1 ; e0 + o0
	psrad m1, %1 ; e1 + o1
	psrad m2, %1 ; e0 - o0
	psrad m0, %1 ; e1 - o1
	;clip16
	packssdw m3, m1
	packssdw m0, m2
	; Transpose
	SBUTTERFLY wd, 3, 0, 1
	SBUTTERFLY wd, 3, 0, 1
	SWAP 3, 1, 0
	%else
	SWAP 3, 2, 0
	%endif
	%endmacro

	%macro DEFINE_BIAS 1
	%assign shift (20 - %1)
	%assign c_add (1 << (shift - 1))
	%define arr_add pd_ %+ c_add
	%endmacro

	; %1 - bit_depth
	; %2 - register add constant
	; is loaded to
	; shift = 20 - bit_depth
	%macro LOAD_BIAS 2
	DEFINE_BIAS %1
	mova %2, [arr_add]
	%endmacro

	; %1, %2 - registers to load packed 16 bit values to
	; %3, %4, %5, %6 - vertical offsets
	; %7 - horizontal offset
	%macro LOAD_BLOCK 7
	movq %1, [r0 + %3 + %7]
	movhps %1, [r0 + %5 + %7]
	movq %2, [r0 + %4 + %7]
	movhps %2, [r0 + %6 + %7]
	%endmacro

	; void ff_hevc_idct_4x4__{8,10}_<opt>(int16_t *coeffs, int col_limit)
	; %1 = bitdepth
	%macro IDCT_4x4 1
	cglobal hevc_idct_4x4_%1, 1, 1, 5, coeffs
	mova m0, [coeffsq]
	mova m1, [coeffsq + 16]

	TR_4x4 7, 1, 1
	TR_4x4 20 - %1, 1, 1

	mova [coeffsq], m0
	mova [coeffsq + 16], m1
	RET
	%endmacro

	; scale, pack (clip16) and store the residuals 0 e8[0] + o8[0] --> + %1
	; 4 at one time (4 columns) 1 e8[1] + o8[1]
	; from %5: e8/16 + o8/16, with %1 offset ...
	; and %3: e8/16 - o8/16, with %2 offset 6 e8[1] - o8[1]
	; %4 - shift 7 e8[0] - o8[0] --> + %2
	%macro STORE_8 7
	psrad %5, %4
	psrad %3, %4
	packssdw %5, %3
	movq [coeffsq + %1], %5
	movhps [coeffsq + %2], %5
	%endmacro

	; %1 - horizontal offset
	; %2 - shift
	; %3, %4 - transform coeffs
	; %5 - vertical offset for e8 + o8
	; %6 - vertical offset for e8 - o8
	; %7 - register with e8 inside
	; %8 - block_size
	; %9 - register to store e8 +o8
	; %10 - register to store e8 - o8
	%macro E8_O8 10
	pmaddwd m6, m4, %3
	pmaddwd m7, m5, %4

	paddd m6, m7
	paddd m7, m6, %7 ; o8 + e8
	psubd %7, m6 ; e8 - o8
	%if %8 == 8
	STORE_8 %5 + %1, %6 + %1, %7, %2, m7, 0, 0
	%else
	SWAP m7, %9
	SWAP %7, %10
	%endif
	%endmacro

	; 8x4 residuals are processed and stored
	; %1 - horizontal offset
	; %2 - shift
	; %3 - offset of the even row
	; %4 - step: 1 for 8x8, 2 for 16x16, 4 for 32x32
	; %5 - offset of the odd row
	; %6 - block size
	; %7 - 1/0 add a constant in TR_4x4 or not
	; I want to add a constant for 8x8 transform but not for 16x16 and 32x32
	%macro TR_8x4 7
	; load 4 columns of even rows
	LOAD_BLOCK m0, m1, 0, 2 * %4 * %3, %4 * %3, 3 * %4 * %3, %1

	TR_4x4 %2, 0, %7 ; e8: m0, m1, m2, m3, for 4 columns only

	; load 4 columns of odd rows
	LOAD_BLOCK m4, m5, %4 * %5, 3 * %4 * %5, 5 * %4 * %5, 7 * %4 * %5, %1

	; 00 01 02 03
	; 10 11 12 13 m4: 10 30 11 31 12 32 13 33

	; ... -- >
	; m5: 50 70 51 71 52 72 53 73
	; 70 71 72 73
	SBUTTERFLY wd, 4, 5, 6

	E8_O8 %1, %2, [pw_89_75], [pw_50_18], 0, %5 * 7, m0, %6, m8, m15
	E8_O8 %1, %2, [pw_75_m18], [pw_m89_m50], %5, %5 * 6, m1, %6, m9, m14
	E8_O8 %1, %2, [pw_50_m89], [pw_18_75], %5 * 2, %5 * 5, m2, %6, m10, m13
	E8_O8 %1, %2, [pw_18_m50], [pw_75_m89], %5 * 3, %5 * 4, m3, %6, m11, m12
	%endmacro

	%macro STORE_PACKED 7
	movq [r0 + %3 + %7], %1
	movhps [r0 + %4 + %7], %1
	movq [r0 + %5 + %7], %2
	movhps [r0 + %6 + %7], %2
	%endmacro

	; transpose 4x4 block packed
	; in %1 and %2 registers
	; %3 - temporary register
	%macro TRANSPOSE_4x4 3
	SBUTTERFLY wd, %1, %2, %3
	SBUTTERFLY dq, %1, %2, %3
	%endmacro

	; %1 - horizontal offset of the block i
	; %2 - vertical offset of the block i
	; %3 - width in bytes
	; %4 - vertical offset for the block j
	; %5 - horizontal offset for the block j
	%macro SWAP_BLOCKS 5
	; M_j
	LOAD_BLOCK m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
	TRANSPOSE_4x4 4, 5, 6

	; M_i
	LOAD_BLOCK m6, m7, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

	STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1

	; transpose and store M_i
	SWAP m6, m4
	SWAP m7, m5
	TRANSPOSE_4x4 4, 5, 6
	STORE_PACKED m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
	%endmacro

	; %1 - horizontal offset
	; %2 - vertical offset of the block
	; %3 - width in bytes
	%macro TRANSPOSE_BLOCK 3
	LOAD_BLOCK m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
	TRANSPOSE_4x4 4, 5, 6
	STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
	%endmacro

	%macro TRANSPOSE_8x8 0
	cglobal hevc_idct_transpose_8x8, 0, 0, 0
	; M1 M2 ^T = M1^t M3^t
	; M3 M4 M2^t M4^t

	; M1 4x4 block
	TRANSPOSE_BLOCK 0, 0, 16

	; M2 and M3
	SWAP_BLOCKS 0, 64, 16, 0, 8

	; M4
	TRANSPOSE_BLOCK 8, 64, 16

	ret
	%endmacro

	; void ff_hevc_idct_8x8_{8,10}_<opt>(int16_t *coeffs, int col_limit)
	; %1 = bitdepth
	%macro IDCT_8x8 1
	cglobal hevc_idct_8x8_%1, 1, 1, 8, coeffs
	TR_8x4 0, 7, 32, 1, 16, 8, 1
	TR_8x4 8, 7, 32, 1, 16, 8, 1

	call hevc_idct_transpose_8x8_ %+ cpuname

	DEFINE_BIAS %1
	TR_8x4 0, shift, 32, 1, 16, 8, 1
	TR_8x4 8, shift, 32, 1, 16, 8, 1

	TAIL_CALL hevc_idct_transpose_8x8_ %+ cpuname, 1
	%endmacro

	; store intermedite e32 coeffs on stack
	; as 16x4 matrix
	; from m10: e8 + o8, with %6 offset
	; and %3: e8 - o8, with %7 offset
	; %4 - shift, unused here
	%macro STORE_16 7
	mova [rsp + %6], %5
	mova [rsp + %7], %3
	%endmacro

	; %1, %2 - transform constants
	; %3, %4 - regs with interleaved coeffs
	; %5 - 1/0 SWAP or add
	; %6, %7 - registers for intermidiate sums
	; %8 - accumulator register
	%macro ADD_ROWS 8
	pmaddwd %6, %3, %1
	pmaddwd %7, %4, %2
	paddd %6, %7
	%if %5 == 1
	SWAP %6, %8
	%else
	paddd %8, %6
	%endif
	%endmacro

	; %1 - transform coeffs
	; %2, %3 offsets for storing e+o/e-o back to coeffsq
	; %4 - shift
	; %5 - add
	; %6 - block_size
	; %7 - register with e16
	; %8, %9 - stack offsets for storing e+o/e-o
	%macro E16_O16 9
	ADD_ROWS [%1], [%1 + 16], m0, m1, 1, m5, m6, m7
	ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m5, m6, m7

	%if %6 == 8
	paddd %7, %5
	%endif

	paddd m4, m7, %7 ; o16 + e16
	psubd %7, m7 ; e16 - o16
	STORE_%6 %2, %3, %7, %4, m4, %8, %9
	%endmacro

	%macro TR_16x4 10
	; produce 8x4 matrix of e16 coeffs
	; for 4 first rows and store it on stack (128 bytes)
	TR_8x4 %1, 7, %4, %5, %6, %8, 0

	; load 8 even rows
	LOAD_BLOCK m0, m1, %9 * %6, %9 * 3 * %6, %9 * 5 * %6, %9 * 7 * %6, %1
	LOAD_BLOCK m2, m3, %9 * 9 * %6, %9 * 11 * %6, %9 * 13 * %6, %9 * 15 * %6, %1

	SBUTTERFLY wd, 0, 1, 4
	SBUTTERFLY wd, 2, 3, 4

	E16_O16 trans_coeffs16, 0 + %1, 15 * %6 + %1, %2, %3, %7, m8, 0, 15 * 16
	mova m8, %3
	E16_O16 trans_coeffs16 + 64, %6 + %1, 14 * %6 + %1, %2, m8, %7, m9, 16, 14 * 16
	E16_O16 trans_coeffs16 + 2 * 64, 2 * %6 + %1, 13 * %6 + %1, %2, m8, %7, m10, 2 * 16, 13 * 16
	E16_O16 trans_coeffs16 + 3 * 64, 3 * %6 + %1, 12 * %6 + %1, %2, m8, %7, m11, 3 * 16, 12 * 16
	E16_O16 trans_coeffs16 + 4 * 64, 4 * %6 + %1, 11 * %6 + %1, %2, m8, %7, m12, 4 * 16, 11 * 16
	E16_O16 trans_coeffs16 + 5 * 64, 5 * %6 + %1, 10 * %6 + %1, %2, m8, %7, m13, 5 * 16, 10 * 16
	E16_O16 trans_coeffs16 + 6 * 64, 6 * %6 + %1, 9 * %6 + %1, %2, m8, %7, m14, 6 * 16, 9 * 16
	E16_O16 trans_coeffs16 + 7 * 64, 7 * %6 + %1, 8 * %6 + %1, %2, m8, %7, m15, 7 * 16, 8 * 16
	%endmacro

	%macro TRANSPOSE_16x16 0
	cglobal hevc_idct_transpose_16x16, 0, 0, 0
	; M1 M2 M3 M4 ^T m1 m5 m9 m13 M_i^T = m_i
	; M5 M6 M7 M8 --> m2 m6 m10 m14
	; M9 M10 M11 M12 m3 m7 m11 m15
	; M13 M14 M15 M16 m4 m8 m12 m16

	; M1 4x4 block
	TRANSPOSE_BLOCK 0, 0, 32

	; M5, M2
	SWAP_BLOCKS 0, 128, 32, 0, 8
	; M9, M3
	SWAP_BLOCKS 0, 256, 32, 0, 16
	; M13, M4
	SWAP_BLOCKS 0, 384, 32, 0, 24

	;M6
	TRANSPOSE_BLOCK 8, 128, 32

	; M10, M7
	SWAP_BLOCKS 8, 256, 32, 128, 16
	; M14, M8
	SWAP_BLOCKS 8, 384, 32, 128, 24

	;M11
	TRANSPOSE_BLOCK 16, 256, 32

	; M15, M12
	SWAP_BLOCKS 16, 384, 32, 256, 24

	;M16
	TRANSPOSE_BLOCK 24, 384, 32

	ret
	%endmacro

	; void ff_hevc_idct_16x16_{8,10}_<opt>(int16_t *coeffs, int col_limit)
	; %1 = bitdepth
	%macro IDCT_16x16 1
	cglobal hevc_idct_16x16_%1, 1, 2, 16, coeffs
	mov r1d, 3
	.loop16:
	TR_16x4 8 * r1, 7, [pd_64], 64, 2, 32, 8, 16, 1, 0
	dec r1d
	jge .loop16

	call hevc_idct_transpose_16x16_ %+ cpuname

	DEFINE_BIAS %1
	mov r1d, 3
	.loop16_2:
	TR_16x4 8 * r1, shift, [arr_add], 64, 2, 32, 8, 16, 1, 1
	dec r1d
	jge .loop16_2

	TAIL_CALL hevc_idct_transpose_16x16_ %+ cpuname, 1
	%endmacro

	; scale, pack (clip16) and store the residuals 0 e32[0] + o32[0] --> %1
	; 4 at one time (4 columns) 1 e32[1] + o32[1]
	; %1 - address to store e32 + o32
	; %2 - address to store e32 - e32
	; %5 - reg with e32 + o32 ...
	; %3 - reg with e32 - o32 30 e32[1] - o32[1]
	; %4 - shift 31 e32[0] - o32[0] --> %2
	%macro STORE_32 5
	psrad %5, %4
	psrad %3, %4
	packssdw %5, %3
	movq [%1], %5
	movhps [%2], %5
	%endmacro

	; %1 - transform coeffs
	; %2 - stack offset for e32
	; %2, %3 offsets for storing e+o/e-o back to coeffsq
	; %4 - shift
	; %5 - stack offset of e32
	%macro E32_O32 5
	ADD_ROWS [%1], [%1 + 16], m0, m1, 1, m8, m9, m10
	ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m8, m9, m10
	ADD_ROWS [%1 + 4 * 16], [%1 + 5 * 16], m4, m5, 0, m8, m9, m10
	ADD_ROWS [%1 + 6 * 16], [%1 + 7 * 16], m6, m7, 0, m8, m9, m10

	paddd m11, m14, [rsp + %5]
	paddd m12, m10, m11 ; o32 + e32
	psubd m11, m10 ; e32 - o32
	STORE_32 %2, %3, m11, %4, m12
	%endmacro

	; %1 - horizontal offset
	; %2 - bitdepth
	%macro TR_32x4 3
	TR_16x4 %1, 7, [pd_64], 128, 4, 64, 16, 16, 2, 0

	LOAD_BLOCK m0, m1, 64, 3 * 64, 5 * 64, 7 * 64, %1
	LOAD_BLOCK m2, m3, 9 * 64, 11 * 64, 13 * 64, 15 * 64, %1
	LOAD_BLOCK m4, m5, 17 * 64, 19 * 64, 21 * 64, 23 * 64, %1
	LOAD_BLOCK m6, m7, 25 * 64, 27 * 64, 29 * 64, 31 * 64, %1

	SBUTTERFLY wd, 0, 1, 8
	SBUTTERFLY wd, 2, 3, 8
	SBUTTERFLY wd, 4, 5, 8
	SBUTTERFLY wd, 6, 7, 8

	%if %3 == 1
	%assign shift 7
	mova m14, [pd_64]
	%else
	LOAD_BIAS %2, m14
	%endif

	lea r2, [trans_coeff32 + 15 * 128]
	lea r3, [coeffsq + %1]
	lea r4, [r3 + 16 * 64]
	mov r5d, 15 * 16
	%%loop:
	E32_O32 r2, r3 + r5 * 4, r4, shift, r5
	sub r2, 128
	add r4, 64
	sub r5d, 16
	jge %%loop
	%endmacro

	%macro TRANSPOSE_32x32 0
	cglobal hevc_idct_transpose_32x32, 0, 0, 0
	; M0 M1 ... M7
	; M8 M15
	;
	; ...
	;
	; M56 M63

	TRANSPOSE_BLOCK 0, 0, 64 ; M1
	mov r1d, 7
	mov r2d, 7 * 256
	.loop_transpose:
	SWAP_BLOCKS 0, r2, 64, 0, r1 * 8
	sub r2d, 256
	dec r1d
	jg .loop_transpose

	TRANSPOSE_BLOCK 8, 256, 64 ; M9
	mov r1d, 6
	mov r2d, 512
	mov r3d, 16
	.loop_transpose2:
	SWAP_BLOCKS 8, r2, 64, 256, r3
	add r3d, 8
	add r2d, 256
	dec r1d
	jg .loop_transpose2

	TRANSPOSE_BLOCK 2 * 8, 2 * 256, 64 ; M9
	mov r1d, 5
	mov r2d, 768
	mov r3d, 24
	.loop_transpose3:
	SWAP_BLOCKS 2 * 8, r2, 64, 2 * 256, r3
	add r3d, 8
	add r2d, 256
	dec r1d
	jg .loop_transpose3

	TRANSPOSE_BLOCK 3 * 8, 3 * 256, 64 ; M27
	mov r1d, 4
	mov r2d, 1024
	mov r3d, 32
	.loop_transpose4:
	SWAP_BLOCKS 3 * 8, r2, 64, 3 * 256, r3
	add r3d, 8
	add r2d, 256
	dec r1d
	jg .loop_transpose4

	TRANSPOSE_BLOCK 4 * 8, 4 * 256, 64 ; M36
	mov r1d, 3
	mov r2d, 1280
	mov r3d, 40
	.loop_transpose5:
	SWAP_BLOCKS 4 * 8, r2, 64, 4 * 256, r3
	add r3d, 8
	add r2d, 256
	dec r1d
	jg .loop_transpose5

	TRANSPOSE_BLOCK 5 * 8, 5 * 256, 64 ; M45
	SWAP_BLOCKS 5 * 8, 6 * 256, 64, 5 * 256, 6 * 8
	SWAP_BLOCKS 5 * 8, 7 * 256, 64, 5 * 256, 7 * 8

	TRANSPOSE_BLOCK 6 * 8, 6 * 256, 64 ; M54
	SWAP_BLOCKS 6 * 8, 7 * 256, 64, 6 * 256, 7 * 8

	TRANSPOSE_BLOCK 7 * 8, 7 * 256, 64 ; M63

	ret
	%endmacro

	; void ff_hevc_idct_32x32_{8,10}_<opt>(int16_t *coeffs, int col_limit)
	; %1 = bitdepth
	%macro IDCT_32x32 1
	cglobal hevc_idct_32x32_%1, 1, 6, 16, 256, coeffs
	mov r1d, 7
	.loop32:
	TR_32x4 8 * r1, %1, 1
	dec r1d
	jge .loop32

	call hevc_idct_transpose_32x32_ %+ cpuname

	mov r1d, 7
	.loop32_2:
	TR_32x4 8 * r1, %1, 0
	dec r1d
	jge .loop32_2

	TAIL_CALL hevc_idct_transpose_32x32_ %+ cpuname, 1
	%endmacro

	%macro INIT_IDCT_DC 1
	INIT_MMX mmxext
	IDCT_DC_NL 4, %1
	IDCT_DC 8, 2, %1

	INIT_XMM sse2
	IDCT_DC_NL 8, %1
	IDCT_DC 16, 4, %1
	IDCT_DC 32, 16, %1

	%if HAVE_AVX2_EXTERNAL
	INIT_YMM avx2
	IDCT_DC 16, 2, %1
	IDCT_DC 32, 8, %1
	%endif ;HAVE_AVX2_EXTERNAL
	%endmacro

	%macro INIT_IDCT 2
	INIT_XMM %2
	%if %1 == 8
	TRANSPOSE_8x8
	%if ARCH_X86_64
	TRANSPOSE_16x16
	TRANSPOSE_32x32
	%endif
	%endif
	%if ARCH_X86_64
	IDCT_32x32 %1
	IDCT_16x16 %1
	%endif
	IDCT_8x8 %1
	IDCT_4x4 %1
	%endmacro

	INIT_IDCT_DC 8
	INIT_IDCT_DC 10
	INIT_IDCT_DC 12
	INIT_IDCT 8, sse2
	INIT_IDCT 8, avx
	INIT_IDCT 10, sse2
	INIT_IDCT 10, avx
	;INIT_IDCT 12, sse2
	;INIT_IDCT 12, avx