| /* |
| * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com> |
| * |
| * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights |
| * reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * 1. Redistributions of source code must retain the above copyright notice, |
| * this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * 3. All advertising materials mentioning features or use of this software |
| * must display the following acknowledgement: |
| * This product includes software developed by Cavium Networks |
| * 4. Cavium Networks' name may not be used to endorse or promote products |
| * derived from this software without specific prior written permission. |
| * |
| * This Software, including technical data, may be subject to U.S. export |
| * control laws, including the U.S. Export Administration Act and its |
| * associated regulations, and may be subject to export or import regulations |
| * in other countries. You warrant that You will comply strictly in all |
| * respects with all such regulations and acknowledge that you have the |
| * responsibility to obtain licenses to export, re-export or import the |
| * Software. |
| * |
| * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND |
| * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, |
| * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE |
| * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR |
| * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM |
| * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, |
| * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF |
| * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR |
| * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR |
| * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. |
| */ |
| /****************************************************************************/ |
| |
| #include <linux/scatterlist.h> |
| #include <asm/octeon/octeon.h> |
| #include "octeon-asm.h" |
| |
| /****************************************************************************/ |
| |
| extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *); |
| extern void octeon_crypto_disable(struct octeon_cop2_state *, unsigned long); |
| |
| #define SG_INIT(s, p, i, l) \ |
| { \ |
| (i) = 0; \ |
| (l) = (s)[0].length; \ |
| (p) = (typeof(p)) sg_virt((s)); \ |
| CVMX_PREFETCH0((p)); \ |
| } |
| |
| #define SG_CONSUME(s, p, i, l) \ |
| { \ |
| (p)++; \ |
| (l) -= sizeof(*(p)); \ |
| if ((l) < 0) { \ |
| dprintk("%s, %d: l = %d\n", __FILE__, __LINE__, l); \ |
| } else if ((l) == 0) { \ |
| (i)++; \ |
| (l) = (s)[0].length; \ |
| (p) = (typeof(p)) sg_virt(s); \ |
| CVMX_PREFETCH0((p)); \ |
| } \ |
| } |
| |
| #define ESP_HEADER_LENGTH 8 |
| #define DES_CBC_IV_LENGTH 8 |
| #define AES_CBC_IV_LENGTH 16 |
| #define ESP_HMAC_LEN 12 |
| |
| #define ESP_HEADER_LENGTH 8 |
| #define DES_CBC_IV_LENGTH 8 |
| |
| /****************************************************************************/ |
| |
| #define CVM_LOAD_SHA_UNIT(dat, next) { \ |
| if (next == 0) { \ |
| next = 1; \ |
| CVMX_MT_HSH_DAT (dat, 0); \ |
| } else if (next == 1) { \ |
| next = 2; \ |
| CVMX_MT_HSH_DAT (dat, 1); \ |
| } else if (next == 2) { \ |
| next = 3; \ |
| CVMX_MT_HSH_DAT (dat, 2); \ |
| } else if (next == 3) { \ |
| next = 4; \ |
| CVMX_MT_HSH_DAT (dat, 3); \ |
| } else if (next == 4) { \ |
| next = 5; \ |
| CVMX_MT_HSH_DAT (dat, 4); \ |
| } else if (next == 5) { \ |
| next = 6; \ |
| CVMX_MT_HSH_DAT (dat, 5); \ |
| } else if (next == 6) { \ |
| next = 7; \ |
| CVMX_MT_HSH_DAT (dat, 6); \ |
| } else { \ |
| CVMX_MT_HSH_STARTSHA (dat); \ |
| next = 0; \ |
| } \ |
| } |
| |
| #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ |
| if (next == 0) { \ |
| CVMX_MT_HSH_DAT (dat1, 0); \ |
| CVMX_MT_HSH_DAT (dat2, 1); \ |
| next = 2; \ |
| } else if (next == 1) { \ |
| CVMX_MT_HSH_DAT (dat1, 1); \ |
| CVMX_MT_HSH_DAT (dat2, 2); \ |
| next = 3; \ |
| } else if (next == 2) { \ |
| CVMX_MT_HSH_DAT (dat1, 2); \ |
| CVMX_MT_HSH_DAT (dat2, 3); \ |
| next = 4; \ |
| } else if (next == 3) { \ |
| CVMX_MT_HSH_DAT (dat1, 3); \ |
| CVMX_MT_HSH_DAT (dat2, 4); \ |
| next = 5; \ |
| } else if (next == 4) { \ |
| CVMX_MT_HSH_DAT (dat1, 4); \ |
| CVMX_MT_HSH_DAT (dat2, 5); \ |
| next = 6; \ |
| } else if (next == 5) { \ |
| CVMX_MT_HSH_DAT (dat1, 5); \ |
| CVMX_MT_HSH_DAT (dat2, 6); \ |
| next = 7; \ |
| } else if (next == 6) { \ |
| CVMX_MT_HSH_DAT (dat1, 6); \ |
| CVMX_MT_HSH_STARTSHA (dat2); \ |
| next = 0; \ |
| } else { \ |
| CVMX_MT_HSH_STARTSHA (dat1); \ |
| CVMX_MT_HSH_DAT (dat2, 0); \ |
| next = 1; \ |
| } \ |
| } |
| |
| /****************************************************************************/ |
| |
| #define CVM_LOAD_MD5_UNIT(dat, next) { \ |
| if (next == 0) { \ |
| next = 1; \ |
| CVMX_MT_HSH_DAT (dat, 0); \ |
| } else if (next == 1) { \ |
| next = 2; \ |
| CVMX_MT_HSH_DAT (dat, 1); \ |
| } else if (next == 2) { \ |
| next = 3; \ |
| CVMX_MT_HSH_DAT (dat, 2); \ |
| } else if (next == 3) { \ |
| next = 4; \ |
| CVMX_MT_HSH_DAT (dat, 3); \ |
| } else if (next == 4) { \ |
| next = 5; \ |
| CVMX_MT_HSH_DAT (dat, 4); \ |
| } else if (next == 5) { \ |
| next = 6; \ |
| CVMX_MT_HSH_DAT (dat, 5); \ |
| } else if (next == 6) { \ |
| next = 7; \ |
| CVMX_MT_HSH_DAT (dat, 6); \ |
| } else { \ |
| CVMX_MT_HSH_STARTMD5 (dat); \ |
| next = 0; \ |
| } \ |
| } |
| |
| #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ |
| if (next == 0) { \ |
| CVMX_MT_HSH_DAT (dat1, 0); \ |
| CVMX_MT_HSH_DAT (dat2, 1); \ |
| next = 2; \ |
| } else if (next == 1) { \ |
| CVMX_MT_HSH_DAT (dat1, 1); \ |
| CVMX_MT_HSH_DAT (dat2, 2); \ |
| next = 3; \ |
| } else if (next == 2) { \ |
| CVMX_MT_HSH_DAT (dat1, 2); \ |
| CVMX_MT_HSH_DAT (dat2, 3); \ |
| next = 4; \ |
| } else if (next == 3) { \ |
| CVMX_MT_HSH_DAT (dat1, 3); \ |
| CVMX_MT_HSH_DAT (dat2, 4); \ |
| next = 5; \ |
| } else if (next == 4) { \ |
| CVMX_MT_HSH_DAT (dat1, 4); \ |
| CVMX_MT_HSH_DAT (dat2, 5); \ |
| next = 6; \ |
| } else if (next == 5) { \ |
| CVMX_MT_HSH_DAT (dat1, 5); \ |
| CVMX_MT_HSH_DAT (dat2, 6); \ |
| next = 7; \ |
| } else if (next == 6) { \ |
| CVMX_MT_HSH_DAT (dat1, 6); \ |
| CVMX_MT_HSH_STARTMD5 (dat2); \ |
| next = 0; \ |
| } else { \ |
| CVMX_MT_HSH_STARTMD5 (dat1); \ |
| CVMX_MT_HSH_DAT (dat2, 0); \ |
| next = 1; \ |
| } \ |
| } |
| |
| /****************************************************************************/ |
| |
| static inline uint64_t |
| swap64(uint64_t a) |
| { |
| return ((a >> 56) | |
| (((a >> 48) & 0xfful) << 8) | |
| (((a >> 40) & 0xfful) << 16) | |
| (((a >> 32) & 0xfful) << 24) | |
| (((a >> 24) & 0xfful) << 32) | |
| (((a >> 16) & 0xfful) << 40) | |
| (((a >> 8) & 0xfful) << 48) | (((a >> 0) & 0xfful) << 56)); |
| } |
| |
| /****************************************************************************/ |
| |
| void |
| octo_calc_hash(__u8 auth, unsigned char *key, uint64_t *inner, uint64_t *outer) |
| { |
| uint8_t hash_key[64]; |
| uint64_t *key1; |
| register uint64_t xor1 = 0x3636363636363636ULL; |
| register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| memset(hash_key, 0, sizeof(hash_key)); |
| memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); |
| key1 = (uint64_t *) hash_key; |
| flags = octeon_crypto_enable(&state); |
| if (auth) { |
| CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); |
| CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); |
| CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); |
| } else { |
| CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); |
| CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); |
| } |
| |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); |
| key1++; |
| if (auth) |
| CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); |
| else |
| CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); |
| |
| CVMX_MF_HSH_IV(inner[0], 0); |
| CVMX_MF_HSH_IV(inner[1], 1); |
| if (auth) { |
| inner[2] = 0; |
| CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); |
| } |
| |
| memset(hash_key, 0, sizeof(hash_key)); |
| memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); |
| key1 = (uint64_t *) hash_key; |
| if (auth) { |
| CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); |
| CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); |
| CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); |
| } else { |
| CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); |
| CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); |
| } |
| |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); |
| key1++; |
| CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); |
| key1++; |
| if (auth) |
| CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); |
| else |
| CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); |
| |
| CVMX_MF_HSH_IV(outer[0], 0); |
| CVMX_MF_HSH_IV(outer[1], 1); |
| if (auth) { |
| outer[2] = 0; |
| CVMX_MF_HSH_IV(outer[2], 2); |
| } |
| octeon_crypto_disable(&state, flags); |
| return; |
| } |
| |
| /****************************************************************************/ |
| /* DES functions */ |
| |
| int |
| octo_des_cbc_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| uint64_t *data; |
| int data_i, data_l; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| while (crypt_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_off -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| CVMX_MT_3DES_ENC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_len -= 8; |
| } |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| |
| int |
| octo_des_cbc_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| uint64_t *data; |
| int data_i, data_l; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| while (crypt_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_off -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| CVMX_MT_3DES_DEC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_len -= 8; |
| } |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* AES functions */ |
| |
| int |
| octo_aes_cbc_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| uint64_t *data, *pdata; |
| int data_i, data_l; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| while (crypt_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_off -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| pdata = data; |
| CVMX_MT_AES_ENC_CBC0(*data); |
| SG_CONSUME(sg, data, data_i, data_l); |
| CVMX_MT_AES_ENC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_len -= 16; |
| } |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| |
| int |
| octo_aes_cbc_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| uint64_t *data, *pdata; |
| int data_i, data_l; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x7) || (crypt_off + crypt_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| while (crypt_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_off -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| pdata = data; |
| CVMX_MT_AES_DEC_CBC0(*data); |
| SG_CONSUME(sg, data, data_i, data_l); |
| CVMX_MT_AES_DEC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| SG_CONSUME(sg, data, data_i, data_l); |
| crypt_len -= 16; |
| } |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* MD5 */ |
| |
| int |
| octo_null_md5_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| uint64_t *data; |
| uint64_t tmp1, tmp2; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || |
| (auth_off & 0x7) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* Load MD5 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| |
| while (auth_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| auth_off -= 8; |
| } |
| |
| while (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*data, next); |
| auth_len -= 8; |
| SG_CONSUME(sg, data, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVMX_ES64(tmp1, ((alen + 64) << 3)); |
| CVM_LOAD_MD5_UNIT(tmp1, next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_ES64(tmp1, ((64 + 16) << 3)); |
| CVMX_MT_HSH_STARTMD5(tmp1); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| icv_off -= 8; |
| } |
| CVMX_MF_HSH_IV(*data, 0); |
| SG_CONSUME(sg, data, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *(uint32_t *)data = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* SHA1 */ |
| |
| int |
| octo_null_sha1_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| uint64_t *data; |
| uint64_t tmp1, tmp2, tmp3; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || |
| (auth_off & 0x7) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data, data_i, data_l); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* Load SHA1 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hminner[2], 2); |
| |
| while (auth_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| auth_off -= 8; |
| } |
| |
| while (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*data, next); |
| auth_len -= 8; |
| SG_CONSUME(sg, data, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| tmp3 = 0; |
| CVMX_MF_HSH_IV(tmp3, 2); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| tmp3 |= 0x0000000080000000; |
| CVMX_MT_HSH_DAT(tmp3, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data, data_i, data_l); |
| icv_off -= 8; |
| } |
| CVMX_MF_HSH_IV(*data, 0); |
| SG_CONSUME(sg, data, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *(uint32_t *)data = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* DES MD5 */ |
| |
| int |
| octo_des_cbc_md5_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata; |
| uint64_t *data = &mydata.data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| /* Load MD5 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| while (crypt_len > 0 || auth_len > 0) { |
| uint32_t *first = data32; |
| mydata.data32[0] = *first; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata.data32[1] = *data32; |
| if (crypt_off <= 0) { |
| if (crypt_len > 0) { |
| CVMX_MT_3DES_ENC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| crypt_len -= 8; |
| } |
| } else |
| crypt_off -= 8; |
| if (auth_off <= 0) { |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| } else |
| auth_off -= 8; |
| *first = mydata.data32[0]; |
| *data32 = mydata.data32[1]; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVMX_ES64(tmp1, ((alen + 64) << 3)); |
| CVM_LOAD_MD5_UNIT(tmp1, next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_ES64(tmp1, ((64 + 16) << 3)); |
| CVMX_MT_HSH_STARTMD5(tmp1); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| int |
| octo_des_cbc_md5_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata; |
| uint64_t *data = &mydata.data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| /* Load MD5 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| while (crypt_len > 0 || auth_len > 0) { |
| uint32_t *first = data32; |
| mydata.data32[0] = *first; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata.data32[1] = *data32; |
| if (auth_off <= 0) { |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| } else |
| auth_off -= 8; |
| if (crypt_off <= 0) { |
| if (crypt_len > 0) { |
| CVMX_MT_3DES_DEC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| crypt_len -= 8; |
| } |
| } else |
| crypt_off -= 8; |
| *first = mydata.data32[0]; |
| *data32 = mydata.data32[1]; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVMX_ES64(tmp1, ((alen + 64) << 3)); |
| CVM_LOAD_MD5_UNIT(tmp1, next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_ES64(tmp1, ((64 + 16) << 3)); |
| CVMX_MT_HSH_STARTMD5(tmp1); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* DES SHA */ |
| |
| int |
| octo_des_cbc_sha1_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata; |
| uint64_t *data = &mydata.data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2, tmp3; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| /* Load SHA1 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hminner[2], 2); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| while (crypt_len > 0 || auth_len > 0) { |
| uint32_t *first = data32; |
| mydata.data32[0] = *first; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata.data32[1] = *data32; |
| if (crypt_off <= 0) { |
| if (crypt_len > 0) { |
| CVMX_MT_3DES_ENC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| crypt_len -= 8; |
| } |
| } else |
| crypt_off -= 8; |
| if (auth_off <= 0) { |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| } else |
| auth_off -= 8; |
| *first = mydata.data32[0]; |
| *data32 = mydata.data32[1]; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_SHA_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| tmp3 = 0; |
| CVMX_MF_HSH_IV(tmp3, 2); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| tmp3 |= 0x0000000080000000; |
| CVMX_MT_HSH_DAT(tmp3, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| int |
| octo_des_cbc_sha1_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata; |
| uint64_t *data = &mydata.data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2, tmp3; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load 3DES Key */ |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| if (od->octo_encklen == 24) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| } else if (od->octo_encklen == 8) { |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); |
| CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| |
| CVMX_MT_3DES_IV(* (uint64_t *) ivp); |
| |
| /* Load SHA1 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hminner[2], 2); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| while (crypt_len > 0 || auth_len > 0) { |
| uint32_t *first = data32; |
| mydata.data32[0] = *first; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata.data32[1] = *data32; |
| if (auth_off <= 0) { |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| } else |
| auth_off -= 8; |
| if (crypt_off <= 0) { |
| if (crypt_len > 0) { |
| CVMX_MT_3DES_DEC_CBC(*data); |
| CVMX_MF_3DES_RESULT(*data); |
| crypt_len -= 8; |
| } |
| } else |
| crypt_off -= 8; |
| *first = mydata.data32[0]; |
| *data32 = mydata.data32[1]; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_SHA_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| tmp3 = 0; |
| CVMX_MF_HSH_IV(tmp3, 2); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| tmp3 |= 0x0000000080000000; |
| CVMX_MT_HSH_DAT(tmp3, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* AES MD5 */ |
| |
| int |
| octo_aes_cbc_md5_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata[2]; |
| uint64_t *pdata = &mydata[0].data64[0]; |
| uint64_t *data = &mydata[1].data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| /* Load MD5 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| /* align auth and crypt */ |
| while (crypt_off > 0 && auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| crypt_off -= 8; |
| auth_len -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| uint32_t *pdata32[3]; |
| |
| pdata32[0] = data32; |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| |
| pdata32[1] = data32; |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| |
| pdata32[2] = data32; |
| mydata[1].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| |
| mydata[1].data32[1] = *data32; |
| |
| CVMX_MT_AES_ENC_CBC0(*pdata); |
| CVMX_MT_AES_ENC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| crypt_len -= 16; |
| |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| |
| *pdata32[0] = mydata[0].data32[0]; |
| *pdata32[1] = mydata[0].data32[1]; |
| *pdata32[2] = mydata[1].data32[0]; |
| *data32 = mydata[1].data32[1]; |
| |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish any left over hashing */ |
| while (auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVMX_ES64(tmp1, ((alen + 64) << 3)); |
| CVM_LOAD_MD5_UNIT(tmp1, next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_ES64(tmp1, ((64 + 16) << 3)); |
| CVMX_MT_HSH_STARTMD5(tmp1); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| int |
| octo_aes_cbc_md5_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata[2]; |
| uint64_t *pdata = &mydata[0].data64[0]; |
| uint64_t *data = &mydata[1].data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s()\n", __FUNCTION__); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| /* Load MD5 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| /* align auth and crypt */ |
| while (crypt_off > 0 && auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| crypt_off -= 8; |
| auth_len -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| uint32_t *pdata32[3]; |
| |
| pdata32[0] = data32; |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[1] = data32; |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[2] = data32; |
| mydata[1].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[1].data32[1] = *data32; |
| |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| |
| if (auth_len > 0) { |
| CVM_LOAD_MD5_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| |
| CVMX_MT_AES_DEC_CBC0(*pdata); |
| CVMX_MT_AES_DEC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| crypt_len -= 16; |
| |
| *pdata32[0] = mydata[0].data32[0]; |
| *pdata32[1] = mydata[0].data32[1]; |
| *pdata32[2] = mydata[1].data32[0]; |
| *data32 = mydata[1].data32[1]; |
| |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish left over hash if any */ |
| while (auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_MD5_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVMX_ES64(tmp1, ((alen + 64) << 3)); |
| CVM_LOAD_MD5_UNIT(tmp1, next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_ES64(tmp1, ((64 + 16) << 3)); |
| CVMX_MT_HSH_STARTMD5(tmp1); |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |
| /* AES SHA1 */ |
| |
| int |
| octo_aes_cbc_sha1_encrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata[2]; |
| uint64_t *pdata = &mydata[0].data64[0]; |
| uint64_t *data = &mydata[1].data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2, tmp3; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s(a_off=%d a_len=%d c_off=%d c_len=%d icv_off=%d)\n", |
| __FUNCTION__, auth_off, auth_len, crypt_off, crypt_len, icv_off); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| /* Load SHA IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hminner[2], 2); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| /* align auth and crypt */ |
| while (crypt_off > 0 && auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| crypt_off -= 8; |
| auth_len -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| uint32_t *pdata32[3]; |
| |
| pdata32[0] = data32; |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[1] = data32; |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[2] = data32; |
| mydata[1].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[1].data32[1] = *data32; |
| |
| CVMX_MT_AES_ENC_CBC0(*pdata); |
| CVMX_MT_AES_ENC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| crypt_len -= 16; |
| |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| |
| *pdata32[0] = mydata[0].data32[0]; |
| *pdata32[1] = mydata[0].data32[1]; |
| *pdata32[2] = mydata[1].data32[0]; |
| *data32 = mydata[1].data32[1]; |
| |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish and hashing */ |
| while (auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_SHA_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| tmp3 = 0; |
| CVMX_MF_HSH_IV(tmp3, 2); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| tmp3 |= 0x0000000080000000; |
| CVMX_MT_HSH_DAT(tmp3, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| int |
| octo_aes_cbc_sha1_decrypt( |
| struct octo_sess *od, |
| struct scatterlist *sg, int sg_len, |
| int auth_off, int auth_len, |
| int crypt_off, int crypt_len, |
| int icv_off, uint8_t *ivp) |
| { |
| register int next = 0; |
| union { |
| uint32_t data32[2]; |
| uint64_t data64[1]; |
| } mydata[2]; |
| uint64_t *pdata = &mydata[0].data64[0]; |
| uint64_t *data = &mydata[1].data64[0]; |
| uint32_t *data32; |
| uint64_t tmp1, tmp2, tmp3; |
| int data_i, data_l, alen = auth_len; |
| struct octeon_cop2_state state; |
| unsigned long flags; |
| |
| dprintk("%s(a_off=%d a_len=%d c_off=%d c_len=%d icv_off=%d)\n", |
| __FUNCTION__, auth_off, auth_len, crypt_off, crypt_len, icv_off); |
| |
| if (unlikely(od == NULL || sg==NULL || sg_len==0 || ivp==NULL || |
| (crypt_off & 0x3) || (crypt_off + crypt_len > sg_len) || |
| (crypt_len & 0x7) || |
| (auth_len & 0x7) || |
| (auth_off & 0x3) || (auth_off + auth_len > sg_len))) { |
| dprintk("%s: Bad parameters od=%p sg=%p sg_len=%d " |
| "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " |
| "icv_off=%d ivp=%p\n", __FUNCTION__, od, sg, sg_len, |
| auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); |
| return -EINVAL; |
| } |
| |
| SG_INIT(sg, data32, data_i, data_l); |
| |
| CVMX_PREFETCH0(ivp); |
| CVMX_PREFETCH0(od->octo_enckey); |
| |
| flags = octeon_crypto_enable(&state); |
| |
| /* load AES Key */ |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); |
| |
| if (od->octo_encklen == 16) { |
| CVMX_MT_AES_KEY(0x0, 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 24) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(0x0, 3); |
| } else if (od->octo_encklen == 32) { |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); |
| CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); |
| } else { |
| octeon_crypto_disable(&state, flags); |
| dprintk("%s: Bad key length %d\n", __FUNCTION__, od->octo_encklen); |
| return -EINVAL; |
| } |
| CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); |
| |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); |
| CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); |
| |
| /* Load SHA1 IV */ |
| CVMX_MT_HSH_IV(od->octo_hminner[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hminner[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hminner[2], 2); |
| |
| while (crypt_off > 0 && auth_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| crypt_off -= 4; |
| auth_off -= 4; |
| } |
| |
| /* align auth and crypt */ |
| while (crypt_off > 0 && auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| crypt_off -= 8; |
| auth_len -= 8; |
| } |
| |
| while (crypt_len > 0) { |
| uint32_t *pdata32[3]; |
| |
| pdata32[0] = data32; |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[1] = data32; |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| pdata32[2] = data32; |
| mydata[1].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[1].data32[1] = *data32; |
| |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| if (auth_len > 0) { |
| CVM_LOAD_SHA_UNIT(*data, next); |
| auth_len -= 8; |
| } |
| |
| CVMX_MT_AES_DEC_CBC0(*pdata); |
| CVMX_MT_AES_DEC_CBC1(*data); |
| CVMX_MF_AES_RESULT(*pdata, 0); |
| CVMX_MF_AES_RESULT(*data, 1); |
| crypt_len -= 16; |
| |
| *pdata32[0] = mydata[0].data32[0]; |
| *pdata32[1] = mydata[0].data32[1]; |
| *pdata32[2] = mydata[1].data32[0]; |
| *data32 = mydata[1].data32[1]; |
| |
| SG_CONSUME(sg, data32, data_i, data_l); |
| } |
| |
| /* finish and leftover hashing */ |
| while (auth_len > 0) { |
| mydata[0].data32[0] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| mydata[0].data32[1] = *data32; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVM_LOAD_SHA_UNIT(*pdata, next); |
| auth_len -= 8; |
| } |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_SHA_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* Finish Inner hash */ |
| while (next != 7) { |
| CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); |
| } |
| CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); |
| |
| /* Get the inner hash of HMAC */ |
| CVMX_MF_HSH_IV(tmp1, 0); |
| CVMX_MF_HSH_IV(tmp2, 1); |
| tmp3 = 0; |
| CVMX_MF_HSH_IV(tmp3, 2); |
| |
| /* Initialize hash unit */ |
| CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); |
| CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); |
| CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); |
| |
| CVMX_MT_HSH_DAT(tmp1, 0); |
| CVMX_MT_HSH_DAT(tmp2, 1); |
| tmp3 |= 0x0000000080000000; |
| CVMX_MT_HSH_DAT(tmp3, 2); |
| CVMX_MT_HSH_DATZ(3); |
| CVMX_MT_HSH_DATZ(4); |
| CVMX_MT_HSH_DATZ(5); |
| CVMX_MT_HSH_DATZ(6); |
| CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); |
| |
| /* finish the hash */ |
| CVMX_PREFETCH0(od->octo_hmouter); |
| #if 0 |
| if (unlikely(inplen)) { |
| uint64_t tmp = 0; |
| uint8_t *p = (uint8_t *) & tmp; |
| p[inplen] = 0x80; |
| do { |
| inplen--; |
| p[inplen] = ((uint8_t *) data)[inplen]; |
| } while (inplen); |
| CVM_LOAD_MD5_UNIT(tmp, next); |
| } else { |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| } |
| #else |
| CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); |
| #endif |
| |
| /* save the HMAC */ |
| SG_INIT(sg, data32, data_i, data_l); |
| while (icv_off > 0) { |
| SG_CONSUME(sg, data32, data_i, data_l); |
| icv_off -= 4; |
| } |
| CVMX_MF_HSH_IV(tmp1, 0); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| SG_CONSUME(sg, data32, data_i, data_l); |
| *data32 = (uint32_t) tmp1; |
| SG_CONSUME(sg, data32, data_i, data_l); |
| CVMX_MF_HSH_IV(tmp1, 1); |
| *data32 = (uint32_t) (tmp1 >> 32); |
| |
| octeon_crypto_disable(&state, flags); |
| return 0; |
| } |
| |
| /****************************************************************************/ |