/*
 * Copyright (c) 1997-1999 Massachusetts Institute of Technology
 * Copyright (c) 2000-2001 Stefan Kral
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

/* This file was automatically generated --- DO NOT EDIT */
/* Generated on Tue Mar 25 06:42:04 EST 2003 */


/* fnset2/cheap-mode: VECTGRADE_FULLH0 succeeded. (11321 steps) */
/* fnset2/cheap-mode: VECTGRADE_FULL succeeded. (494 steps) */
/* fnset1/cheap-mode: VECTGRADE_FULL succeeded. (134 steps) */
.section .rodata
	.balign 64
	PK923879532PK382683432: .double +0.9238795325112867561281832, +0.3826834323650897717284600
	PK382683432PK923879532: .double +0.3826834323650897717284600, +0.9238795325112867561281832
	chs_lo: .double -1.0, +1.0
	PK707106781PK707106781: .double +0.7071067811865475244008444, +0.7071067811865475244008444
	chs_hi: .double +1.0, -1.0
	chs_lohi: .double -1.0, -1.0
	NK707106781PK707106781: .double -0.7071067811865475244008444, +0.7071067811865475244008444
	PK707106781NK707106781: .double +0.7071067811865475244008444, -0.7071067811865475244008444
	NK923879532PK382683432: .double -0.9238795325112867561281832, +0.3826834323650897717284600
	PK923879532NK382683432: .double +0.9238795325112867561281832, -0.3826834323650897717284600
	PK923879532PK923879532: .double +0.9238795325112867561281832, +0.9238795325112867561281832
	NK382683432PK382683432: .double -0.3826834323650897717284600, +0.3826834323650897717284600
	PK382683432PK382683432: .double +0.3826834323650897717284600, +0.3826834323650897717284600
	NK923879532PK923879532: .double -0.9238795325112867561281832, +0.9238795325112867561281832
	PK2_000000000PK2_000000000: .double +2.0000000000000000000000000, +2.0000000000000000000000000
	PK2_000000000NK2_000000000: .double +2.0000000000000000000000000, -2.0000000000000000000000000
	PK1_414213562PK1_414213562: .double +1.4142135623730950488016887, +1.4142135623730950488016887
	NK1_414213562PK1_414213562: .double -1.4142135623730950488016887, +1.4142135623730950488016887
	NK415734806PK415734806: .double -0.4157348061512726185393942, +0.4157348061512726185393942
	PK277785116PK277785116: .double +0.2777851165098011123714154, +0.2777851165098011123714154
	NK277785116PK277785116: .double -0.2777851165098011123714154, +0.2777851165098011123714154
	PK415734806PK415734806: .double +0.4157348061512726185393942, +0.4157348061512726185393942
	PK490392640PK490392640: .double +0.4903926402016152245630911, +0.4903926402016152245630911
	NK097545161PK097545161: .double -0.0975451610080641339241424, +0.0975451610080641339241424
	PK097545161PK097545161: .double +0.0975451610080641339241424, +0.0975451610080641339241424
	NK490392640PK490392640: .double -0.4903926402016152245630911, +0.4903926402016152245630911

	

.text
	.balign 64
	.globl	fftw_hc2hc_forward_16
	.type	fftw_hc2hc_forward_16, @function
fftw_hc2hc_forward_16:
	/* promise simd cell size = 16 */ 
	subl $16, %esp
	movl 28(%esp), %ecx
	movl 20(%esp), %eax
	movl %ebx, 12(%esp)
	movl %esi, 8(%esp)
	movl %ebp, 0(%esp)
	movl %edi, 4(%esp)
	sall $3, 36(%esp)
	leal (,%ecx,8), %edx
	leal (%edx,%edx,2), %ebx
	movhpd (%eax,%edx,4), %xmm1
	leal (%edx,%edx,8), %esi
	movlpd (%eax), %xmm1
	movlpd (%eax,%edx,8), %xmm0
	leal (%edx,%edx,4), %edi
	movhpd (%eax,%ebx,4), %xmm0
	leal (%esi,%edx,4), %ebp
	movlpd (%eax,%esi), %xmm4
	leal (%edi,%edx,2), %ecx
	movhpd (%eax,%edi), %xmm5
	movlpd (%eax,%edx), %xmm5
	movapd %xmm1, %xmm2
	movapd %xmm1, %xmm3
	movhpd (%eax,%ebp), %xmm4
	movlpd (%eax,%ecx), %xmm1
	addpd %xmm0, %xmm2
	subpd %xmm0, %xmm3
	movhpd (%eax,%ebx), %xmm0
	leal (%esi,%edx,2), %ebx
	leal (%ebp,%edx,2), %esi
	movapd %xmm5, %xmm6
	movapd %xmm3, fftw_stackcell_0
	addpd %xmm4, %xmm6
	movlpd (%eax,%esi), %xmm0
	movhpd (%eax,%ebx), %xmm1
	subpd %xmm4, %xmm5
	movapd %xmm0, %xmm7
	movapd %xmm0, %xmm3
	movlpd (%eax,%edi,2), %xmm0
	leal (%edx,%edx,2), %edi
	addpd %xmm1, %xmm7
	subpd %xmm1, %xmm3
	movhpd (%eax,%ecx,2), %xmm1
	movlpd (%eax,%edx,2), %xmm1
	leal (%edx,%edx,2), %ecx
	movhpd (%eax,%edi,2), %xmm0
	/* simd data load/store barrier */ 
	leal (%edx,%edx,8), %edi
	leal (%ecx,%edx,8), %ebp
	leal (%ecx,%edx,4), %ebx
	movapd %xmm1, %xmm4
	subpd %xmm0, %xmm1
	addpd %xmm0, %xmm4
	movapd %xmm3, %xmm0
	mulpd PK923879532PK382683432, %xmm3
	leal (%ebp,%edx,2), %esi
	movapd %xmm1, fftw_stackcell_1
	movapd %xmm5, %xmm1
	mulpd PK923879532PK382683432, %xmm5
	mulpd PK382683432PK923879532, %xmm0
	movapd %xmm3, fftw_stackcell_2
	movapd %xmm2, %xmm3
	mulpd PK382683432PK923879532, %xmm1
	movapd %xmm5, fftw_stackcell_3
	movapd %xmm2, %xmm5
	unpckhpd %xmm7, %xmm3
	unpcklpd %xmm7, %xmm5
	addpd %xmm5, %xmm3
	movapd %xmm6, %xmm5
	movapd %xmm3, fftw_stackcell_4
	movapd %xmm6, %xmm3
	unpcklpd %xmm7, %xmm5
	unpckhpd %xmm7, %xmm3
	movapd %xmm4, %xmm7
	unpcklpd %xmm6, %xmm7
	subpd %xmm3, %xmm5
	movapd %xmm4, %xmm3
	shufpd $9, %xmm4, %xmm4
	unpckhpd %xmm6, %xmm3
	movapd %xmm0, %xmm6
	unpcklpd %xmm1, %xmm0
	unpckhpd %xmm1, %xmm6
	movapd fftw_stackcell_3, %xmm1
	addpd %xmm7, %xmm3
	movapd fftw_stackcell_2, %xmm7
	mulsd chs_lo, %xmm6
	addpd %xmm0, %xmm6
	movapd %xmm1, %xmm0
	unpckhpd %xmm7, %xmm1
	unpcklpd %xmm7, %xmm0
	movapd %xmm2, %xmm7
	unpckhpd %xmm4, %xmm2
	mulsd chs_lo, %xmm1
	unpcklpd %xmm4, %xmm7
	movapd fftw_stackcell_1, %xmm4
	addpd %xmm0, %xmm1
	subpd %xmm2, %xmm7
	movapd fftw_stackcell_4, %xmm2
	movapd %xmm4, %xmm0
	mulsd chs_lo, %xmm4
	shufpd $9, %xmm0, %xmm0
	addpd %xmm4, %xmm0
	movapd %xmm2, %xmm4
	subpd %xmm3, %xmm2
	addpd %xmm3, %xmm4
	mulpd PK707106781PK707106781, %xmm0
	movlpd %xmm2, (%eax,%edx,4)
	movhpd %xmm2, (%eax,%ecx,4)
	movapd %xmm4, %xmm3
	movapd %xmm1, %xmm2
	shufpd $9, %xmm3, %xmm3
	shufpd $9, %xmm2, %xmm2
	mulsd chs_lo, %xmm3
	mulsd chs_lo, %xmm2
	addpd %xmm3, %xmm4
	movapd %xmm0, %xmm3
	subpd %xmm2, %xmm1
	movapd fftw_stackcell_0, %xmm2
	shufpd $9, %xmm3, %xmm3
	movlpd %xmm4, (%eax,%edx,8)
	movhpd %xmm4, (%eax)
	movapd %xmm2, %xmm4
	addpd %xmm3, %xmm2
	shufpd $9, %xmm4, %xmm4
	movapd %xmm2, %xmm3
	mulpd chs_hi, %xmm3
	subpd %xmm4, %xmm0
	movapd %xmm1, %xmm4
	mulsd chs_lo, %xmm1
	addpd %xmm3, %xmm4
	movapd %xmm6, %xmm3
	addpd %xmm1, %xmm2
	movapd %xmm5, %xmm1
	shufpd $9, %xmm3, %xmm3
	movhpd %xmm4, (%eax,%ebp)
	leal (%edx,%edx,4), %ebp
	movlpd %xmm4, (%eax,%edx)
	shufpd $9, %xmm1, %xmm1
	movlpd %xmm2, (%eax,%ebx)
	movhpd %xmm2, (%eax,%esi)
	leal (%edx,%edx,2), %ebx
	movapd %xmm0, %xmm2
	mulpd chs_hi, %xmm0
	mulsd chs_lo, %xmm3
	mulsd chs_lo, %xmm1
	mulpd chs_lohi, %xmm2
	addpd %xmm3, %xmm6
	subpd %xmm1, %xmm5
	movapd %xmm6, %xmm3
	mulsd chs_lo, %xmm6
	movapd %xmm5, %xmm4
	mulpd NK707106781PK707106781, %xmm5
	addpd %xmm2, %xmm3
	movapd %xmm7, %xmm2
	mulpd chs_hi, %xmm7
	mulpd PK707106781PK707106781, %xmm4
	subpd %xmm6, %xmm0
	movhpd %xmm3, (%eax,%ecx)
	leal (%esi,%edx,2), %ecx
	leal (%ebp,%edx,2), %esi
	addpd %xmm5, %xmm2
	movlpd %xmm3, (%eax,%edi)
	movhpd %xmm0, (%eax,%ebp)
	addpd %xmm7, %xmm4
	movl 24(%esp), %edi
	movlpd %xmm2, (%eax,%ebx,2)
	movlpd %xmm0, (%eax,%ecx)
	movhpd %xmm2, (%eax,%esi,2)
	movl %eax, %ecx
	movlpd %xmm4, (%eax,%edx,2)
	movhpd %xmm4, (%eax,%ebp,2)
	movl %eax, %eax
	jmp .LC2
	.p2align 4,,7
.LC1:
	/* promise simd cell size = 16 */ 
	leal (%edx,%edx,8), %ebp
	leal (%edx,%edx,2), %esi
	movapd 112(%edi), %xmm6
	leal (%edx,%edx), %ebx
	movapd 224(%edi), %xmm7
	movhpd (%eax,%ebp), %xmm1
	leal (%esi,%esi,4), %ebp
	movlpd (%ecx,%edx,8), %xmm1
	movapd %xmm6, %xmm5
	movapd %xmm6, %xmm3
	movhpd (%eax,%ebx,8), %xmm6
	leal (%esi,%edx,8), %ebx
	movapd %xmm7, %xmm0
	movlpd (%ecx,%ebp), %xmm6
	leal (%esi,%edx,4), %ebp
	mulpd %xmm1, %xmm5
	shufpd $9, %xmm1, %xmm1
	mulpd %xmm1, %xmm3
	movapd %xmm5, %xmm4
	movapd %xmm6, %xmm2
	mulpd %xmm6, %xmm7
	movhpd (%eax,%esi,4), %xmm6
	movlpd (%ecx,%ebx), %xmm6
	shufpd $9, %xmm2, %xmm2
	unpckhpd %xmm3, %xmm4
	movapd %xmm7, %xmm1
	unpcklpd %xmm3, %xmm5
	mulpd %xmm2, %xmm0
	mulsd chs_lo, %xmm4
	movapd 160(%edi), %xmm2
	movapd %xmm6, %xmm3
	unpcklpd %xmm0, %xmm1
	addpd %xmm5, %xmm4
	unpckhpd %xmm0, %xmm7
	shufpd $9, %xmm3, %xmm3
	movapd %xmm2, %xmm5
	mulpd %xmm6, %xmm2
	movapd %xmm4, fftw_stackcell_7
	mulsd chs_lo, %xmm7
	movapd 96(%edi), %xmm4
	mulpd %xmm3, %xmm5
	movapd %xmm2, %xmm0
	movapd %xmm2, %xmm6
	movhpd (%eax,%edx,8), %xmm2
	movlpd (%ecx,%ebp), %xmm2
	addpd %xmm1, %xmm7
	unpckhpd %xmm5, %xmm0
	movapd %xmm4, %xmm3
	unpcklpd %xmm5, %xmm6
	mulsd chs_lo, %xmm0
	movapd 32(%edi), %xmm5
	mulpd %xmm2, %xmm4
	shufpd $9, %xmm2, %xmm2
	addpd %xmm6, %xmm0
	mulpd %xmm2, %xmm3
	movapd %xmm4, %xmm1
	movapd %xmm5, %xmm6
	movapd %xmm7, %xmm2
	unpckhpd %xmm3, %xmm1
	unpcklpd %xmm3, %xmm4
	movhpd (%eax,%edx,4), %xmm3
	movlpd (%ecx,%esi), %xmm3
	mulsd chs_lo, %xmm1
	addpd %xmm4, %xmm1
	mulpd %xmm3, %xmm5
	shufpd $9, %xmm3, %xmm3
	addpd %xmm1, %xmm2
	subpd %xmm1, %xmm7
	mulpd %xmm3, %xmm6
	movapd %xmm5, %xmm4
	movapd %xmm7, %xmm1
	unpckhpd %xmm6, %xmm4
	unpcklpd %xmm6, %xmm5
	movapd %xmm2, %xmm6
	shufpd $9, %xmm1, %xmm1
	mulsd chs_lo, %xmm4
	addpd %xmm5, %xmm4
	movapd %xmm4, %xmm3
	subpd %xmm0, %xmm4
	addpd %xmm0, %xmm3
	movapd %xmm4, %xmm5
	movapd %xmm4, %xmm0
	movhpd (%eax,%esi), %xmm4
	movlpd (%ecx,%edx,2), %xmm4
	subpd %xmm3, %xmm6
	shufpd $9, %xmm5, %xmm5
	addpd %xmm3, %xmm2
	movapd %xmm6, fftw_stackcell_8
	movapd 16(%edi), %xmm6
	addpd %xmm1, %xmm0
	subpd %xmm5, %xmm7
	movapd %xmm0, fftw_stackcell_5
	movapd %xmm4, %xmm3
	movapd %xmm7, fftw_stackcell_6
	movapd 80(%edi), %xmm7
	shufpd $9, %xmm3, %xmm3
	movapd %xmm6, %xmm1
	mulpd %xmm4, %xmm6
	mulpd %xmm3, %xmm1
	movapd %xmm6, %xmm5
	movapd %xmm7, %xmm4
	unpckhpd %xmm1, %xmm5
	unpcklpd %xmm1, %xmm6
	movapd %xmm5, %xmm0
	movhpd (%eax,%ebp), %xmm5
	movlpd (%ecx,%esi,2), %xmm5
	leal (%edx,%edx,4), %esi
	mulsd chs_lo, %xmm0
	addpd %xmm6, %xmm0
	movapd 144(%edi), %xmm6
	mulpd %xmm5, %xmm7
	shufpd $9, %xmm5, %xmm5
	mulpd %xmm5, %xmm4
	movapd %xmm7, %xmm3
	movapd %xmm6, %xmm5
	unpckhpd %xmm4, %xmm3
	unpcklpd %xmm4, %xmm7
	movapd %xmm3, %xmm1
	movhpd (%eax,%ebx), %xmm3
	movlpd (%ecx,%esi,2), %xmm3
	leal (%ebp,%edx,8), %ebx
	mulsd chs_lo, %xmm1
	addpd %xmm7, %xmm1
	mulpd %xmm3, %xmm6
	shufpd $9, %xmm3, %xmm3
	mulpd %xmm3, %xmm5
	movapd %xmm6, %xmm4
	movapd 208(%edi), %xmm3
	unpckhpd %xmm5, %xmm4
	unpcklpd %xmm5, %xmm6
	movapd %xmm4, %xmm7
	movhpd (%eax,%ebx), %xmm4
	movlpd (%ecx,%ebp,2), %xmm4
	movapd %xmm3, %xmm5
	leal (%esi,%edx,8), %ebp
	mulsd chs_lo, %xmm7
	addpd %xmm6, %xmm7
	movapd %xmm0, %xmm6
	mulpd %xmm4, %xmm3
	shufpd $9, %xmm4, %xmm4
	addpd %xmm7, %xmm0
	subpd %xmm7, %xmm6
	mulpd %xmm4, %xmm5
	movapd %xmm3, %xmm7
	movapd %xmm0, fftw_stackcell_14
	shufpd $9, %xmm6, %xmm6
	unpckhpd %xmm5, %xmm7
	unpcklpd %xmm5, %xmm3
	movapd %xmm6, %xmm0
	movapd %xmm6, %xmm5
	movhpd (%eax,%esi), %xmm6
	movlpd (%ecx,%edx,4), %xmm6
	leal (%edx,%edx,2), %esi
	mulsd chs_lo, %xmm7
	addpd %xmm3, %xmm7
	leal (%esi,%edx,4), %ebx
	movapd %xmm7, %xmm4
	addpd %xmm1, %xmm7
	movapd %xmm6, %xmm3
	subpd %xmm1, %xmm4
	movapd %xmm7, fftw_stackcell_13
	movapd 48(%edi), %xmm1
	shufpd $9, %xmm3, %xmm3
	unpckhpd %xmm4, %xmm0
	unpcklpd %xmm4, %xmm5
	movapd %xmm5, fftw_stackcell_10
	mulsd chs_lo, %xmm0
	movapd %xmm1, %xmm7
	mulpd %xmm3, %xmm1
	movapd 176(%edi), %xmm3
	mulpd %xmm6, %xmm7
	movapd %xmm0, fftw_stackcell_9
	movapd %xmm7, %xmm4
	unpcklpd %xmm1, %xmm7
	movapd %xmm3, %xmm6
	unpckhpd %xmm1, %xmm4
	movapd %xmm4, %xmm5
	movhpd (%eax,%ebp), %xmm4
	movlpd (%ecx,%esi,4), %xmm4
	leal (%edx,%edx,8), %esi
	mulsd chs_lo, %xmm5
	addpd %xmm7, %xmm5
	mulpd %xmm4, %xmm6
	shufpd $9, %xmm4, %xmm4
	movapd %xmm5, %xmm1
	mulpd %xmm4, %xmm3
	movapd %xmm6, %xmm0
	movapd (%edi), %xmm4
	unpckhpd %xmm3, %xmm0
	unpcklpd %xmm3, %xmm6
	mulsd chs_lo, %xmm0
	movapd %xmm4, %xmm3
	addpd %xmm6, %xmm0
	subpd %xmm0, %xmm1
	addpd %xmm0, %xmm5
	movapd %xmm1, %xmm7
	movhpd (%eax,%edx,2), %xmm1
	movlpd (%ecx,%edx), %xmm1
	movapd %xmm5, fftw_stackcell_12
	movapd 192(%edi), %xmm5
	shufpd $9, %xmm7, %xmm7
	movapd %xmm7, fftw_stackcell_11
	mulpd %xmm1, %xmm4
	shufpd $9, %xmm1, %xmm1
	movapd %xmm5, %xmm7
	mulpd %xmm1, %xmm3
	movapd %xmm4, %xmm6
	unpckhpd %xmm3, %xmm6
	unpcklpd %xmm3, %xmm4
	movapd %xmm6, %xmm0
	movhpd (%eax,%ebx,2), %xmm6
	movlpd (%ecx,%ebp), %xmm6
	leal (%edx,%edx,4), %ebp
	leal (%edx,%edx,2), %ebx
	mulsd chs_lo, %xmm0
	addpd %xmm4, %xmm0
	movapd 128(%edi), %xmm4
	mulpd %xmm6, %xmm5
	shufpd $9, %xmm6, %xmm6
	mulpd %xmm6, %xmm7
	movapd %xmm5, %xmm1
	movapd %xmm4, %xmm6
	unpckhpd %xmm7, %xmm1
	unpcklpd %xmm7, %xmm5
	movapd %xmm1, %xmm3
	movhpd (%eax,%ebp,2), %xmm1
	movlpd (%ecx,%esi), %xmm1
	leal (%edx,%edx,2), %esi
	mulsd chs_lo, %xmm3
	addpd %xmm5, %xmm3
	mulpd %xmm1, %xmm4
	shufpd $9, %xmm1, %xmm1
	mulpd %xmm1, %xmm6
	movapd %xmm4, %xmm7
	movapd 64(%edi), %xmm1
	addl $240, %edi
	unpckhpd %xmm6, %xmm7
	unpcklpd %xmm6, %xmm4
	movapd %xmm7, %xmm5
	movhpd (%eax,%ebx,2), %xmm7
	movlpd (%ecx,%ebp), %xmm7
	movapd %xmm1, %xmm6
	leal (%edx,%edx), %ebp
	leal (%edx,%esi,2), %ebx
	mulsd chs_lo, %xmm5
	addpd %xmm4, %xmm5
	movapd %xmm0, %xmm4
	mulpd %xmm7, %xmm1
	shufpd $9, %xmm7, %xmm7
	subpd %xmm5, %xmm0
	addpd %xmm5, %xmm4
	mulpd %xmm7, %xmm6
	movapd %xmm1, %xmm5
	unpckhpd %xmm6, %xmm5
	unpcklpd %xmm6, %xmm1
	movapd %xmm0, %xmm6
	mulsd chs_lo, %xmm5
	shufpd $9, %xmm6, %xmm6
	addpd %xmm1, %xmm5
	movapd %xmm4, %xmm1
	movapd %xmm5, %xmm7
	subpd %xmm3, %xmm5
	addpd %xmm3, %xmm7
	addpd %xmm5, %xmm6
	shufpd $9, %xmm5, %xmm5
	subpd %xmm7, %xmm1
	addpd %xmm7, %xmm4
	movapd %xmm6, fftw_stackcell_15
	movapd fftw_stackcell_8, %xmm6
	movapd %xmm1, %xmm7
	subpd %xmm5, %xmm0
	movapd %xmm4, %xmm3
	addpd %xmm2, %xmm4
	shufpd $9, %xmm7, %xmm7
	movapd %xmm0, fftw_stackcell_16
	movapd fftw_stackcell_13, %xmm0
	subpd %xmm2, %xmm3
	movapd fftw_stackcell_14, %xmm2
	movapd %xmm6, %xmm5
	mulsd chs_lo, %xmm7
	shufpd $9, %xmm5, %xmm5
	mulsd chs_lo, %xmm3
	subpd %xmm7, %xmm1
	movapd %xmm2, %xmm7
	mulsd chs_lo, %xmm5
	subpd %xmm0, %xmm2
	shufpd $9, %xmm3, %xmm3
	addpd %xmm0, %xmm7
	mulsd chs_lo, %xmm2
	addpd %xmm5, %xmm6
	movapd %xmm3, fftw_stackcell_17
	movapd fftw_stackcell_7, %xmm5
	movapd %xmm1, %xmm3
	addpd %xmm6, %xmm1
	movapd %xmm2, %xmm0
	movhpd (%eax,%edx), %xmm2
	movlpd (%ecx), %xmm2
	subpd %xmm6, %xmm3
	mulpd PK707106781PK707106781, %xmm1
	/* simd data load/store barrier */ 
	shufpd $9, %xmm0, %xmm0
	shufpd $9, %xmm3, %xmm3
	movapd %xmm2, %xmm6
	subpd %xmm5, %xmm2
	mulpd PK707106781NK707106781, %xmm3
	addpd %xmm5, %xmm6
	movapd %xmm2, fftw_stackcell_18
	movapd fftw_stackcell_12, %xmm5
	movapd %xmm6, %xmm2
	addpd %xmm5, %xmm2
	subpd %xmm5, %xmm6
	movapd %xmm2, %xmm5
	subpd %xmm7, %xmm2
	addpd %xmm7, %xmm5
	movapd %xmm6, %xmm7
	subpd %xmm0, %xmm6
	addpd %xmm0, %xmm7
	movapd %xmm5, %xmm0
	subpd %xmm4, %xmm5
	addpd %xmm4, %xmm0
	movapd %xmm7, %xmm4
	mulpd chs_hi, %xmm5
	subpd %xmm1, %xmm7
	addpd %xmm1, %xmm4
	movapd fftw_stackcell_17, %xmm1
	movlpd %xmm0, (%ecx)
	movhpd %xmm0, (%eax,%ebp,8)
	movapd %xmm6, %xmm0
	leal (%edx,%edx,4), %ebp
	addpd %xmm3, %xmm6
	movlpd %xmm5, (%eax,%edx,8)
	movhpd %xmm5, (%ecx,%edx,8)
	mulpd chs_hi, %xmm7
	movapd %xmm2, %xmm5
	subpd %xmm3, %xmm0
	movlpd %xmm4, (%ecx,%edx,2)
	movhpd %xmm4, (%eax,%ebx,2)
	movapd fftw_stackcell_10, %xmm4
	subpd %xmm1, %xmm2
	addpd %xmm1, %xmm5
	mulpd chs_hi, %xmm0
	movlpd %xmm6, (%ecx,%esi,2)
	movapd fftw_stackcell_18, %xmm3
	movhpd %xmm6, (%eax,%ebp,2)
	movapd fftw_stackcell_11, %xmm6
	mulpd chs_hi, %xmm2
	movlpd %xmm5, (%ecx,%edx,4)
	movhpd %xmm5, (%eax,%esi,4)
	movapd fftw_stackcell_9, %xmm5
	movlpd %xmm0, (%eax,%edx,2)
	movhpd %xmm0, (%ecx,%ebx,2)
	movapd %xmm4, %xmm0
	movlpd %xmm7, (%eax,%esi,2)
	movhpd %xmm7, (%ecx,%ebp,2)
	movapd %xmm3, %xmm1
	addpd %xmm6, %xmm3
	movlpd %xmm2, (%eax,%edx,4)
	addpd %xmm5, %xmm0
	movhpd %xmm2, (%ecx,%esi,4)
	movapd fftw_stackcell_16, %xmm2
	subpd %xmm5, %xmm4
	movapd %xmm3, fftw_stackcell_20
	movapd %xmm0, %xmm7
	subpd %xmm6, %xmm1
	movapd fftw_stackcell_15, %xmm6
	movapd %xmm4, %xmm3
	shufpd $9, %xmm7, %xmm7
	movapd %xmm2, %xmm5
	mulpd NK923879532PK382683432, %xmm2
	shufpd $9, %xmm3, %xmm3
	mulpd PK382683432PK923879532, %xmm5
	mulsd chs_lo, %xmm7
	mulsd chs_lo, %xmm3
	addpd %xmm7, %xmm0
	movapd %xmm6, %xmm7
	mulpd PK382683432PK923879532, %xmm6
	subpd %xmm3, %xmm4
	movapd fftw_stackcell_5, %xmm3
	mulpd PK923879532NK382683432, %xmm7
	mulpd PK707106781PK707106781, %xmm4
	addpd %xmm2, %xmm6
	mulpd PK707106781PK707106781, %xmm0
	addpd %xmm7, %xmm5
	movapd fftw_stackcell_6, %xmm7
	movapd %xmm3, %xmm2
	mulpd PK382683432PK923879532, %xmm3
	movapd %xmm4, fftw_stackcell_19
	mulpd NK923879532PK382683432, %xmm2
	movapd %xmm7, %xmm4
	mulpd PK923879532NK382683432, %xmm7
	mulpd PK382683432PK923879532, %xmm4
	addpd %xmm7, %xmm3
	movapd %xmm6, %xmm7
	addpd %xmm2, %xmm4
	movapd %xmm5, %xmm2
	subpd %xmm3, %xmm6
	addpd %xmm4, %xmm2
	subpd %xmm4, %xmm5
	movapd %xmm1, %xmm4
	mulpd chs_hi, %xmm6
	addpd %xmm3, %xmm7
	addpd %xmm0, %xmm4
	mulsd chs_lo, %xmm5
	subpd %xmm0, %xmm1
	shufpd $9, %xmm7, %xmm7
	movapd %xmm4, %xmm3
	addpd %xmm2, %xmm4
	movapd %xmm1, %xmm0
	addpd %xmm6, %xmm1
	shufpd $9, %xmm5, %xmm5
	movlpd %xmm4, (%ecx,%esi)
	subpd %xmm2, %xmm3
	leal (%ebx,%edx,8), %esi
	movapd fftw_stackcell_20, %xmm2
	subpd %xmm6, %xmm0
	movapd fftw_stackcell_19, %xmm6
	leal (%edx,%edx,8), %ebx
	mulpd chs_hi, %xmm3
	mulpd chs_hi, %xmm0
	movhpd %xmm4, (%eax,%esi)
	movapd %xmm2, %xmm4
	leal (%ebp,%edx,2), %esi
	subpd %xmm6, %xmm2
	movlpd %xmm3, (%eax,%ebp)
	movhpd %xmm3, (%ecx,%ebx)
	leal (%ebx,%edx,2), %ebp
	addpd %xmm6, %xmm4
	movlpd %xmm0, (%eax,%edx)
	movapd %xmm2, %xmm3
	subpd %xmm5, %xmm2
	movapd %xmm4, %xmm6
	movlpd %xmm1, (%ecx,%esi)
	addpd %xmm7, %xmm4
	addpd %xmm5, %xmm3
	movhpd %xmm1, (%eax,%ebp)
	leal (%ebp,%edx,2), %ebx
	mulpd chs_hi, %xmm2
	subpd %xmm7, %xmm6
	movlpd %xmm4, (%ecx,%edx)
	mulpd chs_hi, %xmm6
	movhpd %xmm0, (%ecx,%ebx)
	movhpd %xmm4, (%eax,%ebx)
	leal (%edx,%edx,2), %ebx
	movlpd %xmm6, (%eax,%esi)
	leal (%edx,%edx,4), %esi
	movhpd %xmm6, (%ecx,%ebp)
	leal (%edx,%edx,8), %ebp
	movlpd %xmm2, (%eax,%ebx)
	movlpd %xmm3, (%ecx,%esi)
	leal (%edx,%edx,4), %esi
	movhpd %xmm3, (%eax,%ebp)
	leal (%esi,%esi,2), %ebp
	movhpd %xmm2, (%ecx,%ebp)
	addl 36(%esp), %ecx
	subl 36(%esp), %eax
	subl $2, 32(%esp)
	jg .LC1
	je .LC3
	jmp .LC4
	.p2align 4,,7
.LC2:
	addl 36(%esp), %ecx
	subl 36(%esp), %eax
	subl $2, 32(%esp)
	jg .LC1
	je .LC3
	jmp .LC4
	.p2align 4,,7
.LC3:
	/* promise simd cell size = 16 */ 
	leal (%edx,%edx,2), %ebx
	leal (%edx,%edx,4), %esi
	movhpd (%ecx,%edx,8), %xmm3
	movlpd (%ecx), %xmm3
	movhpd (%ecx,%ebx,4), %xmm4
	movlpd (%ecx,%edx,4), %xmm4
	leal (%esi,%edx,2), %ebp
	movapd %xmm3, %xmm0
	leal (%ebp,%edx,8), %edi
	movapd %xmm4, %xmm5
	shufpd $9, %xmm5, %xmm5
	movapd %xmm5, %xmm1
	movhpd (%ecx,%esi,2), %xmm5
	movlpd (%ecx,%edx,2), %xmm5
	mulsd chs_lo, %xmm1
	addpd %xmm1, %xmm4
	movapd %xmm5, %xmm7
	mulpd PK923879532PK923879532, %xmm5
	mulpd PK707106781PK707106781, %xmm4
	shufpd $9, %xmm7, %xmm7
	subpd %xmm4, %xmm0
	addpd %xmm4, %xmm3
	movhpd (%ecx,%ebp,2), %xmm4
	movlpd (%ecx,%ebx,2), %xmm4
	mulpd NK382683432PK382683432, %xmm7
	leal (%edx,%edx,8), %ebx
	movapd %xmm3, fftw_stackcell_22
	addpd %xmm7, %xmm5
	movapd %xmm5, %xmm2
	leal (%ebx,%edx,4), %esi
	movapd %xmm4, %xmm6
	mulpd PK382683432PK382683432, %xmm4
	shufpd $9, %xmm6, %xmm6
	mulpd NK923879532PK923879532, %xmm6
	addpd %xmm6, %xmm4
	addpd %xmm4, %xmm2
	subpd %xmm4, %xmm5
	movapd %xmm2, %xmm3
	movhpd (%ecx,%ebp), %xmm2
	movlpd (%ecx,%ebx), %xmm2
	leal (%edx,%edx,2), %ebp
	movapd %xmm5, %xmm1
	movhpd (%ecx,%edi), %xmm5
	leal (%ebx,%edx,2), %edi
	leal (%edx,%edx,4), %ebx
	mulsd chs_lo, %xmm3
	movlpd (%ecx,%edx), %xmm5
	shufpd $9, %xmm1, %xmm1
	movapd %xmm3, fftw_stackcell_21
	movhpd (%ecx,%ebp), %xmm3
	movlpd (%ecx,%ebx), %xmm3
	movapd %xmm2, %xmm7
	movlpd (%ecx,%esi), %xmm2
	leal (%edx,%edx,2), %esi
	movhpd (%ecx,%edi), %xmm2
	leal (%edx,%edx,4), %edi
	/* simd data load/store barrier */ 
	movapd %xmm5, %xmm6
	mulpd PK2_000000000PK2_000000000, %xmm7
	mulpd PK2_000000000PK2_000000000, %xmm5
	mulpd PK2_000000000NK2_000000000, %xmm6
	leal (%esi,%esi,4), %ebp
	leal (%esi,%edx,8), %ebx
	movapd %xmm3, %xmm4
	addpd %xmm2, %xmm3
	subpd %xmm2, %xmm4
	mulpd PK1_414213562PK1_414213562, %xmm3
	movapd %xmm4, %xmm2
	mulpd NK1_414213562PK1_414213562, %xmm4
	mulpd PK1_414213562PK1_414213562, %xmm2
	addpd %xmm4, %xmm5
	addpd %xmm6, %xmm2
	movapd %xmm7, %xmm6
	movapd %xmm5, %xmm4
	mulpd NK415734806PK415734806, %xmm5
	subpd %xmm3, %xmm6
	shufpd $9, %xmm4, %xmm4
	addpd %xmm7, %xmm3
	movapd %xmm6, %xmm7
	shufpd $9, %xmm6, %xmm6
	mulpd PK277785116PK277785116, %xmm4
	mulpd NK277785116PK277785116, %xmm7
	mulpd PK415734806PK415734806, %xmm6
	addpd %xmm7, %xmm5
	movapd %xmm2, %xmm7
	mulpd PK490392640PK490392640, %xmm2
	subpd %xmm4, %xmm6
	movapd %xmm3, %xmm4
	mulpd NK097545161PK097545161, %xmm3
	mulpd PK097545161PK097545161, %xmm7
	mulpd NK490392640PK490392640, %xmm4
	addpd %xmm3, %xmm2
	movapd %xmm0, %xmm3
	mulpd chs_hi, %xmm0
	subpd %xmm4, %xmm7
	movapd %xmm1, %xmm4
	mulpd chs_hi, %xmm1
	mulpd chs_lohi, %xmm4
	addpd %xmm0, %xmm1
	movapd %xmm6, %xmm0
	shufpd $9, %xmm0, %xmm0
	addpd %xmm4, %xmm3
	movapd %xmm5, %xmm4
	mulpd chs_hi, %xmm5
	shufpd $9, %xmm4, %xmm4
	mulsd chs_lo, %xmm0
	addpd %xmm5, %xmm4
	movapd %xmm3, %xmm5
	addpd %xmm0, %xmm6
	movapd %xmm7, %xmm0
	subpd %xmm6, %xmm3
	shufpd $9, %xmm0, %xmm0
	addpd %xmm6, %xmm5
	movapd %xmm4, %xmm6
	mulsd chs_lo, %xmm4
	mulpd chs_hi, %xmm3
	movlpd %xmm5, (%ecx,%edx,2)
	movhpd %xmm5, (%eax,%ebp)
	mulsd chs_lo, %xmm0
	leal (%edx,%edx,2), %ebp
	movapd %xmm1, %xmm5
	addpd %xmm4, %xmm1
	movapd %xmm2, %xmm4
	movlpd %xmm3, (%ecx,%edi)
	movhpd %xmm3, (%eax,%edi,2)
	movapd fftw_stackcell_22, %xmm3
	mulpd chs_hi, %xmm5
	shufpd $9, %xmm4, %xmm4
	movlpd %xmm1, (%ecx,%edx)
	subpd %xmm0, %xmm7
	movapd fftw_stackcell_21, %xmm0
	addpd %xmm5, %xmm6
	mulsd chs_lo, %xmm4
	movapd %xmm3, %xmm5
	subpd %xmm0, %xmm3
	movlpd %xmm6, (%ecx,%esi,2)
	leal (%edi,%edx,2), %esi
	leal (%edx,%edx,8), %edi
	addpd %xmm0, %xmm5
	movhpd %xmm6, (%eax,%ebx)
	leal (%edx,%edx), %ebx
	subpd %xmm4, %xmm2
	movapd %xmm3, %xmm0
	movapd %xmm5, %xmm6
	addpd %xmm7, %xmm5
	subpd %xmm2, %xmm0
	movhpd %xmm1, (%eax,%esi,2)
	leal (%edi,%edx,4), %esi
	subpd %xmm7, %xmm6
	movhpd %xmm5, (%eax,%edi)
	leal (%ebp,%edx,4), %edi
	movlpd %xmm5, (%ecx,%ebp)
	mulpd chs_hi, %xmm0
	addpd %xmm2, %xmm3
	mulpd chs_hi, %xmm6
	movlpd %xmm3, (%ecx)
	movhpd %xmm3, (%eax,%esi)
	movlpd %xmm0, (%ecx,%edi)
	movhpd %xmm0, (%eax,%ebp,4)
	movlpd %xmm6, (%ecx,%edx,4)
	movhpd %xmm6, (%eax,%ebx,8)
	.p2align 4,,7
.LC4:
	movl 12(%esp), %ebx
	movl 8(%esp), %esi
	movl 4(%esp), %edi
	movl 0(%esp), %ebp
	addl $16, %esp
	ret 


.section .rodata
	.balign 4
twiddle_order:
	.long 1
	.long 2
	.long 3
	.long 4
	.long 5
	.long 6
	.long 7
	.long 8
	.long 9
	.long 10
	.long 11
	.long 12
	.long 13
	.long 14
	.long 15

	.globl fftw_hc2hc_forward_16_desc
	.balign 32
fftw_hc2hc_forward_16_desc:
	/* descr_str_ptr */	.long .LC0
	/* codelet_ptr */	.long fftw_hc2hc_forward_16
	/* n */			.long 16
	/* cdir_int */		.long -1
	/* ctype_int */		.long 6
	/* signature */		.long 355
	/* num_twiddles */	.long 15
	/* twiddle_order */	.long twiddle_order
.LC0:	.string "fftw_hc2hc_forward_16"
