diff -Nru a/Makefile b/Makefile
--- a/Makefile	2004-08-25 04:10:06 -07:00
+++ b/Makefile	2004-08-28 02:48:21 -07:00
@@ -1,7 +1,7 @@
 VERSION = 2
 PATCHLEVEL = 4
 SUBLEVEL = 28
-EXTRAVERSION = -pre2
+EXTRAVERSION = -pre2-bk1
 
 KERNELRELEASE=$(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
 
diff -Nru a/arch/sparc64/defconfig b/arch/sparc64/defconfig
--- a/arch/sparc64/defconfig	2004-05-19 16:32:32 -07:00
+++ b/arch/sparc64/defconfig	2004-08-26 14:42:36 -07:00
@@ -375,7 +375,7 @@
 CONFIG_NET_SCH_TEQL=m
 CONFIG_NET_SCH_TBF=m
 CONFIG_NET_SCH_GRED=m
-CONFIG_NET_SCH_DELAY=m
+CONFIG_NET_SCH_NETEM=m
 CONFIG_NET_SCH_DSMARK=m
 CONFIG_NET_SCH_INGRESS=m
 CONFIG_NET_QOS=y
@@ -410,6 +410,7 @@
 #
 # CONFIG_BLK_DEV_HD_IDE is not set
 # CONFIG_BLK_DEV_HD is not set
+# CONFIG_BLK_DEV_IDE_SATA is not set
 CONFIG_BLK_DEV_IDEDISK=y
 # CONFIG_IDEDISK_MULTI_MODE is not set
 # CONFIG_IDEDISK_STROKE is not set
@@ -1134,6 +1135,8 @@
 CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_TEA=m
+CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_ARC4=m
 CONFIG_CRYPTO_DEFLATE=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
diff -Nru a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
--- a/arch/sparc64/kernel/pci_iommu.c	2003-07-27 21:19:59 -07:00
+++ b/arch/sparc64/kernel/pci_iommu.c	2004-08-18 13:58:50 -07:00
@@ -56,6 +56,39 @@
 	}
 }
 
+#define IOPTE_CONSISTENT(CTX) \
+	(IOPTE_VALID | IOPTE_CACHE | \
+	 (((CTX) << 47) & IOPTE_CONTEXT))
+
+#define IOPTE_STREAMING(CTX) \
+	(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
+
+/* Existing mappings are never marked invalid, instead they
+ * are pointed to a dummy page.
+ */
+#define IOPTE_IS_DUMMY(iommu, iopte)	\
+	((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
+
+static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
+{
+	unsigned long val = iopte_val(*iopte);
+
+	val &= ~IOPTE_PAGE;
+	val |= iommu->dummy_page_pa;
+
+	iopte_val(*iopte) = val;
+}
+
+void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize)
+{
+	int i;
+
+	tsbsize /= sizeof(iopte_t);
+
+	for (i = 0; i < tsbsize; i++)
+		iopte_make_dummy(iommu, &iommu->page_table[i]);
+}
+
 static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
 {
 	iopte_t *iopte, *limit, *first;
@@ -79,7 +112,7 @@
 	
 	first = iopte;
 	for (;;) {
-		if (iopte_val(*iopte) == 0UL) {
+		if (IOPTE_IS_DUMMY(iommu, iopte)) {
 			if ((iopte + (1 << cnum)) >= limit)
 				ent = 0;
 			else
@@ -142,12 +175,12 @@
 	iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
 	while (iopte > iommu->page_table) {
 		iopte--;
-		if (!(iopte_val(*iopte) & IOPTE_VALID)) {
+		if (IOPTE_IS_DUMMY(iommu, iopte)) {
 			unsigned long tmp = npages;
 
 			while (--tmp) {
 				iopte--;
-				if (iopte_val(*iopte) & IOPTE_VALID)
+				if (!IOPTE_IS_DUMMY(iommu, iopte))
 					break;
 			}
 			if (tmp == 0) {
@@ -162,15 +195,6 @@
 	return NULL;
 }
 
-#define IOPTE_CONSISTENT(CTX) \
-	(IOPTE_VALID | IOPTE_CACHE | \
-	 (((CTX) << 47) & IOPTE_CONTEXT))
-
-#define IOPTE_STREAMING(CTX) \
-	(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
-
-#define IOPTE_INVALID	0UL
-
 /* Allocate and map kernel buffer of size SIZE using consistent mode
  * DMA for PCI device PDEV.  Return non-NULL cpu-side address if
  * successful and set *DMA_ADDRP to the PCI side dma address.
@@ -261,7 +285,7 @@
 		limit = (iommu->page_table +
 			 (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
 		while (walk < limit) {
-			if (iopte_val(*walk) != IOPTE_INVALID)
+			if (!IOPTE_IS_DUMMY(iommu, walk))
 				break;
 			walk++;
 		}
@@ -280,7 +304,7 @@
 		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
 
 	for (i = 0; i < npages; i++, iopte++)
-		iopte_val(*iopte) = IOPTE_INVALID;
+		iopte_make_dummy(iommu, iopte);
 
 	if (iommu->iommu_ctxflush) {
 		pci_iommu_write(iommu->iommu_ctxflush, ctx);
@@ -377,7 +401,7 @@
 	base = iommu->page_table +
 		((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 #ifdef DEBUG_PCI_IOMMU
-	if (iopte_val(*base) == IOPTE_INVALID)
+	if (IOPTE_IS_DUMMY(iommu, base))
 		printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
 		       bus_addr, sz, __builtin_return_address(0));
 #endif
@@ -416,7 +440,7 @@
 	}
 
 	/* Step 2: Clear out first TSB entry. */
-	iopte_val(*base) = IOPTE_INVALID;
+	iopte_make_dummy(iommu, base);
 
 	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
 			       npages, ctx);
@@ -617,7 +641,7 @@
 		((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 
 #ifdef DEBUG_PCI_IOMMU
-	if (iopte_val(*base) == IOPTE_INVALID)
+	if (IOPTE_IS_DUMMY(iommu, base))
 		printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
 #endif
 
@@ -654,7 +678,7 @@
 	}
 
 	/* Step 2: Clear out first TSB entry. */
-	iopte_val(*base) = IOPTE_INVALID;
+	iopte_make_dummy(iommu, base);
 
 	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
 			       npages, ctx);
diff -Nru a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
--- a/arch/sparc64/kernel/pci_psycho.c	2003-08-24 02:38:14 -07:00
+++ b/arch/sparc64/kernel/pci_psycho.c	2004-08-18 13:58:50 -07:00
@@ -1373,6 +1373,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	/* Using assumed page size 8K with 128K entries we need 1MB iommu page
 	 * table (128K ioptes * 8 bytes per iopte).  This is
 	 * page order 7 on UltraSparc.
@@ -1386,7 +1394,7 @@
 	iommu->page_table_sz_bits = 17;
 	iommu->page_table_map_base = 0xc0000000;
 	iommu->dma_addr_mask = 0xffffffff;
-	memset((char *)tsbbase, 0, IO_TSB_SIZE);
+	pci_iommu_table_init(iommu, IO_TSB_SIZE);
 
 	/* We start with no consistent mappings. */
 	iommu->lowest_consistent_map =
diff -Nru a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
--- a/arch/sparc64/kernel/pci_sabre.c	2003-12-18 12:24:22 -08:00
+++ b/arch/sparc64/kernel/pci_sabre.c	2004-08-18 13:58:50 -07:00
@@ -1315,6 +1315,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
 	if (!tsbbase) {
 		prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n");
@@ -1323,7 +1331,7 @@
 	iommu->page_table = (iopte_t *)tsbbase;
 	iommu->page_table_map_base = dvma_offset;
 	iommu->dma_addr_mask = dma_mask;
-	memset((char *)tsbbase, 0, PAGE_SIZE << order);
+	pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
 	sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
 
diff -Nru a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c
--- a/arch/sparc64/kernel/pci_schizo.c	2003-08-24 02:38:14 -07:00
+++ b/arch/sparc64/kernel/pci_schizo.c	2004-08-18 13:58:50 -07:00
@@ -1843,6 +1843,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	/* Using assumed page size 8K with 128K entries we need 1MB iommu page
 	 * table (128K ioptes * 8 bytes per iopte).  This is
 	 * page order 7 on UltraSparc.
@@ -1857,7 +1865,7 @@
 	iommu->page_table = (iopte_t *)tsbbase;
 	iommu->page_table_map_base = vdma[0];
 	iommu->dma_addr_mask = dma_mask;
-	memset((char *)tsbbase, 0, PAGE_SIZE << order);
+	pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
 	switch (tsbsize) {
 	case 64:
diff -Nru a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
--- a/arch/sparc64/kernel/process.c	2004-04-28 13:35:32 -07:00
+++ b/arch/sparc64/kernel/process.c	2004-08-26 14:58:34 -07:00
@@ -503,7 +503,7 @@
 
 	distance = fp - psp;
 	rval = (csp - distance);
-	if (copy_in_user(rval, psp, distance))
+	if (copy_in_user((void *)rval, (void *)psp, distance))
 		rval = 0;
 	else if (current->thread.flags & SPARC_FLAG_32BIT) {
 		if (put_user(((u32)csp), &(((struct reg_window32 *)rval)->ins[6])))
diff -Nru a/arch/sparc64/kernel/signal.c b/arch/sparc64/kernel/signal.c
--- a/arch/sparc64/kernel/signal.c	2003-06-29 15:00:05 -07:00
+++ b/arch/sparc64/kernel/signal.c	2004-08-24 17:28:32 -07:00
@@ -116,8 +116,8 @@
 	regs->tnpc = npc;
 	err |= __get_user(regs->y, &((*grp)[MC_Y]));
 	err |= __get_user(tstate, &((*grp)[MC_TSTATE]));
-	regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC);
-	regs->tstate |= (tstate & (TSTATE_ICC | TSTATE_XCC));
+	regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC);
+	regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC));
 	err |= __get_user(regs->u_regs[UREG_G1], (&(*grp)[MC_G1]));
 	err |= __get_user(regs->u_regs[UREG_G2], (&(*grp)[MC_G2]));
 	err |= __get_user(regs->u_regs[UREG_G3], (&(*grp)[MC_G3]));
@@ -437,9 +437,9 @@
 	err |= __get_user(tstate, &sf->regs.tstate);
 	err |= copy_from_user(regs->u_regs, sf->regs.u_regs, sizeof(regs->u_regs));
 
-	/* User can only change condition codes in %tstate. */
-	regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC);
-	regs->tstate |= (tstate & (TSTATE_ICC | TSTATE_XCC));
+	/* User can only change condition codes and %asi in %tstate. */
+	regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC);
+	regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC));
 
 	err |= __get_user(fpu_save, &sf->fpu_save);
 	if (fpu_save)
diff -Nru a/arch/sparc64/kernel/signal32.c b/arch/sparc64/kernel/signal32.c
--- a/arch/sparc64/kernel/signal32.c	2004-03-26 21:08:52 -08:00
+++ b/arch/sparc64/kernel/signal32.c	2004-08-24 17:28:32 -07:00
@@ -62,6 +62,16 @@
 	unsigned extramask[_NSIG_WORDS32 - 1];
 };
 
+/* This magic should be in g_upper[0] for all upper parts
+ * to be valid.
+ */
+#define SIGINFO_EXTRA_V8PLUS_MAGIC	0x130e269
+typedef struct {
+	unsigned int g_upper[8];
+	unsigned int o_upper[8];
+	unsigned int asi;
+} siginfo_extra_v8plus_t;
+
 /* 
  * And the new one, intended to be used for Linux applications only
  * (we have enough in there to work with clone).
@@ -292,8 +302,13 @@
 	if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) {
 		err |= __get_user(i, &sf->v8plus.g_upper[0]);
 		if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) {
+			unsigned long asi;
+
 			for (i = UREG_G1; i <= UREG_I7; i++)
 				err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+			err |= __get_user(asi, &sf->v8plus.asi);
+			regs->tstate &= ~TSTATE_ASI;
+			regs->tstate |= ((asi & 0xffUL) << 24UL);
 		}
 	}
 
@@ -431,8 +446,13 @@
 	if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) {
 		err |= __get_user(i, &sf->v8plus.g_upper[0]);
 		if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) {
+			unsigned long asi;
+
 			for (i = UREG_G1; i <= UREG_I7; i++)
 				err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+			err |= __get_user(asi, &sf->v8plus.asi);
+			regs->tstate &= ~TSTATE_ASI;
+			regs->tstate |= ((asi & 0xffUL) << 24UL);
 		}
 	}
 
@@ -728,7 +748,10 @@
 	err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size);
 	err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]);
 	for (i = 1; i < 16; i++)
-		err |= __put_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+		err |= __put_user(((u32 *)regs->u_regs)[2*i],
+				  &sf->v8plus.g_upper[i]);
+	err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL,
+			  &sf->v8plus.asi);
 
 	if (psr & PSR_EF) {
 		err |= save_fpu_state32(regs, &sf->fpu_state);
@@ -1175,7 +1198,10 @@
 	err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size);
 	err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]);
 	for (i = 1; i < 16; i++)
-		err |= __put_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+		err |= __put_user(((u32 *)regs->u_regs)[2*i],
+				  &sf->v8plus.g_upper[i]);
+	err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL,
+			  &sf->v8plus.asi);
 
 	if (psr & PSR_EF) {
 		err |= save_fpu_state32(regs, &sf->fpu_state);
diff -Nru a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
--- a/arch/sparc64/kernel/sparc64_ksyms.c	2004-07-27 13:08:37 -07:00
+++ b/arch/sparc64/kernel/sparc64_ksyms.c	2004-08-26 14:25:45 -07:00
@@ -324,7 +324,6 @@
 #endif
 
 /* Special internal versions of library functions. */
-EXPORT_SYMBOL(__memset);
 EXPORT_SYMBOL(_clear_page);
 EXPORT_SYMBOL(clear_user_page);
 EXPORT_SYMBOL(copy_user_page);
@@ -333,14 +332,17 @@
 EXPORT_SYMBOL(__memscan_generic);
 EXPORT_SYMBOL(__memcmp);
 EXPORT_SYMBOL(__strncmp);
-EXPORT_SYMBOL(__memmove);
+EXPORT_SYMBOL(__memset);
 
 EXPORT_SYMBOL(csum_partial_copy_sparc64);
 
 /* Moving data to/from userspace. */
-EXPORT_SYMBOL(__copy_to_user);
-EXPORT_SYMBOL(__copy_from_user);
-EXPORT_SYMBOL(__strncpy_from_user);
+EXPORT_SYMBOL(___copy_to_user);
+EXPORT_SYMBOL(___copy_from_user);
+EXPORT_SYMBOL(___copy_in_user);
+EXPORT_SYMBOL(copy_to_user_fixup);
+EXPORT_SYMBOL(copy_from_user_fixup);
+EXPORT_SYMBOL(copy_in_user_fixup);
 EXPORT_SYMBOL(__bzero_noasi);
 
 /* Various address conversion macros use this. */
diff -Nru a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c
--- a/arch/sparc64/kernel/time.c	2003-07-23 08:51:40 -07:00
+++ b/arch/sparc64/kernel/time.c	2004-08-26 14:57:34 -07:00
@@ -432,18 +432,12 @@
 		extern int _stext;
 		extern int rwlock_impl_begin, rwlock_impl_end;
 		extern int atomic_impl_begin, atomic_impl_end;
-		extern int __memcpy_begin, __memcpy_end;
-		extern int __bzero_begin, __bzero_end;
 		extern int __bitops_begin, __bitops_end;
 
 		if ((pc >= (unsigned long) &atomic_impl_begin &&
 		     pc < (unsigned long) &atomic_impl_end) ||
 		    (pc >= (unsigned long) &rwlock_impl_begin &&
 		     pc < (unsigned long) &rwlock_impl_end) ||
-		    (pc >= (unsigned long) &__memcpy_begin &&
-		     pc < (unsigned long) &__memcpy_end) ||
-		    (pc >= (unsigned long) &__bzero_begin &&
-		     pc < (unsigned long) &__bzero_end) ||
 		    (pc >= (unsigned long) &__bitops_begin &&
 		     pc < (unsigned long) &__bitops_end))
 			pc = o7;
diff -Nru a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
--- a/arch/sparc64/lib/Makefile	2002-05-08 23:18:22 -07:00
+++ b/arch/sparc64/lib/Makefile	2004-08-26 14:25:45 -07:00
@@ -8,9 +8,11 @@
 L_TARGET = lib.a
 obj-y := PeeCeeI.o blockops.o debuglocks.o strlen.o strncmp.o \
 	 memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \
-	 VIScopy.o VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
+	 VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
 	 VIScsumcopyusr.o VISsave.o atomic.o rwlock.o bitops.o \
-	 dec_and_lock.o U3memcpy.o U3copy_from_user.o U3copy_to_user.o \
-	 U3copy_in_user.o mcount.o
+	 U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
+	 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
+	 copy_in_user.o user_fixup.o memmove.o \
+	 dec_and_lock.o mcount.o
 
 include $(TOPDIR)/Rules.make
diff -Nru a/arch/sparc64/lib/U1copy_from_user.S b/arch/sparc64/lib/U1copy_from_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1copy_from_user.S	2004-08-26 14:25:45 -07:00
@@ -0,0 +1,33 @@
+/* U1copy_from_user.S: UltraSparc-I/II/IIi/IIe optimized copy from userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#define EX_LD(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		___copy_from_user
+#define LOAD(type,addr,dest)	type##a [addr] %asi, dest
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_AIUS, dest
+#define EX_RETVAL(x)		0
+
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
+
+#include "U1memcpy.S"
diff -Nru a/arch/sparc64/lib/U1copy_to_user.S b/arch/sparc64/lib/U1copy_to_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1copy_to_user.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,33 @@
+/* U1copy_to_user.S: UltraSparc-I/II/IIi/IIe optimized copy to userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#define EX_ST(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		___copy_to_user
+#define STORE(type,src,addr)	type##a src, [addr] ASI_AIUS
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_AIUS
+#define EX_RETVAL(x)		0
+
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
+
+#include "U1memcpy.S"
diff -Nru a/arch/sparc64/lib/U1memcpy.S b/arch/sparc64/lib/U1memcpy.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1memcpy.S	2004-08-26 14:25:45 -07:00
@@ -0,0 +1,552 @@
+/* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
+ *
+ * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+
+#ifdef __KERNEL__
+#include <asm/visasm.h>
+#include <asm/asi.h>
+#else
+#define ASI_BLK_P 0xf0
+#define FPRS_FEF  0x04
+#ifdef MEMCPY_DEBUG
+#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
+		 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#else
+#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
+#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#endif
+#endif
+
+#ifndef EX_LD
+#define EX_LD(x)	x
+#endif
+
+#ifndef EX_ST
+#define EX_ST(x)	x
+#endif
+
+#ifndef EX_RETVAL
+#define EX_RETVAL(x)	x
+#endif
+
+#ifndef LOAD
+#define LOAD(type,addr,dest)	type [addr], dest
+#endif
+
+#ifndef LOAD_BLK
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_P, dest
+#endif
+
+#ifndef STORE
+#define STORE(type,src,addr)	type src, [addr]
+#endif
+
+#ifndef STORE_BLK
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
+#endif
+
+#ifndef FUNC_NAME
+#define FUNC_NAME	memcpy
+#endif
+
+#ifndef PREAMBLE
+#define PREAMBLE
+#endif
+
+#ifndef XCC
+#define XCC xcc
+#endif
+
+#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
+	faligndata		%f1, %f2, %f48;			\
+	faligndata		%f2, %f3, %f50;			\
+	faligndata		%f3, %f4, %f52;			\
+	faligndata		%f4, %f5, %f54;			\
+	faligndata		%f5, %f6, %f56;			\
+	faligndata		%f6, %f7, %f58;			\
+	faligndata		%f7, %f8, %f60;			\
+	faligndata		%f8, %f9, %f62;
+
+#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt)	\
+	EX_LD(LOAD_BLK(%src, %fdest));				\
+	EX_ST(STORE_BLK(%fsrc, %dest));				\
+	add			%src, 0x40, %src;		\
+	subcc			%len, 0x40, %len;		\
+	be,pn			%xcc, jmptgt;			\
+	 add			%dest, 0x40, %dest;		\
+
+#define LOOP_CHUNK1(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f0,  f48, len, branch_dest)
+#define LOOP_CHUNK2(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
+#define LOOP_CHUNK3(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
+
+#define STORE_SYNC(dest, fsrc)				\
+	EX_ST(STORE_BLK(%fsrc, %dest));			\
+	add			%dest, 0x40, %dest;
+
+#define STORE_JUMP(dest, fsrc, target)			\
+	EX_ST(STORE_BLK(%fsrc, %dest));			\
+	add			%dest, 0x40, %dest;	\
+	ba,pt			%xcc, target;
+
+#define FINISH_VISCHUNK(dest, f0, f1, left)	\
+	subcc			%left, 8, %left;\
+	bl,pn			%xcc, 95f;	\
+	 faligndata		%f0, %f1, %f48;	\
+	EX_ST(STORE(std, %f48, %dest));		\
+	add			%dest, 8, %dest;
+
+#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
+	subcc			%left, 8, %left;	\
+	bl,pn			%xcc, 95f;		\
+	 fsrc1			%f0, %f1;
+
+#define UNEVEN_VISCHUNK(dest, f0, f1, left)		\
+	UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
+	ba,a,pt			%xcc, 93f;
+
+	.register	%g2,#scratch
+	.register	%g3,#scratch
+
+	.text
+	.align		64
+
+	.globl		FUNC_NAME
+FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+	PREAMBLE
+	mov		%o0, %g5
+	cmp		%o2, 0
+	be,pn		%XCC, 85f
+	 or		%o0, %o1, %o3
+	cmp		%o2, 16
+	blu,a,pn	%XCC, 80f
+	 or		%o3, %o2, %o3
+
+	cmp		%o2, (5 * 64)
+	blu,pt		%XCC, 70f
+	 andcc		%o3, 0x7, %g0
+
+	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  */
+	VISEntry
+
+	/* Is 'dst' already aligned on an 64-byte boundary? */
+	andcc		%o0, 0x3f, %g2
+	be,pt		%XCC, 2f
+
+	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
+	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
+	 * subtract this from 'len'.
+	 */
+	 sub		%o0, %o1, %o4
+	sub		%g2, 0x40, %g2
+	sub		%g0, %g2, %g2
+	sub		%o2, %g2, %o2
+	andcc		%g2, 0x7, %g1
+	be,pt		%icc, 2f
+	 and		%g2, 0x38, %g2
+
+1:	subcc		%g1, 0x1, %g1
+	EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
+	EX_ST(STORE(stb, %o3, %o1 + %o4))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x1, %o1
+
+	add		%o1, %o4, %o0
+
+2:	cmp		%g2, 0x0
+	and		%o1, 0x7, %g1
+	be,pt		%icc, 3f
+	 alignaddr	%o1, %g0, %o1
+
+	EX_LD(LOAD(ldd, %o1, %f4))
+1:	EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f4, %f6, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	be,pn		%icc, 3f
+	 add		%o0, 0x8, %o0
+
+	EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f6, %f4, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	bne,pt		%icc, 1b
+	 add		%o0, 0x8, %o0
+
+	/* Destination is 64-byte aligned.  */
+3:	
+	membar		  #LoadStore | #StoreStore | #StoreLoad
+
+	subcc		%o2, 0x40, %o4
+	add		%o1, %g1, %g1
+	andncc		%o4, (0x40 - 1), %o4
+	srl		%g1, 3, %g2
+	sub		%o2, %o4, %g3
+	andn		%o1, (0x40 - 1), %o1
+	and		%g2, 7, %g2
+	andncc		%g3, 0x7, %g3
+	fmovd		%f0, %f2
+	sub		%g3, 0x8, %g3
+	sub		%o2, %o4, %o2
+
+	add		%g1, %o4, %g1
+	subcc		%o2, %g3, %o2
+
+	EX_LD(LOAD_BLK(%o1, %f0))
+	add		%o1, 0x40, %o1
+	add		%g1, %g3, %g1
+	EX_LD(LOAD_BLK(%o1, %f16))
+	add		%o1, 0x40, %o1
+	sub		%o4, 0x80, %o4
+	EX_LD(LOAD_BLK(%o1, %f32))
+	add		%o1, 0x40, %o1
+
+	/* There are 8 instances of the unrolled loop,
+	 * one for each possible alignment of the
+	 * source buffer.  Each loop instance is 452
+	 * bytes.
+	 */
+	sll		%g2, 3, %o3
+	sub		%o3, %g2, %o3
+	sllx		%o3, 4, %o3
+	add		%o3, %g2, %o3
+	sllx		%o3, 2, %g2
+1:	rd		%pc, %o3
+	add		%o3, %lo(1f - 1b), %o3
+	jmpl		%o3 + %g2, %g0
+	 nop
+
+	.align		64
+1:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f0, %f2, %f48
+1:	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	STORE_JUMP(o0, f48, 40f) membar #Sync
+2:	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	STORE_JUMP(o0, f48, 48f) membar #Sync
+3:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	STORE_JUMP(o0, f48, 56f) membar #Sync
+
+1:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f2, %f4, %f48
+1:	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	STORE_JUMP(o0, f48, 41f) membar #Sync
+2:	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	STORE_JUMP(o0, f48, 49f) membar #Sync
+3:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	STORE_JUMP(o0, f48, 57f) membar #Sync
+
+1:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f4, %f6, %f48
+1:	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	STORE_JUMP(o0, f48, 42f) membar #Sync
+2:	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	STORE_JUMP(o0, f48, 50f) membar #Sync
+3:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	STORE_JUMP(o0, f48, 58f) membar #Sync
+
+1:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f6, %f8, %f48
+1:	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
+	STORE_JUMP(o0, f48, 43f) membar #Sync
+2:	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	STORE_JUMP(o0, f48, 51f) membar #Sync
+3:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	STORE_JUMP(o0, f48, 59f) membar #Sync
+
+1:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f8, %f10, %f48
+1:	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	STORE_JUMP(o0, f48, 44f) membar #Sync
+2:	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	STORE_JUMP(o0, f48, 52f) membar #Sync
+3:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	STORE_JUMP(o0, f48, 60f) membar #Sync
+
+1:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f10, %f12, %f48
+1:	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	STORE_JUMP(o0, f48, 45f) membar #Sync
+2:	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	STORE_JUMP(o0, f48, 53f) membar #Sync
+3:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	STORE_JUMP(o0, f48, 61f) membar #Sync
+
+1:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f12, %f14, %f48
+1:	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	STORE_JUMP(o0, f48, 46f) membar #Sync
+2:	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	STORE_JUMP(o0, f48, 54f) membar #Sync
+3:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	STORE_JUMP(o0, f48, 62f) membar #Sync
+
+1:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f14, %f16, %f48
+1:	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	STORE_JUMP(o0, f48, 47f) membar #Sync
+2:	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	STORE_JUMP(o0, f48, 55f) membar #Sync
+3:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	STORE_JUMP(o0, f48, 63f) membar #Sync
+
+40:	FINISH_VISCHUNK(o0, f0,  f2,  g3)
+41:	FINISH_VISCHUNK(o0, f2,  f4,  g3)
+42:	FINISH_VISCHUNK(o0, f4,  f6,  g3)
+43:	FINISH_VISCHUNK(o0, f6,  f8,  g3)
+44:	FINISH_VISCHUNK(o0, f8,  f10, g3)
+45:	FINISH_VISCHUNK(o0, f10, f12, g3)
+46:	FINISH_VISCHUNK(o0, f12, f14, g3)
+47:	UNEVEN_VISCHUNK(o0, f14, f0,  g3)
+48:	FINISH_VISCHUNK(o0, f16, f18, g3)
+49:	FINISH_VISCHUNK(o0, f18, f20, g3)
+50:	FINISH_VISCHUNK(o0, f20, f22, g3)
+51:	FINISH_VISCHUNK(o0, f22, f24, g3)
+52:	FINISH_VISCHUNK(o0, f24, f26, g3)
+53:	FINISH_VISCHUNK(o0, f26, f28, g3)
+54:	FINISH_VISCHUNK(o0, f28, f30, g3)
+55:	UNEVEN_VISCHUNK(o0, f30, f0,  g3)
+56:	FINISH_VISCHUNK(o0, f32, f34, g3)
+57:	FINISH_VISCHUNK(o0, f34, f36, g3)
+58:	FINISH_VISCHUNK(o0, f36, f38, g3)
+59:	FINISH_VISCHUNK(o0, f38, f40, g3)
+60:	FINISH_VISCHUNK(o0, f40, f42, g3)
+61:	FINISH_VISCHUNK(o0, f42, f44, g3)
+62:	FINISH_VISCHUNK(o0, f44, f46, g3)
+63:	UNEVEN_VISCHUNK_LAST(o0, f46, f0,  g3)
+
+93:	EX_LD(LOAD(ldd, %o1, %f2))
+	add		%o1, 8, %o1
+	subcc		%g3, 8, %g3
+	faligndata	%f0, %f2, %f8
+	EX_ST(STORE(std, %f8, %o0))
+	bl,pn		%xcc, 95f
+	 add		%o0, 8, %o0
+	EX_LD(LOAD(ldd, %o1, %f0))
+	add		%o1, 8, %o1
+	subcc		%g3, 8, %g3
+	faligndata	%f2, %f0, %f8
+	EX_ST(STORE(std, %f8, %o0))
+	bge,pt		%xcc, 93b
+	 add		%o0, 8, %o0
+
+95:	brz,pt		%o2, 2f
+	 mov		%g1, %o1
+
+1:	EX_LD(LOAD(ldub, %o1, %o3))
+	add		%o1, 1, %o1
+	subcc		%o2, 1, %o2
+	EX_ST(STORE(stb, %o3, %o0))
+	bne,pt		%xcc, 1b
+	 add		%o0, 1, %o0
+
+2:	membar		#StoreLoad | #StoreStore
+	VISExit
+	retl
+	 mov		EX_RETVAL(%g5), %o0
+
+	.align		64
+70:	/* 16 < len <= (5 * 64) */
+	bne,pn		%XCC, 75f
+	 sub		%o0, %o1, %o3
+
+72:	andn		%o2, 0xf, %o4
+	and		%o2, 0xf, %o2
+1:	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
+	subcc		%o4, 0x10, %o4
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+	EX_ST(STORE(stx, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x8, %o1
+73:	andcc		%o2, 0x8, %g0
+	be,pt		%XCC, 1f
+	 nop
+	EX_LD(LOAD(ldx, %o1, %o5))
+	sub		%o2, 0x8, %o2
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+1:	andcc		%o2, 0x4, %g0
+	be,pt		%XCC, 1f
+	 nop
+	EX_LD(LOAD(lduw, %o1, %o5))
+	sub		%o2, 0x4, %o2
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
+	add		%o1, 0x4, %o1
+1:	cmp		%o2, 0
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
+	 nop
+
+75:	andcc		%o0, 0x7, %g1
+	sub		%g1, 0x8, %g1
+	be,pn		%icc, 2f
+	 sub		%g0, %g1, %g1
+	sub		%o2, %g1, %o2
+
+1:	EX_LD(LOAD(ldub, %o1, %o5))
+	subcc		%g1, 1, %g1
+	EX_ST(STORE(stb, %o5, %o1 + %o3))
+	bgu,pt		%icc, 1b
+	 add		%o1, 1, %o1
+
+2:	add		%o1, %o3, %o0
+	andcc		%o1, 0x7, %g1
+	bne,pt		%icc, 8f
+	 sll		%g1, 3, %g1
+
+	cmp		%o2, 16
+	bgeu,pt		%icc, 72b
+	 nop
+	ba,a,pt		%xcc, 73b
+
+8:	mov		64, %o3
+	andn		%o1, 0x7, %o1
+	EX_LD(LOAD(ldx, %o1, %g2))
+	sub		%o3, %g1, %o3
+	andn		%o2, 0x7, %o4
+	sllx		%g2, %g1, %g2
+1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
+	subcc		%o4, 0x8, %o4
+	add		%o1, 0x8, %o1
+	srlx		%g3, %o3, %o5
+	or		%o5, %g2, %o5
+	EX_ST(STORE(stx, %o5, %o0))
+	add		%o0, 0x8, %o0
+	bgu,pt		%icc, 1b
+	 sllx		%g3, %g1, %g2
+
+	srl		%g1, 3, %g1
+	andcc		%o2, 0x7, %o2
+	be,pn		%icc, 85f
+	 add		%o1, %g1, %o1
+	ba,pt		%xcc, 90f
+	 sub		%o0, %o1, %o3
+
+	.align		64
+80:	/* 0 < len <= 16 */
+	andcc		%o3, 0x3, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+1:	EX_LD(LOAD(lduw, %o1, %g1))
+	subcc		%o2, 4, %o2
+	EX_ST(STORE(stw, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 4, %o1
+
+85:	retl
+	 mov		EX_RETVAL(%g5), %o0
+
+	.align		32
+90:	EX_LD(LOAD(ldub, %o1, %g1))
+	subcc		%o2, 1, %o2
+	EX_ST(STORE(stb, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 90b
+	 add		%o1, 1, %o1
+	retl
+	 mov		EX_RETVAL(%g5), %o0
diff -Nru a/arch/sparc64/lib/U3copy_from_user.S b/arch/sparc64/lib/U3copy_from_user.S
--- a/arch/sparc64/lib/U3copy_from_user.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3copy_from_user.S	2004-08-26 14:25:45 -07:00
@@ -3,428 +3,20 @@
  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
  */
 
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
+#define EX_LD(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		U3copy_from_user
+#define LOAD(type,addr,dest)	type##a [addr] %asi, dest
+#define EX_RETVAL(x)		0
 
-#define XCC xcc
-
-#define EXNV_RAW(x,y,a,b)		\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	a, b, %o1;			\
-	ba U3cfu_fixup;			\
-	 add %o1, 4, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV8(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	a, b, %o1;			\
-	ba U3cfu_fixup;			\
-	 add %o1, 8, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX2(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	add %o1, %o4, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, 0x1c0, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX3(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	sll %g3, 6, %g3;		\
-	add %o1, 0x80, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, %g3, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX4(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	add %o1, 0x40, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, %g3, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	/* Special/non-trivial issues of this code:
-	 *
-	 * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
-	 * 2) Only low 32 FPU registers are used so that only the
-	 *    lower half of the FPU register set is dirtied by this
-	 *    code.  This is especially important in the kernel.
-	 * 3) This code never prefetches cachelines past the end
-	 *    of the source buffer.
-	 */
-
-	.text
-	.align	32
-
-	/* The cheetah's flexible spine, oversized liver, enlarged heart,
-	 * slender muscular body, and claws make it the swiftest hunter
-	 * in Africa and the fastest animal on land.  Can reach speeds
-	 * of up to 2.4GB per second.
-	 */
-
-	.globl	U3copy_from_user
-U3copy_from_user:	/* %o0=dst, %o1=src, %o2=len */
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
-	 andcc		%o3, 0x7, %g0
-
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
-	 */
-	.align		64
-enter:
-	/* Is 'dst' already aligned on an 64-byte boundary? */
-	be,pt		%XCC, 2f
-
-	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
-	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
-	 * subtract this from 'len'.
-	 */
-	 sub		%g2, 0x40, %g2
-	sub		%g0, %g2, %g2
-	sub		%o2, %g2, %o2
-
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	EXNV_RAW(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 stb		%o3, [%o0 + -1]
-
-2:	VISEntryHalf
-	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
-	 alignaddr	%o1, %g0, %o1
-
-	.align		64
-begin:
-
-	.globl		U3copy_from_user_nop_1_6
-U3copy_from_user_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-
-	prefetcha	[%o1 + 0x000] %asi, #one_read
-	prefetcha	[%o1 + 0x040] %asi, #one_read
-	andn		%o2, (0x40 - 1), %o4
-	prefetcha	[%o1 + 0x080] %asi, #one_read
-	prefetcha	[%o1 + 0x0c0] %asi, #one_read
-	EX(ldda [%o1 + 0x000] %asi, %f0, add %o2, %g0)
-	prefetcha	[%o1 + 0x100] %asi, #one_read
-	EX(ldda [%o1 + 0x008] %asi, %f2, add %o2, %g0)
-	prefetcha	[%o1 + 0x140] %asi, #one_read
-	EX(ldda [%o1 + 0x010] %asi, %f4, add %o2, %g0)
-	prefetcha	[%o1 + 0x180] %asi, #one_read
-	faligndata	%f0, %f2, %f16
-	EX(ldda [%o1 + 0x018] %asi, %f6, add %o2, %g0)
-	faligndata	%f2, %f4, %f18
-	EX(ldda [%o1 + 0x020] %asi, %f8, add %o2, %g0)
-	faligndata	%f4, %f6, %f20
-	EX(ldda [%o1 + 0x028] %asi, %f10, add %o2, %g0)
-	faligndata	%f6, %f8, %f22
-
-	EX(ldda [%o1 + 0x030] %asi, %f12, add %o2, %g0)
-	faligndata	%f8, %f10, %f24
-	EX(ldda [%o1 + 0x038] %asi, %f14, add %o2, %g0)
-	faligndata	%f10, %f12, %f26
-	EX(ldda [%o1 + 0x040] %asi, %f0, add %o2, %g0)
-
-	sub		%o4, 0x80, %o4
-	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
-	 srl		%o4, 6, %o3
-
-	.align		64
-loop:
-	EX3(ldda [%o1 + 0x008] %asi, %f2)
-	faligndata	%f12, %f14, %f28
-	EX3(ldda [%o1 + 0x010] %asi, %f4)
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	EX3(ldda [%o1 + 0x018] %asi, %f6)
-	faligndata	%f0, %f2, %f16
-
-	EX3(ldda [%o1 + 0x020] %asi, %f8)
-	faligndata	%f2, %f4, %f18
-	EX3(ldda [%o1 + 0x028] %asi, %f10)
-	faligndata	%f4, %f6, %f20
-	EX3(ldda [%o1 + 0x030] %asi, %f12)
-	faligndata	%f6, %f8, %f22
-	EX3(ldda [%o1 + 0x038] %asi, %f14)
-	faligndata	%f8, %f10, %f24
-
-	EX3(ldda [%o1 + 0x040] %asi, %f0)
-	prefetcha	[%o1 + 0x180] %asi, #one_read
-	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
-
-	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	EX3(ldda [%o1 + 0x008] %asi, %f2)
-	faligndata	%f12, %f14, %f28
-	EX3(ldda [%o1 + 0x010] %asi, %f4)
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	EX3(ldda [%o1 + 0x018] %asi, %f6)
-	faligndata	%f0, %f2, %f16
-	EX3(ldda [%o1 + 0x020] %asi, %f8)
-	faligndata	%f2, %f4, %f18
-	EX3(ldda [%o1 + 0x028] %asi, %f10)
-	faligndata	%f4, %f6, %f20
-	EX3(ldda [%o1 + 0x030] %asi, %f12)
-	faligndata	%f6, %f8, %f22
-	EX3(ldda [%o1 + 0x038] %asi, %f14)
-	faligndata	%f8, %f10, %f24
-	cmp		%g1, 0
-	be,pt		%XCC, 1f
-	 add		%o0, 0x40, %o0
-	EX4(ldda [%o1 + 0x040] %asi, %f0)
-1:	faligndata	%f10, %f12, %f26
-	faligndata	%f12, %f14, %f28
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	add		%o0, 0x40, %o0
-	add		%o1, 0x40, %o1
-
-	.globl		U3copy_from_user_nop_2_3
-U3copy_from_user_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-
-	membar		#Sync
-
-	/* Now we copy the (len modulo 64) bytes at the end.
-	 * Note how we borrow the %f0 loaded above.
-	 *
-	 * Also notice how this code is careful not to perform a
-	 * load past the end of the src buffer.
-	 */
-loopend:
-	and		%o2, 0x3f, %o2
-	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
-	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
-	 cmp		%g1, 0
-
-	be,a,pt		%XCC, 1f
-	 EX(ldda [%o1 + 0x00] %asi, %f0, add %o2, %g0)
-
-1:	EX(ldda [%o1 + 0x08] %asi, %f2, add %o2, %g0)
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f0, %f2, %f8
-	std		%f8, [%o0 + 0x00]
-	be,pn		%XCC, endcruft
-	 add		%o0, 0x8, %o0
-	EX(ldda [%o1 + 0x08] %asi, %f0, add %o2, %g0)
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f2, %f0, %f8
-	std		%f8, [%o0 + 0x00]
-	bne,pn		%XCC, 1b
-	 add		%o0, 0x8, %o0
-
-	/* If anything is left, we copy it one byte at a time.
-	 * Note that %g1 is (src & 0x3) saved above before the
-	 * alignaddr was performed.
-	 */
-endcruft:
-	cmp		%o2, 0
-	add		%o1, %g1, %o1
-	VISExitHalf
-	be,pn		%XCC, out
-	 sub		%o0, %o1, %o3
-
-	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
-	 andcc		%o2, 0x8, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(ldxa [%o1] %asi, %o5, add %o2, %g0)
-	stx		%o5, [%o1 + %o3]
-	add		%o1, 0x8, %o1
-
-1:	andcc		%o2, 0x4, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(lduwa [%o1] %asi, %o5, and %o2, 0x7)
-	stw		%o5, [%o1 + %o3]
-	add		%o1, 0x4, %o1
-
-1:	andcc		%o2, 0x2, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(lduha [%o1] %asi, %o5, and %o2, 0x3)
-	sth		%o5, [%o1 + %o3]
-	add		%o1, 0x2, %o1
-
-1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
-	 nop
-	EXNV(lduba [%o1] %asi, %o5, and %o2, 0x1)
-	ba,pt		%xcc, out
-	 stb		%o5, [%o1 + %o3]
-
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	EXNV8(ldxa [%o1] %asi, %o5, add %o2, %o4)
-	stx		%o5, [%o1 + %o3]
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	EXNV4(lduwa [%o1] %asi, %o5, add %o2, %g0)
-	stw		%o5, [%o1 + %o3]
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	EXNV(lduwa [%o1] %asi, %g1, add %o2, %g0)
-	stw		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
-
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	EXNV(lduba [%o1] %asi, %g1, add %o2, %g0)
-	stb		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
-
-U3cfu_fixup:
-	/* Since this is copy_from_user(), zero out the rest of the
-	 * kernel buffer.
-	 */
-	cmp		%o1, 0
-	ble,pn		%icc, 2f
-	 mov		%o1, %g2
-
-1:	subcc		%g2, 1, %g2
-	stb		%g0, [%o0]
-	bne,pt		%icc, 1b
-	 add		%o0, 1, %o0
-
-2:	retl
-	 mov		%o1, %o0
+#include "U3memcpy.S"
diff -Nru a/arch/sparc64/lib/U3copy_in_user.S b/arch/sparc64/lib/U3copy_in_user.S
--- a/arch/sparc64/lib/U3copy_in_user.S	2004-07-29 16:29:51 -07:00
+++ /dev/null	Wed Dec 31 16:00:00 196900
@@ -1,140 +0,0 @@
-/* U3copy_in_user.S: UltraSparc-III optimized memcpy.
- *
- * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
- */
-
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
-
-#define XCC xcc
-
-#define EXNV(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV1(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 1, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 4, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV8(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 8, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	.text
-	.align	32
-
-	/* Don't try to get too fancy here, just nice and
-	 * simple.  This is predominantly used for well aligned
-	 * small copies in the compat layer.  It is also used
-	 * to copy register windows around during thread cloning.
-	 */
-
-	.globl	U3copy_in_user
-U3copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
-	/* Writing to %asi is _expensive_ so we hardcode it.
-	 * Reading %asi to check for KERNEL_DS is comparatively
-	 * cheap.
-	 */
-	rd		%asi, %g1
-	cmp		%g1, ASI_AIUS
-	bne,pn		%icc, U3memcpy_user_stub
-	 nop
-
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-medium_copy: /* 16 < len <= 64 */
-	andcc		%o3, 0x7, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	EXNV8(ldxa [%o1] %asi, %o5, add %o4, %o2)
-	EXNV8(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	EXNV4(lduwa [%o1] %asi, %o5, add %o4, %o2)
-	EXNV4(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	EXNV4(lduwa [%o1] %asi, %g1, add %o2, %g0)
-	EXNV4(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
-
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	EXNV1(lduba [%o1] %asi, %g1, add %o2, %g0)
-	EXNV1(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
diff -Nru a/arch/sparc64/lib/U3copy_to_user.S b/arch/sparc64/lib/U3copy_to_user.S
--- a/arch/sparc64/lib/U3copy_to_user.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3copy_to_user.S	2004-08-26 14:25:45 -07:00
@@ -1,433 +1,33 @@
-/* U3copy_to_user.S: UltraSparc-III optimized memcpy.
+/* U3copy_to_user.S: UltraSparc-III optimized copy to userspace.
  *
  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
  */
 
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
+#define EX_ST(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		U3copy_to_user
+#define STORE(type,src,addr)	type##a src, [addr] ASI_AIUS
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_AIUS
+#define EX_RETVAL(x)		0
 
-#define XCC xcc
-
-#define EXNV(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV2(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 1, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV3(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 4, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 8, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK1(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	add %o4, 0x1c0, %o1;		\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o1, %o2, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK2(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	sll %o3, 6, %o3;		\
-	and %o2, (0x40 - 1), %o2;	\
-	add %o3, 0x80, %o1;		\
-	retl;				\
-	 add %o1, %o2, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK3(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o2, 0x80, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK4(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o2, 0x40, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	/* Special/non-trivial issues of this code:
-	 *
-	 * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
-	 * 2) Only low 32 FPU registers are used so that only the
-	 *    lower half of the FPU register set is dirtied by this
-	 *    code.  This is especially important in the kernel.
-	 * 3) This code never prefetches cachelines past the end
-	 *    of the source buffer.
-	 */
-
-	.text
-	.align	32
-
-	/* The cheetah's flexible spine, oversized liver, enlarged heart,
-	 * slender muscular body, and claws make it the swiftest hunter
-	 * in Africa and the fastest animal on land.  Can reach speeds
-	 * of up to 2.4GB per second.
-	 */
-
-	.globl	U3copy_to_user
-U3copy_to_user:	/* %o0=dst, %o1=src, %o2=len */
 	/* Writing to %asi is _expensive_ so we hardcode it.
 	 * Reading %asi to check for KERNEL_DS is comparatively
 	 * cheap.
 	 */
-	rd		%asi, %g1
-	cmp		%g1, ASI_AIUS
-	bne,pn		%icc, U3memcpy_user_stub
-	 nop
-
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
-	 andcc		%o3, 0x7, %g0
-
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
-	 */
-	.align		64
-enter:
-	/* Is 'dst' already aligned on an 64-byte boundary? */
-	be,pt		%XCC, 2f
-
-	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
-	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
-	 * subtract this from 'len'.
-	 */
-	 sub		%g2, 0x40, %g2
-	sub		%g0, %g2, %g2
-	sub		%o2, %g2, %o2
-
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	ldub		[%o1 + 0x00], %o3
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2)
-
-2:	VISEntryHalf
-	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
-	 alignaddr	%o1, %g0, %o1
-
-	.align		64
-begin:
-
-	.globl		U3copy_to_user_nop_1_6
-U3copy_to_user_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-
-	prefetch	[%o1 + 0x000], #one_read
-	prefetch	[%o1 + 0x040], #one_read
-	andn		%o2, (0x40 - 1), %o4
-	prefetch	[%o1 + 0x080], #one_read
-	prefetch	[%o1 + 0x0c0], #one_read
-	ldd		[%o1 + 0x000], %f0
-	prefetch	[%o1 + 0x100], #one_read
-	ldd		[%o1 + 0x008], %f2
-	prefetch	[%o1 + 0x140], #one_read
-	ldd		[%o1 + 0x010], %f4
-	prefetch	[%o1 + 0x180], #one_read
-	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f6, %f8, %f22
-
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f8, %f10, %f24
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f10, %f12, %f26
-	ldd		[%o1 + 0x040], %f0
-
-	sub		%o4, 0x80, %o4
-	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
-	 srl		%o4, 6, %o3
-
-	.align		64
-loop:
-	ldd		[%o1 + 0x008], %f2
-	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
-	faligndata	%f14, %f0, %f30
-	EXBLK2(stda %f16, [%o0] ASI_BLK_AIUS)
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f0, %f2, %f16
-
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
-
-	ldd		[%o1 + 0x040], %f0
-	prefetch	[%o1 + 0x180], #one_read
-	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
-
-	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	ldd		[%o1 + 0x008], %f2
-	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
-	faligndata	%f14, %f0, %f30
-	EXBLK3(stda %f16, [%o0] ASI_BLK_AIUS)
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
-	cmp		%g1, 0
-	be,pt		%XCC, 1f
-	 add		%o0, 0x40, %o0
-	ldd		[%o1 + 0x040], %f0
-1:	faligndata	%f10, %f12, %f26
-	faligndata	%f12, %f14, %f28
-	faligndata	%f14, %f0, %f30
-	EXBLK4(stda %f16, [%o0] ASI_BLK_AIUS)
-	add		%o0, 0x40, %o0
-	add		%o1, 0x40, %o1
-
-	.globl		U3copy_to_user_nop_2_3
-U3copy_to_user_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-
-	membar		#Sync
-
-	/* Now we copy the (len modulo 64) bytes at the end.
-	 * Note how we borrow the %f0 loaded above.
-	 *
-	 * Also notice how this code is careful not to perform a
-	 * load past the end of the src buffer.
-	 */
-loopend:
-	and		%o2, 0x3f, %o2
-	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
-	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
-	 cmp		%g1, 0
-
-	be,a,pt		%XCC, 1f
-	 ldd		[%o1 + 0x00], %f0
-
-1:	ldd		[%o1 + 0x08], %f2
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f0, %f2, %f8
-	EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
-	be,pn		%XCC, endcruft
-	 add		%o0, 0x8, %o0
-	ldd		[%o1 + 0x08], %f0
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f2, %f0, %f8
-	EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
-	bne,pn		%XCC, 1b
-	 add		%o0, 0x8, %o0
-
-	/* If anything is left, we copy it one byte at a time.
-	 * Note that %g1 is (src & 0x3) saved above before the
-	 * alignaddr was performed.
-	 */
-endcruft:
-	cmp		%o2, 0
-	add		%o1, %g1, %o1
-	VISExitHalf
-	be,pn		%XCC, out
-	 sub		%o0, %o1, %o3
-
-	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
-	 andcc		%o2, 0x8, %g0
-	be,pt		%icc, 1f
-	 nop
-	ldx		[%o1], %o5
-	EXNV(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	add		%o1, 0x8, %o1
-
-1:	andcc		%o2, 0x4, %g0
-	be,pt		%icc, 1f
-	 nop
-	lduw		[%o1], %o5
-	EXNV(stwa %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x7)
-	add		%o1, 0x4, %o1
-
-1:	andcc		%o2, 0x2, %g0
-	be,pt		%icc, 1f
-	 nop
-	lduh		[%o1], %o5
-	EXNV(stha %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x3)
-	add		%o1, 0x2, %o1
-
-1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
-	 nop
-	ldub		[%o1], %o5
-	ba,pt		%xcc, out
-	 EXNV(stba %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x1)
-
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	ldx		[%o1], %o5
-	EXNV4(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %o4)
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	lduw		[%o1], %o5
-	EXNV3(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	lduw		[%o1], %g1
-	EXNV3(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
 
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	ldub		[%o1], %g1
-	EXNV2(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
+#include "U3memcpy.S"
diff -Nru a/arch/sparc64/lib/U3memcpy.S b/arch/sparc64/lib/U3memcpy.S
--- a/arch/sparc64/lib/U3memcpy.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3memcpy.S	2004-08-26 14:25:45 -07:00
@@ -6,14 +6,50 @@
 #ifdef __KERNEL__
 #include <asm/visasm.h>
 #include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
 #else
 #define ASI_BLK_P 0xf0
 #define FPRS_FEF  0x04
+#ifdef MEMCPY_DEBUG
+#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
+		     clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#else
 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
 #endif
+#endif
+
+#ifndef EX_LD
+#define EX_LD(x)	x
+#endif
+
+#ifndef EX_ST
+#define EX_ST(x)	x
+#endif
+
+#ifndef EX_RETVAL
+#define EX_RETVAL(x)	x
+#endif
+
+#ifndef LOAD
+#define LOAD(type,addr,dest)	type [addr], dest
+#endif
+
+#ifndef STORE
+#define STORE(type,src,addr)	type src, [addr]
+#endif
+
+#ifndef STORE_BLK
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
+#endif
+
+#ifndef FUNC_NAME
+#define FUNC_NAME	U3memcpy
+#endif
+
+#ifndef PREAMBLE
+#define PREAMBLE
+#endif
 
 #ifndef XCC
 #define XCC xcc
@@ -33,7 +69,7 @@
 	 */
 
 	.text
-	.align	32
+	.align		64
 
 	/* The cheetah's flexible spine, oversized liver, enlarged heart,
 	 * slender muscular body, and claws make it the swiftest hunter
@@ -41,156 +77,159 @@
 	 * of up to 2.4GB per second.
 	 */
 
-	.globl	U3memcpy
-U3memcpy:	/* %o0=dst, %o1=src, %o2=len */
+	.globl	FUNC_NAME
+FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+	PREAMBLE
 	mov		%o0, %g5
 	cmp		%o2, 0
-	be,pn		%XCC, out
+	be,pn		%XCC, 85f
 	 or		%o0, %o1, %o3
 	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
+	blu,a,pn	%XCC, 80f
 	 or		%o3, %o2, %o3
 
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
+	cmp		%o2, (3 * 64)
+	blu,pt		%XCC, 70f
 	 andcc		%o3, 0x7, %g0
 
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
+	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  We must preserve
+	 * o5 from here until we hit VISExitHalf.
 	 */
-	.align		64
-enter:
+	VISEntryHalf
+
 	/* Is 'dst' already aligned on an 64-byte boundary? */
+	andcc		%o0, 0x3f, %g2
 	be,pt		%XCC, 2f
 
 	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
 	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
 	 * subtract this from 'len'.
 	 */
-	 sub		%g2, 0x40, %g2
+	 sub		%o0, %o1, %o4
+	sub		%g2, 0x40, %g2
 	sub		%g0, %g2, %g2
 	sub		%o2, %g2, %o2
+	andcc		%g2, 0x7, %g1
+	be,pt		%icc, 2f
+	 and		%g2, 0x38, %g2
+
+1:	subcc		%g1, 0x1, %g1
+	EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
+	EX_ST(STORE(stb, %o3, %o1 + %o4))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x1, %o1
 
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	ldub		[%o1 + 0x00], %o3
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 stb		%o3, [%o0 + -1]
+	add		%o1, %o4, %o0
 
-2:	VISEntryHalf
+2:	cmp		%g2, 0x0
 	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
+	be,pt		%icc, 3f
 	 alignaddr	%o1, %g0, %o1
 
-	.align		64
-begin:
-#ifdef __KERNEL__
-	.globl		U3memcpy_nop_1_6
-U3memcpy_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-#endif
-	prefetch	[%o1 + 0x000], #one_read
-	prefetch	[%o1 + 0x040], #one_read
+	EX_LD(LOAD(ldd, %o1, %f4))
+1:	EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f4, %f6, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	be,pn		%icc, 3f
+	 add		%o0, 0x8, %o0
+
+	EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f6, %f4, %f2
+	EX_ST(STORE(std, %f2, %o0))
+	bne,pt		%icc, 1b
+	 add		%o0, 0x8, %o0
+
+3:	LOAD(prefetch, %o1 + 0x000, #one_read)
+	LOAD(prefetch, %o1 + 0x040, #one_read)
 	andn		%o2, (0x40 - 1), %o4
-	prefetch	[%o1 + 0x080], #one_read
-	prefetch	[%o1 + 0x0c0], #one_read
-	ldd		[%o1 + 0x000], %f0
-	prefetch	[%o1 + 0x100], #one_read
-	ldd		[%o1 + 0x008], %f2
-	prefetch	[%o1 + 0x140], #one_read
-	ldd		[%o1 + 0x010], %f4
-	prefetch	[%o1 + 0x180], #one_read
+	LOAD(prefetch, %o1 + 0x080, #one_read)
+	LOAD(prefetch, %o1 + 0x0c0, #one_read)
+	LOAD(prefetch, %o1 + 0x100, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x000, %f0))
+	LOAD(prefetch, %o1 + 0x140, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
+	LOAD(prefetch, %o1 + 0x180, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
+	LOAD(prefetch, %o1 + 0x1c0, #one_read)
 	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x018], %f6
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f6, %f8, %f22
 
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
 	faligndata	%f8, %f10, %f24
-	ldd		[%o1 + 0x038], %f14
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 	faligndata	%f10, %f12, %f26
-	ldd		[%o1 + 0x040], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
 
-	sub		%o4, 0x80, %o4
+	subcc		%o4, 0x80, %o4
 	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
+	bgu,pt		%XCC, 1f
 	 srl		%o4, 6, %o3
+	ba,pt		%xcc, 2f
+	 nop
 
 	.align		64
-loop:
-	ldd		[%o1 + 0x008], %f2
+1:
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
 	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	ldd		[%o1 + 0x018], %f6
+	EX_ST(STORE_BLK(%f16, %o0))
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f0, %f2, %f16
+	add		%o0, 0x40, %o0
 
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
+	subcc		%o3, 0x01, %o3
 	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 
-	ldd		[%o1 + 0x040], %f0
-	prefetch	[%o1 + 0x180], #one_read
+	faligndata	%f8, %f10, %f24
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
+	LOAD(prefetch, %o1 + 0x1c0, #one_read)
 	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
+	bg,pt		%XCC, 1b
+	 add		%o1, 0x40, %o1
 
 	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	ldd		[%o1 + 0x008], %f2
+2:
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
 	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	ldd		[%o1 + 0x018], %f6
+	EX_ST(STORE_BLK(%f16, %o0))
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
 	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 	faligndata	%f8, %f10, %f24
 	cmp		%g1, 0
 	be,pt		%XCC, 1f
 	 add		%o0, 0x40, %o0
-	ldd		[%o1 + 0x040], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
 1:	faligndata	%f10, %f12, %f26
 	faligndata	%f12, %f14, %f28
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
+	EX_ST(STORE_BLK(%f16, %o0))
 	add		%o0, 0x40, %o0
 	add		%o1, 0x40, %o1
-#ifdef __KERNEL__
-	.globl		U3memcpy_nop_2_3
-U3memcpy_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-#endif
 	membar		#Sync
 
 	/* Now we copy the (len modulo 64) bytes at the end.
@@ -199,31 +238,29 @@
 	 * Also notice how this code is careful not to perform a
 	 * load past the end of the src buffer.
 	 */
-loopend:
 	and		%o2, 0x3f, %o2
 	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
+	be,pn		%XCC, 2f
 	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
+	be,pn		%XCC, 2f
 	 cmp		%g1, 0
 
+	sub		%o2, %g2, %o2
 	be,a,pt		%XCC, 1f
-	 ldd		[%o1 + 0x00], %f0
+	 EX_LD(LOAD(ldd, %o1 + 0x00, %f0))
 
-1:	ldd		[%o1 + 0x08], %f2
+1:	EX_LD(LOAD(ldd, %o1 + 0x08, %f2))
 	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
 	subcc		%g2, 0x8, %g2
 	faligndata	%f0, %f2, %f8
-	std		%f8, [%o0 + 0x00]
-	be,pn		%XCC, endcruft
+	EX_ST(STORE(std, %f8, %o0))
+	be,pn		%XCC, 2f
 	 add		%o0, 0x8, %o0
-	ldd		[%o1 + 0x08], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x08, %f0))
 	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
 	subcc		%g2, 0x8, %g2
 	faligndata	%f2, %f0, %f8
-	std		%f8, [%o0 + 0x00]
+	EX_ST(STORE(std, %f8, %o0))
 	bne,pn		%XCC, 1b
 	 add		%o0, 0x8, %o0
 
@@ -231,103 +268,147 @@
 	 * Note that %g1 is (src & 0x3) saved above before the
 	 * alignaddr was performed.
 	 */
-endcruft:
+2:
 	cmp		%o2, 0
 	add		%o1, %g1, %o1
 	VISExitHalf
-	be,pn		%XCC, out
+	be,pn		%XCC, 85f
 	 sub		%o0, %o1, %o3
 
 	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
+	bne,pn		%icc, 90f
 	 andcc		%o2, 0x8, %g0
 	be,pt		%icc, 1f
 	 nop
-	ldx		[%o1], %o5
-	stx		%o5, [%o1 + %o3]
+	EX_LD(LOAD(ldx, %o1, %o5))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
 	add		%o1, 0x8, %o1
 
 1:	andcc		%o2, 0x4, %g0
 	be,pt		%icc, 1f
 	 nop
-	lduw		[%o1], %o5
-	stw		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduw, %o1, %o5))
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
 	add		%o1, 0x4, %o1
 
 1:	andcc		%o2, 0x2, %g0
 	be,pt		%icc, 1f
 	 nop
-	lduh		[%o1], %o5
-	sth		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduh, %o1, %o5))
+	EX_ST(STORE(sth, %o5, %o1 + %o3))
 	add		%o1, 0x2, %o1
 
 1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
+	be,pt		%icc, 85f
 	 nop
-	ldub		[%o1], %o5
-	ba,pt		%xcc, out
-	 stb		%o5, [%o1 + %o3]
+	EX_LD(LOAD(ldub, %o1, %o5))
+	ba,pt		%xcc, 85f
+	 EX_ST(STORE(stb, %o5, %o1 + %o3))
 
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
+	.align		64
+70: /* 16 < len <= 64 */
+	bne,pn		%XCC, 75f
 	 sub		%o0, %o1, %o3
 
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	ldx		[%o1], %o5
-	stx		%o5, [%o1 + %o3]
+72:
+	andn		%o2, 0xf, %o4
+	and		%o2, 0xf, %o2
+1:	subcc		%o4, 0x10, %o4
+	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+	EX_ST(STORE(stx, %g1, %o1 + %o3))
 	bgu,pt		%XCC, 1b
 	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
+73:	andcc		%o2, 0x8, %g0
+	be,pt		%XCC, 1f
+	 nop
+	sub		%o2, 0x8, %o2
+	EX_LD(LOAD(ldx, %o1, %o5))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+1:	andcc		%o2, 0x4, %g0
 	be,pt		%XCC, 1f
 	 nop
 	sub		%o2, 0x4, %o2
-	lduw		[%o1], %o5
-	stw		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduw, %o1, %o5))
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
 	add		%o1, 0x4, %o1
 1:	cmp		%o2, 0
-	be,pt		%XCC, out
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
 	 nop
-	ba,pt		%xcc, small_copy_unaligned
+
+75:
+	andcc		%o0, 0x7, %g1
+	sub		%g1, 0x8, %g1
+	be,pn		%icc, 2f
+	 sub		%g0, %g1, %g1
+	sub		%o2, %g1, %o2
+
+1:	subcc		%g1, 1, %g1
+	EX_LD(LOAD(ldub, %o1, %o5))
+	EX_ST(STORE(stb, %o5, %o1 + %o3))
+	bgu,pt		%icc, 1b
+	 add		%o1, 1, %o1
+
+2:	add		%o1, %o3, %o0
+	andcc		%o1, 0x7, %g1
+	bne,pt		%icc, 8f
+	 sll		%g1, 3, %g1
+
+	cmp		%o2, 16
+	bgeu,pt		%icc, 72b
 	 nop
+	ba,a,pt		%xcc, 73b
+
+8:	mov		64, %o3
+	andn		%o1, 0x7, %o1
+	EX_LD(LOAD(ldx, %o1, %g2))
+	sub		%o3, %g1, %o3
+	andn		%o2, 0x7, %o4
+	sllx		%g2, %g1, %g2
+1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
+	subcc		%o4, 0x8, %o4
+	add		%o1, 0x8, %o1
+	srlx		%g3, %o3, %o5
+	or		%o5, %g2, %o5
+	EX_ST(STORE(stx, %o5, %o0))
+	add		%o0, 0x8, %o0
+	bgu,pt		%icc, 1b
+	 sllx		%g3, %g1, %g2
+
+	srl		%g1, 3, %g1
+	andcc		%o2, 0x7, %o2
+	be,pn		%icc, 85f
+	 add		%o1, %g1, %o1
+	ba,pt		%xcc, 90f
+	 sub		%o0, %o1, %o3
 
-small_copy: /* 0 < len <= 16 */
+	.align		64
+80: /* 0 < len <= 16 */
 	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
+	bne,pn		%XCC, 90f
 	 sub		%o0, %o1, %o3
 
-small_copy_aligned:
+1:
 	subcc		%o2, 4, %o2
-	lduw		[%o1], %g1
-	stw		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_aligned
+	EX_LD(LOAD(lduw, %o1, %g1))
+	EX_ST(STORE(stw, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
 	 add		%o1, 4, %o1
 
-out:	retl
-	 mov		%g5, %o0
+85:	retl
+	 mov		EX_RETVAL(%g5), %o0
 
-	.align	32
-small_copy_unaligned:
+	.align		32
+90:
 	subcc		%o2, 1, %o2
-	ldub		[%o1], %g1
-	stb		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_unaligned
+	EX_LD(LOAD(ldub, %o1, %g1))
+	EX_ST(STORE(stb, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 90b
 	 add		%o1, 1, %o1
 	retl
-	 mov		%g5, %o0
-
-	/* Act like copy_{to,in}_user(), ie. return zero instead
-	 * of original destination pointer.  This is invoked when
-	 * copy_{to,in}_user() finds that %asi is kernel space.
-	 */
-	.globl	U3memcpy_user_stub
-U3memcpy_user_stub:
-	save		%sp, -192, %sp
-	mov		%i0, %o0
-	mov		%i1, %o1
-	call		U3memcpy
-	 mov		%i2, %o2
-	ret
-	 restore	%g0, %g0, %o0
+	 mov		EX_RETVAL(%g5), %o0
diff -Nru a/arch/sparc64/lib/U3patch.S b/arch/sparc64/lib/U3patch.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U3patch.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,30 @@
+/* U3patch.S: Patch Ultra-I routines with Ultra-III variant.
+ *
+ * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+ */
+
+#define BRANCH_ALWAYS	0x10680000
+#define NOP		0x01000000
+#define ULTRA3_DO_PATCH(OLD, NEW)	\
+	sethi	%hi(NEW), %g1; \
+	or	%g1, %lo(NEW), %g1; \
+	sethi	%hi(OLD), %g2; \
+	or	%g2, %lo(OLD), %g2; \
+	sub	%g1, %g2, %g1; \
+	sethi	%hi(BRANCH_ALWAYS), %g3; \
+	srl	%g1, 2, %g1; \
+	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
+	or	%g3, %g1, %g3; \
+	stw	%g3, [%g2]; \
+	sethi	%hi(NOP), %g3; \
+	or	%g3, %lo(NOP), %g3; \
+	stw	%g3, [%g2 + 0x4]; \
+	flush	%g2;
+
+	.globl	cheetah_patch_copyops
+cheetah_patch_copyops:
+	ULTRA3_DO_PATCH(memcpy, U3memcpy)
+	ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user)
+	ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user)
+	retl
+	 nop
diff -Nru a/arch/sparc64/lib/VIScopy.S b/arch/sparc64/lib/VIScopy.S
--- a/arch/sparc64/lib/VIScopy.S	2004-07-27 13:08:37 -07:00
+++ /dev/null	Wed Dec 31 16:00:00 196900
@@ -1,1183 +0,0 @@
-/* $Id: VIScopy.S,v 1.26 2001/09/27 04:36:24 kanoj Exp $
- * VIScopy.S: High speed copy operations utilizing the UltraSparc
- *            Visual Instruction Set.
- *
- * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
- * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- */
-
-#include "VIS.h"
-
-	/* VIS code can be used for numerous copy/set operation variants.
-	 * It can be made to work in the kernel, one single instance,
-	 * for all of memcpy, copy_to_user, and copy_from_user by setting
-	 * the ASI src/dest globals correctly.  Furthermore it can
-	 * be used for kernel-->kernel page copies as well, a hook label
-	 * is put in here just for this purpose.
-	 *
-	 * For userland, compiling this without __KERNEL__ defined makes
-	 * it work just fine as a generic libc bcopy and memcpy.
-	 * If for userland it is compiled with a 32bit gcc (but you need
-	 * -Wa,-Av9a for as), the code will just rely on lower 32bits of
-	 * IEU registers, if you compile it with 64bit gcc (ie. define
-	 * __sparc_v9__), the code will use full 64bit.
-	 */
-	 
-#ifdef __KERNEL__
-
-#include <asm/visasm.h>
-#include <asm/asm_offsets.h>
-
-#define FPU_CLEAN_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	VISExit									\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define FPU_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	VISExit									\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define NORMAL_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define EX(x,y,a,b) 				\
-98: 	x,y;					\
-	.section .fixup;			\
-	.align	4;				\
-99:	ba	VIScopyfixup_ret;		\
-	 a, b, %o1;				\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, 99b;			\
-	.text;					\
-	.align	4;
-#define EX2(x,y,c,d,e,a,b) 			\
-98: 	x,y;					\
-	.section .fixup;			\
-	.align	4;				\
-99:	c, d, e;				\
-	ba	VIScopyfixup_ret;		\
-	 a, b, %o1;				\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, 99b;			\
-	.text;					\
-	.align	4;
-#define EXO2(x,y) 				\
-98: 	x,y;					\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, VIScopyfixup_reto2;	\
-	.text;					\
-	.align	4;
-#define EXVISN(x,y,n) 				\
-98: 	x,y;					\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, VIScopyfixup_vis##n;	\
-	.text;					\
-	.align	4;
-#define EXT(start,end,handler) 			\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	start, 0, end, handler;		\
-	.text;					\
-	.align	4;
-#else
-#ifdef REGS_64BIT
-#define FPU_CLEAN_RETL				\
-	retl;					\
-	 mov	%g6, %o0;
-#define FPU_RETL				\
-	retl;					\
-	 mov	%g6, %o0;
-#else
-#define FPU_CLEAN_RETL				\
-	wr	%g0, FPRS_FEF, %fprs;		\
-	retl;					\
-	 mov	%g6, %o0;
-#define FPU_RETL				\
-	wr	%g0, FPRS_FEF, %fprs;		\
-	retl;					\
-	 mov	%g6, %o0;
-#endif
-#define NORMAL_RETL	\
-	retl;		\
-	 mov	%g6, %o0;
-#define EX(x,y,a,b)		x,y
-#define EX2(x,y,c,d,e,a,b)	x,y
-#define EXO2(x,y)		x,y
-#define EXVISN(x,y,n)		x,y
-#define EXT(a,b,c)
-#endif
-#define EXVIS(x,y) EXVISN(x,y,0)
-#define EXVIS1(x,y) EXVISN(x,y,1)
-#define EXVIS2(x,y) EXVISN(x,y,2)
-#define EXVIS3(x,y) EXVISN(x,y,3)
-#define EXVIS4(x,y) EXVISN(x,y,4)
-
-#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
-	faligndata		%f1, %f2, %f48;			\
-	faligndata		%f2, %f3, %f50;			\
-	faligndata		%f3, %f4, %f52;			\
-	faligndata		%f4, %f5, %f54;			\
-	faligndata		%f5, %f6, %f56;			\
-	faligndata		%f6, %f7, %f58;			\
-	faligndata		%f7, %f8, %f60;			\
-	faligndata		%f8, %f9, %f62;
-
-#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt)	\
-	EXVIS(LDBLK		[%src] ASIBLK, %fdest);		\
-	ASI_SETDST_BLK						\
-	EXVIS(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%src, 0x40, %src;		\
-	subcc			%len, 0x40, %len;		\
-	be,pn			%xcc, jmptgt;			\
-	 add			%dest, 0x40, %dest;		\
-	ASI_SETSRC_BLK
-
-#define LOOP_CHUNK1(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f0,  f48, len, branch_dest)
-#define LOOP_CHUNK2(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
-#define LOOP_CHUNK3(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
-
-#define STORE_SYNC(dest, fsrc)					\
-	EXVIS(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%dest, 0x40, %dest;
-
-#ifdef __KERNEL__
-#define STORE_JUMP(dest, fsrc, target)				\
-	srl			asi_dest, 3, %g5;		\
-	EXVIS2(STBLK		%fsrc, [%dest] ASIBLK);		\
-	xor		       asi_dest, ASI_BLK_XOR1, asi_dest;\
-	add			%dest, 0x40, %dest;		\
-	xor			asi_dest, %g5, asi_dest;	\
-	ba,pt			%xcc, target;
-#else
-#define STORE_JUMP(dest, fsrc, target)				\
-	EXVIS2(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%dest, 0x40, %dest;		\
-	ba,pt			%xcc, target;
-#endif
-
-#ifndef __KERNEL__
-#define VISLOOP_PAD nop; nop; nop; nop; \
-		    nop; nop; nop; nop; \
-		    nop; nop; nop; nop; \
-		    nop; nop; nop;
-#else
-#define VISLOOP_PAD
-#endif
-
-#define FINISH_VISCHUNK(dest, f0, f1, left)			\
-	ASI_SETDST_NOBLK					\
-	subcc			%left, 8, %left;		\
-	bl,pn			%xcc, vis_out;			\
-	 faligndata		%f0, %f1, %f48;			\
-	EXVIS3(STDF		%f48, [%dest] ASINORMAL);	\
-	add			%dest, 8, %dest;
-
-#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)		\
-	subcc			%left, 8, %left;		\
-	bl,pn			%xcc, vis_out;			\
-	 fsrc1			%f0, %f1;
-#define UNEVEN_VISCHUNK(dest, f0, f1, left)			\
-	UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)		\
-	ba,a,pt			%xcc, vis_out_slk;
-
-	/* Macros for non-VIS memcpy code. */
-#ifdef REGS_64BIT
-
-#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3)			\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x00] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x08] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x10] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x18] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STW			%t0, [%dst + offset + 0x04] ASINORMAL; 	\
-	srlx			%t0, 32, %t0;				\
-	STW			%t0, [%dst + offset + 0x00] ASINORMAL; 	\
-	STW			%t1, [%dst + offset + 0x0c] ASINORMAL; 	\
-	srlx			%t1, 32, %t1;				\
-	STW			%t1, [%dst + offset + 0x08] ASINORMAL; 	\
-	STW			%t2, [%dst + offset + 0x14] ASINORMAL; 	\
-	srlx			%t2, 32, %t2;				\
-	STW			%t2, [%dst + offset + 0x10] ASINORMAL; 	\
-	STW			%t3, [%dst + offset + 0x1c] ASINORMAL;	\
-	srlx			%t3, 32, %t3;				\
-	STW			%t3, [%dst + offset + 0x18] ASINORMAL;
-
-#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x00] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x08] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x10] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x18] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst + offset + 0x00] ASINORMAL; 	\
-	STX			%t1, [%dst + offset + 0x08] ASINORMAL; 	\
-	STX			%t2, [%dst + offset + 0x10] ASINORMAL; 	\
-	STX			%t3, [%dst + offset + 0x18] ASINORMAL; 	\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x20] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x28] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x30] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x38] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst + offset + 0x20] ASINORMAL; 	\
-	STX			%t1, [%dst + offset + 0x28] ASINORMAL; 	\
-	STX			%t2, [%dst + offset + 0x30] ASINORMAL; 	\
-	STX			%t3, [%dst + offset + 0x38] ASINORMAL;
-
-#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src - offset - 0x10] ASINORMAL, %t0;	\
-	LDX			[%src - offset - 0x08] ASINORMAL, %t1; 	\
-	ASI_SETDST_NOBLK						\
-	STW			%t0, [%dst - offset - 0x0c] ASINORMAL; 	\
-	srlx			%t0, 32, %t2;				\
-	STW			%t2, [%dst - offset - 0x10] ASINORMAL; 	\
-	STW			%t1, [%dst - offset - 0x04] ASINORMAL; 	\
-	srlx			%t1, 32, %t3;				\
-	STW			%t3, [%dst - offset - 0x08] ASINORMAL;
-
-#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1)			\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src - offset - 0x10] ASINORMAL, %t0; 	\
-	LDX			[%src - offset - 0x08] ASINORMAL, %t1; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst - offset - 0x10] ASINORMAL; 	\
-	STX			%t1, [%dst - offset - 0x08] ASINORMAL;
-
-#else /* !REGS_64BIT */
-
-#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3)			\
-	lduw			[%src + offset + 0x00], %t0; 		\
-	lduw			[%src + offset + 0x04], %t1; 		\
-	lduw			[%src + offset + 0x08], %t2; 		\
-	lduw			[%src + offset + 0x0c], %t3; 		\
-	stw			%t0, [%dst + offset + 0x00]; 		\
-	stw			%t1, [%dst + offset + 0x04]; 		\
-	stw			%t2, [%dst + offset + 0x08]; 		\
-	stw			%t3, [%dst + offset + 0x0c]; 		\
-	lduw			[%src + offset + 0x10], %t0; 		\
-	lduw			[%src + offset + 0x14], %t1; 		\
-	lduw			[%src + offset + 0x18], %t2; 		\
-	lduw			[%src + offset + 0x1c], %t3; 		\
-	stw			%t0, [%dst + offset + 0x10]; 		\
-	stw			%t1, [%dst + offset + 0x14]; 		\
-	stw			%t2, [%dst + offset + 0x18]; 		\
-	stw			%t3, [%dst + offset + 0x1c];
-
-#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	lduw			[%src - offset - 0x10], %t0; 		\
-	lduw			[%src - offset - 0x0c], %t1; 		\
-	lduw			[%src - offset - 0x08], %t2; 		\
-	lduw			[%src - offset - 0x04], %t3; 		\
-	stw			%t0, [%dst - offset - 0x10]; 		\
-	stw			%t1, [%dst - offset - 0x0c]; 		\
-	stw			%t2, [%dst - offset - 0x08]; 		\
-	stw			%t3, [%dst - offset - 0x04];
-
-#endif /* !REGS_64BIT */
-
-#ifdef __KERNEL__
-		.section	__ex_table,#alloc
-		.section	.fixup,#alloc,#execinstr
-#endif
-
-		.text
-		.align			32
-		.globl			memcpy
-		.type			memcpy,@function
-
-		.globl			bcopy
-		.type			bcopy,@function
-
-#ifdef __KERNEL__
-		.globl			__memcpy_begin
-__memcpy_begin:
-
-memcpy_private:
-memcpy:		mov		ASI_P, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		ASI_P, asi_dest			! IEU1
-		retl
-		 clr		%o0
-
-		.align			32
-		.globl			__copy_from_user
-		.type			__copy_from_user,@function
-__copy_from_user:rd		%asi, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		ASI_P, asi_dest			! IEU1
-
-		.globl			__copy_to_user
-		.type			__copy_to_user,@function
-__copy_to_user:	mov		ASI_P, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 rd		%asi, asi_dest			! IEU1
-		retl						! CTI	Group
-		 clr		%o0				! IEU0	Group
-
-		.globl			__copy_in_user
-		.type			__copy_in_user,@function
-__copy_in_user:	rd		%asi, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		asi_src, asi_dest		! IEU1
-		retl						! CTI	Group
-		 clr		%o0				! IEU0	Group
-#endif
-
-bcopy:		or		%o0, 0, %g3			! IEU0	Group
-		addcc		%o1, 0, %o0			! IEU1
-		brgez,pt	%o2, memcpy_private		! CTI
-		 or		%g3, 0, %o1			! IEU0	Group
-		retl						! CTI	Group brk forced
-		 clr		%o0				! IEU0
-
-
-#ifdef __KERNEL__
-#define BRANCH_ALWAYS	0x10680000
-#define NOP		0x01000000
-#define ULTRA3_DO_PATCH(OLD, NEW)	\
-	sethi	%hi(NEW), %g1; \
-	or	%g1, %lo(NEW), %g1; \
-	sethi	%hi(OLD), %g2; \
-	or	%g2, %lo(OLD), %g2; \
-	sub	%g1, %g2, %g1; \
-	sethi	%hi(BRANCH_ALWAYS), %g3; \
-	srl	%g1, 2, %g1; \
-	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
-	or	%g3, %g1, %g3; \
-	stw	%g3, [%g2]; \
-	sethi	%hi(NOP), %g3; \
-	or	%g3, %lo(NOP), %g3; \
-	stw	%g3, [%g2 + 0x4]; \
-	flush	%g2;
-#define ULTRA3_PCACHE_DO_NOP(symbol)	\
-	sethi	%hi(symbol##_nop_1_6), %g1; \
-	or	%g1, %lo(symbol##_nop_1_6), %g1; \
-	sethi	%hi(NOP), %g2; \
-	stw	%g2, [%g1 + 0x00]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x00; \
-	stw	%g2, [%g1 + 0x08]; \
-	stw	%g2, [%g1 + 0x0c]; \
-	flush	%g1 + 0x08; \
-	stw	%g2, [%g1 + 0x10]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x10; \
-	sethi	%hi(symbol##_nop_2_3), %g1; \
-	or	%g1, %lo(symbol##_nop_2_3), %g1; \
-	stw	%g2, [%g1 + 0x00]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x00; \
-	stw	%g2, [%g1 + 0x08]; \
-	flush	%g1 + 0x08;
-
-#include <asm/dcu.h>
-
-	.globl	cheetah_patch_copyops
-cheetah_patch_copyops:
-	ULTRA3_DO_PATCH(memcpy, U3memcpy)
-	ULTRA3_DO_PATCH(__copy_from_user, U3copy_from_user)
-	ULTRA3_DO_PATCH(__copy_to_user, U3copy_to_user)
-	ULTRA3_DO_PATCH(__copy_in_user, U3copy_in_user)
-#if 0 /* Causes data corruption, nop out the optimization
-       * for now -DaveM
-       */
-	ldxa			[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi			%uhi(DCU_PE), %o3
-	sllx			%o3, 32, %o3
-	andcc			%g3, %o3, %g0
-	be,pn			%xcc, pcache_disabled
-	 nop
-#endif
-	ULTRA3_PCACHE_DO_NOP(U3memcpy)
-	ULTRA3_PCACHE_DO_NOP(U3copy_from_user)
-	ULTRA3_PCACHE_DO_NOP(U3copy_to_user)
-	ULTRA3_PCACHE_DO_NOP(cheetah_copy_user_page)
-#if 0
-pcache_disabled:
-#endif
-	retl
-	 nop
-#undef BRANCH_ALWAYS
-#undef NOP
-#undef ULTRA3_DO_PATCH
-#endif /* __KERNEL__ */
-
-	.align			32
-#ifdef __KERNEL__
-	andcc			%o0, 7, %g2			! IEU1	Group
-#endif
-VIS_enter:
-	be,pt			%xcc, dest_is_8byte_aligned	! CTI
-#ifdef __KERNEL__
-	 nop							! IEU0	Group
-#else
-	 andcc			%o0, 0x38, %g5			! IEU1	Group
-#endif
-do_dest_8byte_align:
-	mov			8, %g1				! IEU0
-	sub			%g1, %g2, %g2			! IEU0	Group
-	andcc			%o0, 1, %g0			! IEU1
-	be,pt			%icc, 2f			! CTI
-	 sub			%o2, %g2, %o2			! IEU0	Group
-1:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %o5, 
-				add %o2, %g2)			! Load	Group
-	add			%o1, 1, %o1			! IEU0
-	add			%o0, 1, %o0			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 1, %g2			! IEU1	Group
-	be,pn			%xcc, 3f			! CTI
-	 EX2(STB		%o5, [%o0 - 1] ASINORMAL,
-				add %g2, 1, %g2,
-				add %o2, %g2)			! Store
-2:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %o5, 
-				add %o2, %g2)			! Load	Group
-	add			%o0, 2, %o0			! IEU0
-	EX2(LDUB		[%o1 + 1] ASINORMAL, %g3,
-				sub %o0, 2, %o0,
-				add %o2, %g2)			! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 2, %g2			! IEU1	Group
-	EX2(STB			%o5, [%o0 - 2] ASINORMAL,
-				add %g2, 2, %g2,
-				add %o2, %g2)			! Store
-	add			%o1, 2, %o1			! IEU0
-	bne,pt			%xcc, 2b			! CTI	Group
-	 EX2(STB		%g3, [%o0 - 1] ASINORMAL,
-				add %g2, 1, %g2,
-				add %o2, %g2)			! Store
-#ifdef __KERNEL__
-3:
-dest_is_8byte_aligned:
-	VISEntry
-	andcc			%o0, 0x38, %g5			! IEU1	Group
-#else
-3:	andcc			%o0, 0x38, %g5			! IEU1	Group
-dest_is_8byte_aligned:
-#endif
-	be,pt			%icc, dest_is_64byte_aligned	! CTI
-	 mov			64, %g1				! IEU0
-	fmovd			%f0, %f2			! FPU
-	sub			%g1, %g5, %g5			! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	alignaddr		%o1, %g0, %g1			! GRU	Group
-	EXO2(LDDF		[%g1] ASINORMAL, %f4)		! Load	Group
-	sub			%o2, %g5, %o2			! IEU0
-1:	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f6,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 0x8, %g1			! IEU0	Group
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f4, %f6, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %g5, 8, %g5,
-				add %o2, %g5)			! Store
-	add			%o1, 8, %o1			! IEU0	Group
-	be,pn			%xcc, dest_is_64byte_aligned	! CTI
-	 add			%o0, 8, %o0			! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f4,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 8, %g1			! IEU0
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f6, %f4, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %g5, 8, %g5,
-				add %o2, %g5)			! Store
-	add			%o1, 8, %o1			! IEU0
-	ASI_SETSRC_NOBLK					! LSU	Group
-	bne,pt			%xcc, 1b			! CTI	Group
-	 add			%o0, 8, %o0			! IEU0
-dest_is_64byte_aligned:
-	membar		  #LoadStore | #StoreStore | #StoreLoad	! LSU	Group
-#ifndef __KERNEL__
-	wr			%g0, ASI_BLK_P, %asi		! LSU	Group
-#endif
-	subcc			%o2, 0x40, %g7			! IEU1	Group
-	mov			%o1, %g1			! IEU0
-	andncc			%g7, (0x40 - 1), %g7		! IEU1	Group
-	srl			%g1, 3, %g2			! IEU0
-	sub			%o2, %g7, %g3			! IEU0	Group
-	andn			%o1, (0x40 - 1), %o1		! IEU1
-	and			%g2, 7, %g2			! IEU0	Group
-	andncc			%g3, 0x7, %g3			! IEU1
-	fmovd			%f0, %f2			! FPU
-	sub			%g3, 0x10, %g3			! IEU0	Group
-	sub			%o2, %g7, %o2			! IEU1
-#ifdef __KERNEL__
-	or			asi_src, ASI_BLK_OR, asi_src	! IEU0	Group
-	or			asi_dest, ASI_BLK_OR, asi_dest	! IEU1
-#endif
-	alignaddr		%g1, %g0, %g0			! GRU	Group
-	add			%g1, %g7, %g1			! IEU0	Group
-	subcc			%o2, %g3, %o2			! IEU1
-	ASI_SETSRC_BLK						! LSU	Group
-	EXVIS1(LDBLK		[%o1 + 0x00] ASIBLK, %f0)	! LSU	Group
-	add			%g1, %g3, %g1			! IEU0
-	EXVIS1(LDBLK		[%o1 + 0x40] ASIBLK, %f16)	! LSU	Group
-	sub			%g7, 0x80, %g7			! IEU0
-	EXVIS(LDBLK		[%o1 + 0x80] ASIBLK, %f32)	! LSU	Group
-#ifdef __KERNEL__
-vispc:	sll			%g2, 9, %g2			! IEU0	Group
-	sethi			%hi(vis00), %g5			! IEU1
-	or			%g5, %lo(vis00), %g5		! IEU0	Group
-	jmpl			%g5 + %g2, %g0			! CTI	Group brk forced
-	 addcc			%o1, 0xc0, %o1			! IEU1	Group
-#else
-								! Clk1	Group 8-(
-								! Clk2	Group 8-(
-								! Clk3	Group 8-(
-								! Clk4	Group 8-(
-vispc:	rd			%pc, %g5			! PDU	Group 8-(
-	addcc			%g5, %lo(vis00 - vispc), %g5	! IEU1	Group
-	sll			%g2, 9, %g2			! IEU0
-	jmpl			%g5 + %g2, %g0			! CTI	Group brk forced
-	 addcc			%o1, 0xc0, %o1			! IEU1	Group
-#endif
-	.align			512		/* OK, here comes the fun part... */
-vis00:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) LOOP_CHUNK1(o1, o0, g7, vis01)
-      FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) LOOP_CHUNK2(o1, o0, g7, vis02)
-      FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  LOOP_CHUNK3(o1, o0, g7, vis03)
-      b,pt			%xcc, vis00+4; faligndata %f0, %f2, %f48
-vis01:FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  STORE_JUMP(o0, f48, finish_f0) membar #Sync
-vis02:FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_JUMP(o0, f48, finish_f16) membar #Sync
-vis03:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_JUMP(o0, f48, finish_f32) membar #Sync
-      VISLOOP_PAD
-vis10:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) LOOP_CHUNK1(o1, o0, g7, vis11)
-      FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) LOOP_CHUNK2(o1, o0, g7, vis12)
-      FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  LOOP_CHUNK3(o1, o0, g7, vis13)
-      b,pt			%xcc, vis10+4; faligndata %f2, %f4, %f48
-vis11:FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  STORE_JUMP(o0, f48, finish_f2) membar #Sync
-vis12:FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_JUMP(o0, f48, finish_f18) membar #Sync
-vis13:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_JUMP(o0, f48, finish_f34) membar #Sync
-      VISLOOP_PAD
-vis20:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) LOOP_CHUNK1(o1, o0, g7, vis21)
-      FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) LOOP_CHUNK2(o1, o0, g7, vis22)
-      FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  LOOP_CHUNK3(o1, o0, g7, vis23)
-      b,pt			%xcc, vis20+4; faligndata %f4, %f6, %f48
-vis21:FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  STORE_JUMP(o0, f48, finish_f4) membar #Sync
-vis22:FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_JUMP(o0, f48, finish_f20) membar #Sync
-vis23:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_JUMP(o0, f48, finish_f36) membar #Sync
-      VISLOOP_PAD
-vis30:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) LOOP_CHUNK1(o1, o0, g7, vis31)
-      FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) LOOP_CHUNK2(o1, o0, g7, vis32)
-      FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  LOOP_CHUNK3(o1, o0, g7, vis33)
-      b,pt			%xcc, vis30+4; faligndata %f6, %f8, %f48
-vis31:FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  STORE_JUMP(o0, f48, finish_f6) membar #Sync
-vis32:FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_JUMP(o0, f48, finish_f22) membar #Sync
-vis33:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_JUMP(o0, f48, finish_f38) membar #Sync
-      VISLOOP_PAD
-vis40:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) LOOP_CHUNK1(o1, o0, g7, vis41)
-      FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) LOOP_CHUNK2(o1, o0, g7, vis42)
-      FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  LOOP_CHUNK3(o1, o0, g7, vis43)
-      b,pt			%xcc, vis40+4; faligndata %f8, %f10, %f48
-vis41:FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  STORE_JUMP(o0, f48, finish_f8) membar #Sync
-vis42:FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_JUMP(o0, f48, finish_f24) membar #Sync
-vis43:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_JUMP(o0, f48, finish_f40) membar #Sync
-      VISLOOP_PAD
-vis50:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) LOOP_CHUNK1(o1, o0, g7, vis51)
-      FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) LOOP_CHUNK2(o1, o0, g7, vis52)
-      FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) LOOP_CHUNK3(o1, o0, g7, vis53)
-      b,pt			%xcc, vis50+4; faligndata %f10, %f12, %f48
-vis51:FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_JUMP(o0, f48, finish_f10) membar #Sync
-vis52:FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_JUMP(o0, f48, finish_f26) membar #Sync
-vis53:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_JUMP(o0, f48, finish_f42) membar #Sync
-      VISLOOP_PAD
-vis60:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) LOOP_CHUNK1(o1, o0, g7, vis61)
-      FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) LOOP_CHUNK2(o1, o0, g7, vis62)
-      FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) LOOP_CHUNK3(o1, o0, g7, vis63)
-      b,pt			%xcc, vis60+4; faligndata %f12, %f14, %f48
-vis61:FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_JUMP(o0, f48, finish_f12) membar #Sync
-vis62:FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_JUMP(o0, f48, finish_f28) membar #Sync
-vis63:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_JUMP(o0, f48, finish_f44) membar #Sync
-      VISLOOP_PAD
-vis70:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) LOOP_CHUNK1(o1, o0, g7, vis71)
-      FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) LOOP_CHUNK2(o1, o0, g7, vis72)
-      FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) LOOP_CHUNK3(o1, o0, g7, vis73)
-      b,pt			%xcc, vis70+4; faligndata %f14, %f16, %f48
-vis71:FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_JUMP(o0, f48, finish_f14) membar #Sync
-vis72:FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_JUMP(o0, f48, finish_f30) membar #Sync
-vis73:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, finish_f46) membar #Sync
-      VISLOOP_PAD
-finish_f0:	FINISH_VISCHUNK(o0, f0,  f2,  g3)
-finish_f2:	FINISH_VISCHUNK(o0, f2,  f4,  g3)
-finish_f4:	FINISH_VISCHUNK(o0, f4,  f6,  g3)
-finish_f6:	FINISH_VISCHUNK(o0, f6,  f8,  g3)
-finish_f8:	FINISH_VISCHUNK(o0, f8,  f10, g3)
-finish_f10:	FINISH_VISCHUNK(o0, f10, f12, g3)
-finish_f12:	FINISH_VISCHUNK(o0, f12, f14, g3)
-finish_f14:	UNEVEN_VISCHUNK(o0, f14, f0,  g3)
-finish_f16:	FINISH_VISCHUNK(o0, f16, f18, g3)
-finish_f18:	FINISH_VISCHUNK(o0, f18, f20, g3)
-finish_f20:	FINISH_VISCHUNK(o0, f20, f22, g3)
-finish_f22:	FINISH_VISCHUNK(o0, f22, f24, g3)
-finish_f24:	FINISH_VISCHUNK(o0, f24, f26, g3)
-finish_f26:	FINISH_VISCHUNK(o0, f26, f28, g3)
-finish_f28:	FINISH_VISCHUNK(o0, f28, f30, g3)
-finish_f30:	UNEVEN_VISCHUNK(o0, f30, f0,  g3)
-finish_f32:	FINISH_VISCHUNK(o0, f32, f34, g3)
-finish_f34:	FINISH_VISCHUNK(o0, f34, f36, g3)
-finish_f36:	FINISH_VISCHUNK(o0, f36, f38, g3)
-finish_f38:	FINISH_VISCHUNK(o0, f38, f40, g3)
-finish_f40:	FINISH_VISCHUNK(o0, f40, f42, g3)
-finish_f42:	FINISH_VISCHUNK(o0, f42, f44, g3)
-finish_f44:	FINISH_VISCHUNK(o0, f44, f46, g3)
-finish_f46:	UNEVEN_VISCHUNK_LAST(o0, f46, f0,  g3)
-vis_out_slk:
-#ifdef __KERNEL__
-	srl		asi_src, 3, %g5				! IEU0	Group
-	xor		asi_src, ASI_BLK_XOR1, asi_src		! IEU1
-	xor		asi_src, %g5, asi_src			! IEU0	Group
-#endif
-vis_slk:ASI_SETSRC_NOBLK					! LSU	Group
-	EXVIS3(LDDF	[%o1] ASINORMAL, %f2)			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	subcc		%g3, 8, %g3				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata	%f0, %f2, %f8				! GRU	Group
-	EXVIS4(STDF	%f8, [%o0] ASINORMAL)			! Store
-	bl,pn		%xcc, vis_out_slp			! CTI
-	 add		%o0, 8, %o0				! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXVIS3(LDDF	[%o1] ASINORMAL, %f0)			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	subcc		%g3, 8, %g3				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata	%f2, %f0, %f8				! GRU	Group
-	EXVIS4(STDF	%f8, [%o0] ASINORMAL)			! Store
-	bge,pt		%xcc, vis_slk				! CTI
-	 add		%o0, 8, %o0				! IEU0	Group
-vis_out_slp:
-#ifdef __KERNEL__
-	brz,pt		%o2, vis_ret				! CTI	Group
-	 mov		%g1, %o1				! IEU0
-	ba,pt		%xcc, vis_slp+4				! CTI	Group
-	 ASI_SETSRC_NOBLK					! LSU	Group
-#endif
-vis_out:brz,pt		%o2, vis_ret				! CTI	Group
-	 mov		%g1, %o1				! IEU0
-#ifdef __KERNEL__
-	srl		asi_src, 3, %g5				! IEU0	Group
-	xor		asi_src, ASI_BLK_XOR1, asi_src		! IEU1
-	xor		asi_src, %g5, asi_src			! IEU0	Group
-#endif
-vis_slp:ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1
-	bne,pt		%xcc, vis_slp				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store	Group
-vis_ret:membar		#StoreLoad | #StoreStore		! LSU	Group
-	FPU_CLEAN_RETL
-
-
-__memcpy_short:
-	andcc		%o2, 1, %g0				! IEU1	Group
-	be,pt		%icc, 2f				! CTI
-1:	 ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD	Group
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1	Group
-	be,pn		%xcc, short_ret				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-2:	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD	Group
-	add		%o0, 2, %o0				! IEU0
-	EX2(LDUB	[%o1 + 1] ASINORMAL, %o5,
-			sub %o0, 2, %o0,
-			add %o2, %g0)				! LOAD	Group
-	add		%o1, 2, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 2, %o2				! IEU1	Group
-	EX(STB		%g5, [%o0 - 2] ASINORMAL,
-			add %o2, 2)				! Store
-	bne,pt		%xcc, 2b				! CTI
-	 EX(STB		%o5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-short_ret:
-	NORMAL_RETL
-
-#ifndef __KERNEL__
-memcpy_private:
-memcpy:
-#ifndef REGS_64BIT
-	srl		%o2, 0, %o2				! IEU1	Group
-#endif	
-	brz,pn		%o2, short_ret				! CTI	Group
-	 mov		%o0, %g6				! IEU0
-#endif
-__memcpy_entry:
-	cmp		%o2, 15					! IEU1	Group
-	bleu,pn		%xcc, __memcpy_short			! CTI
-	 cmp		%o2, (64 * 6)				! IEU1	Group
-	bgeu,pn		%xcc, VIS_enter				! CTI
-	 andcc		%o0, 7, %g2				! IEU1	Group
-	sub		%o0, %o1, %g5				! IEU0
-	andcc		%g5, 3, %o5				! IEU1	Group
-	bne,pn		%xcc, memcpy_noVIS_misaligned		! CTI
-	 andcc		%o1, 3, %g0				! IEU1	Group
-#ifdef REGS_64BIT
-	be,a,pt		%xcc, 3f				! CTI
-	 andcc		%o1, 4, %g0				! IEU1	Group
-	andcc		%o1, 1, %g0				! IEU1	Group
-#else /* !REGS_64BIT */
-	be,pt		%xcc, 5f				! CTI
-	 andcc		%o1, 1, %g0				! IEU1	Group
-#endif /* !REGS_64BIT */
-	be,pn		%xcc, 4f				! CTI
-	 andcc		%o1, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g2)			! Load	Group
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	sub		%o2, 1, %o2				! IEU0	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	bne,pn		%xcc, 5f				! CTI	Group
-	 EX(STB		%g2, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-4:	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUH	[%o1] ASINORMAL, %g2)			! Load	Group
-	add		%o1, 2, %o1				! IEU0
-	add		%o0, 2, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	sub		%o2, 2, %o2				! IEU0
-	EX(STH		%g2, [%o0 - 2] ASINORMAL,
-			add %o2, 2)				! Store	Group + bubble
-#ifdef REGS_64BIT
-5:	andcc		%o1, 4, %g0				! IEU1
-3:	be,a,pn		%xcc, 2f				! CTI
-	 andcc		%o2, -128, %g7				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUW	[%o1] ASINORMAL, %g5)			! Load	Group
-	add		%o1, 4, %o1				! IEU0
-	add		%o0, 4, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	sub		%o2, 4, %o2				! IEU0	Group
-	EX(STW		%g5, [%o0 - 4] ASINORMAL,
-			add %o2, 4)				! Store
-	andcc		%o2, -128, %g7				! IEU1	Group
-2:	be,pn		%xcc, 3f				! CTI
-	 andcc		%o0, 4, %g0				! IEU1	Group
-	be,pn		%xcc, 82f + 4				! CTI	Group
-#else /* !REGS_64BIT */
-5:	andcc		%o2, -128, %g7				! IEU1
-	be,a,pn		%xcc, 41f				! CTI
-	 andcc		%o2, 0x70, %g7				! IEU1	Group
-#endif /* !REGS_64BIT */
-5:	MOVE_BIGCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x20, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x60, g1, g3, g5, o5)
-	EXT(5b,35f,VIScopyfixup1)
-35:	subcc		%g7, 128, %g7				! IEU1	Group
-	add		%o1, 128, %o1				! IEU0
-	bne,pt		%xcc, 5b				! CTI
-	 add		%o0, 128, %o0				! IEU0	Group
-3:	andcc		%o2, 0x70, %g7				! IEU1	Group
-41:	be,pn		%xcc, 80f				! CTI
-	 andcc		%o2, 8, %g0				! IEU1	Group
-#ifdef __KERNEL__
-79:	sethi		%hi(80f), %o5				! IEU0
-	sll		%g7, 1, %g5				! IEU0	Group
-	add		%o1, %g7, %o1				! IEU1
-	srl		%g7, 1, %g2				! IEU0  Group
-	sub		%o5, %g5, %o5				! IEU1
-	sub		%o5, %g2, %o5				! IEU0  Group
-	jmpl		%o5 + %lo(80f), %g0			! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#else
-								! Clk1 8-(
-								! Clk2 8-(
-								! Clk3 8-(
-								! Clk4 8-(
-79:	rd		%pc, %o5				! PDU	Group
-	sll		%g7, 1, %g5				! IEU0	Group
-	add		%o1, %g7, %o1				! IEU1
-	sub		%o5, %g5, %o5				! IEU0  Group
-	jmpl		%o5 + %lo(80f - 79b), %g0		! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#endif
-36:	MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g5, o5)
-	EXT(36b,80f,VIScopyfixup2)
-80:	be,pt		%xcc, 81f				! CTI
-	 andcc		%o2, 4, %g0				! IEU1
-#ifdef REGS_64BIT
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDX		[%o1] ASINORMAL, %g2,
-			and %o2, 0xf)				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STW		%g2, [%o0 - 0x4] ASINORMAL,
-			and %o2, 0xf)				! Store	Group
-	add		%o1, 8, %o1				! IEU1
-	srlx		%g2, 32, %g2				! IEU0	Group
-	EX2(STW		%g2, [%o0 - 0x8] ASINORMAL,
-			and %o2, 0xf, %o2,
-			sub %o2, 4)				! Store
-#else /* !REGS_64BIT */
-	lduw		[%o1], %g2				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	lduw		[%o1 + 0x4], %g3			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	stw		%g2, [%o0 - 0x8]			! Store	Group
-	stw		%g3, [%o0 - 0x4]			! Store	Group
-#endif /* !REGS_64BIT */
-81:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUW		[%o1] ASINORMAL, %g2,
-			and %o2, 0x7)				! Load	Group
-	add		%o1, 4, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STW		%g2, [%o0] ASINORMAL,
-			and %o2, 0x7)				! Store	Group
-	add		%o0, 4, %o0				! IEU0
-1:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 1, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUH		[%o1] ASINORMAL, %g2,
-			and %o2, 0x3)				! Load	Group
-	add		%o1, 2, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STH		%g2, [%o0] ASINORMAL,
-			and %o2, 0x3)				! Store	Group
-	add		%o0, 2, %o0				! IEU0
-1:	be,pt		%xcc, normal_retl			! CTI
-	 nop							! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB		[%o1] ASINORMAL, %g2,
-			add %g0, 1)				! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STB		%g2, [%o0] ASINORMAL,
-			add %g0, 1)				! Store	Group + bubble
-normal_retl:
-	NORMAL_RETL
-
-#ifdef REGS_64BIT
-82:	MOVE_BIGALIGNCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
-	MOVE_BIGALIGNCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
-	EXT(82b,37f,VIScopyfixup3)
-37:	subcc		%g7, 128, %g7				! IEU1	Group
-	add		%o1, 128, %o1				! IEU0
-	bne,pt		%xcc, 82b				! CTI
-	 add		%o0, 128, %o0				! IEU0	Group
-	andcc		%o2, 0x70, %g7				! IEU1
-	be,pn		%xcc, 84f				! CTI
-	 andcc		%o2, 8, %g0				! IEU1	Group
-#ifdef __KERNEL__
-83:	srl		%g7, 1, %g5				! IEU0
-	sethi		%hi(84f), %o5				! IEU0	Group
-	add		%g7, %g5, %g5				! IEU1
-	add		%o1, %g7, %o1				! IEU0	Group
-	sub		%o5, %g5, %o5				! IEU1
-	jmpl		%o5 + %lo(84f), %g0			! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#else
-								! Clk1 8-(
-								! Clk2 8-(
-								! Clk3 8-(
-								! Clk4 8-(
-83:	rd		%pc, %o5				! PDU	Group
-	add		%o1, %g7, %o1				! IEU0	Group
-	sub		%o5, %g7, %o5				! IEU1
-	jmpl		%o5 + %lo(84f - 83b), %g0		! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#endif
-38:	MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3)
-	EXT(38b,84f,VIScopyfixup4)
-84:	be,pt		%xcc, 85f				! CTI	Group
-	 andcc		%o2, 4, %g0				! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDX		[%o1] ASINORMAL, %g2,
-			and %o2, 0xf)				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 8, %o1				! IEU0	Group
-	EX(STX		%g2, [%o0 - 0x8] ASINORMAL,
-			and %o2, 0xf)				! Store
-85:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUW		[%o1] ASINORMAL, %g2,
-			and %o2, 0x7)				! Load	Group
-	add		%o0, 4, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 4, %o1				! IEU0	Group
-	EX(STW		%g2, [%o0 - 0x4] ASINORMAL,
-			and %o2, 0x7)				! Store
-1:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 1, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUH		[%o1] ASINORMAL, %g2,
-			and %o2, 0x3)				! Load	Group
-	add		%o0, 2, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 2, %o1				! IEU0	Group
-	EX(STH		%g2, [%o0 - 0x2] ASINORMAL,
-			and %o2, 0x3)				! Store
-1:	be,pt		%xcc, 1f				! CTI
-	 nop							! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB		[%o1] ASINORMAL, %g2,
-			add %g0, 1)				! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STB		%g2, [%o0] ASINORMAL,
-			add %g0, 1)				! Store	Group + bubble
-1:	NORMAL_RETL
-#endif	/* REGS_64BIT */
-
-memcpy_noVIS_misaligned:
-	brz,pt			%g2, 2f				! CTI	Group
-	 mov			8, %g1				! IEU0
-	sub			%g1, %g2, %g2			! IEU0	Group
-	sub			%o2, %g2, %o2			! IEU0	Group
-1:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %g5,
-				add %o2, %g2)			! Load	Group
-	add			%o1, 1, %o1			! IEU0
-	add			%o0, 1, %o0			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 1, %g2			! IEU1	Group
-	bne,pt			%xcc, 1b			! CTI
-	 EX2(STB		%g5, [%o0 - 1] ASINORMAL,
-				add %o2, %g2, %o2,
-				add %o2, 1)			! Store
-2:
-#ifdef __KERNEL__
-	VISEntry
-#endif
-	andn			%o2, 7, %g5 			! IEU0	Group
-	and			%o2, 7, %o2			! IEU1
-	fmovd			%f0, %f2			! FPU
-	ASI_SETSRC_NOBLK					! LSU	Group
-	alignaddr		%o1, %g0, %g1			! GRU	Group
-	EXO2(LDDF		[%g1] ASINORMAL, %f4)		! Load	Group
-1:	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f6,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 0x8, %g1			! IEU0	Group
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f4, %f6, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %o2, %g5, %o2,
-				add %o2, 8)			! Store
-	add			%o1, 8, %o1			! IEU0	Group
-	be,pn			%xcc, end_cruft			! CTI
-	 add			%o0, 8, %o0			! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f4,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 8, %g1			! IEU0
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f6, %f4, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %o2, %g5, %o2,
-				add %o2, 8)			! Store
-	add			%o1, 8, %o1			! IEU0
-	ASI_SETSRC_NOBLK					! LSU	Group
-	bne,pn			%xcc, 1b			! CTI	Group
-	 add			%o0, 8, %o0			! IEU0
-end_cruft:
-	brz,pn			%o2, fpu_retl			! CTI	Group
-#ifndef __KERNEL__
-	 nop							! IEU0
-#else
-	 ASI_SETSRC_NOBLK					! LSU	Group
-#endif
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1
-	bne,pt		%xcc, vis_slp				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store	Group
-fpu_retl:
-	FPU_RETL
-
-#ifdef __KERNEL__
-	.globl		__memcpy_end
-__memcpy_end:
-
-		.section	.fixup
-		.align		4
-VIScopyfixup_reto2:
-		mov		%o2, %o1
-VIScopyfixup_ret:
-		/* If this is copy_from_user(), zero out the rest of the
-		 * kernel buffer.
-		 */
-		ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o4
-		andcc		asi_src, 0x1, %g0
-		be,pt		%icc, 1f
-		 VISExit
-		andcc		asi_dest, 0x1, %g0
-		bne,pn		%icc, 1f
-		 nop
-		save		%sp, -160, %sp
-		mov		%i0, %o0
-		call		__bzero
-		 mov		%i1, %o1
-		restore
-1:		mov		%o1, %o0
-		retl
-		 wr		%o4, %g0, %asi
-VIScopyfixup1:	subcc		%g2, 18, %g2
-		add		%o0, 32, %o0
-		bgeu,a,pt	%icc, VIScopyfixup1
-		 sub		%g7, 32, %g7
-		sub		%o0, 32, %o0
-		rd		%pc, %g5
-		add		%g2, (18 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 2f
-.byte		0, 0, 0, 0, 0, 0, 0, 4, 4, 8, 12, 12, 16, 20, 20, 24, 28, 28
-		.align		4
-VIScopyfixup2:	mov		(7 * 16), %g7
-1:		subcc		%g2, 10, %g2
-		bgeu,a,pt	%icc, 1b
-		 sub		%g7, 16, %g7
-		sub		%o0, %g7, %o0
-		rd		%pc, %g5
-		add		%g2, (10 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 4f
-.byte		0, 0, 0, 0, 0, 4, 4, 8, 12, 12
-		.align		4
-VIScopyfixup3:	subcc		%g2, 10, %g2
-		add		%o0, 32, %o0
-		bgeu,a,pt	%icc, VIScopyfixup3
-		 sub		%g7, 32, %g7
-		sub		%o0, 32, %o0
-		rd		%pc, %g5
-		add		%g2, (10 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 2f
-.byte		0, 0, 0, 0, 0, 0, 0, 8, 16, 24
-		.align		4
-2:		and		%o2, 0x7f, %o2
-		sub		%g7, %g2, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%g7, %o2, %o1
-VIScopyfixup4:	mov		(7 * 16), %g7
-3:		subcc		%g2, 6, %g2
-		bgeu,a,pt	%icc, 3b
-		 sub		%g7, 16, %g7
-		sub		%o0, %g7, %o0
-		rd		%pc, %g5
-		add		%g2, (6 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 4f
-.byte		0, 0, 0, 0, 0, 8
-		.align		4
-4:		and		%o2, 0xf, %o2
-		sub		%g7, %g2, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%g7, %o2, %o1
-VIScopyfixup_vis2:
-		sub		%o2, 0x40, %o2
-VIScopyfixup_vis0:
-		add		%o2, 0x80, %o2
-VIScopyfixup_vis1:
-		add		%g7, %g3, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%o2, %g7, %o1
-VIScopyfixup_vis4:
-		add		%g3, 8, %g3
-VIScopyfixup_vis3:
-		add		%g3, 8, %g3
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%o2, %g3, %o1
-#endif
-
-#ifdef __KERNEL__
-		.text
-		.align		32
-
-		.globl		__memmove
-		.type		__memmove,@function
-
-		.globl		memmove
-		.type		memmove,@function
-
-memmove:
-__memmove:	cmp		%o0, %o1
-		blu,pt		%xcc, memcpy_private
-		 sub		%o0, %o1, %g5
-		add		%o1, %o2, %g3
-		cmp		%g3, %o0
-		bleu,pt		%xcc, memcpy_private
-		 add		%o1, %o2, %g5
-		add		%o0, %o2, %o5
-
-		sub		%g5, 1, %o1
-		sub		%o5, 1, %o0
-1:		ldub		[%o1], %g5
-		subcc		%o2, 1, %o2
-		sub		%o1, 1, %o1
-		stb		%g5, [%o0]
-		bne,pt		%icc, 1b
-		 sub		%o0, 1, %o0
-
-		retl
-		 clr		%o0
-#endif
diff -Nru a/arch/sparc64/lib/copy_in_user.S b/arch/sparc64/lib/copy_in_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/copy_in_user.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,114 @@
+/* copy_in_user.S: Copy from userspace to userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#include <asm/asi.h>
+
+#define XCC xcc
+
+#define EX(x,y)			\
+98:	x,y;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov 1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+	.register	%g2,#scratch
+	.register	%g3,#scratch
+
+	.text
+	.align	32
+
+	/* Don't try to get too fancy here, just nice and
+	 * simple.  This is predominantly used for well aligned
+	 * small copies in the compat layer.  It is also used
+	 * to copy register windows around during thread cloning.
+	 */
+
+	.globl	___copy_in_user
+___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+	rd		%asi, %g1
+	cmp		%g1, ASI_AIUS
+	bne,pn		%icc, memcpy_user_stub
+	 nop
+
+	cmp		%o2, 0
+	be,pn		%XCC, 85f
+	 or		%o0, %o1, %o3
+	cmp		%o2, 16
+	bleu,a,pn	%XCC, 80f
+	 or		%o3, %o2, %o3
+
+	/* 16 < len <= 64 */
+	andcc		%o3, 0x7, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+	andn		%o2, 0x7, %o4
+	and		%o2, 0x7, %o2
+1:	subcc		%o4, 0x8, %o4
+	EX(ldxa [%o1] %asi, %o5)
+	EX(stxa %o5, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x8, %o1
+	andcc		%o2, 0x4, %g0
+	be,pt		%XCC, 1f
+	 nop
+	sub		%o2, 0x4, %o2
+	EX(lduwa [%o1] %asi, %o5)
+	EX(stwa %o5, [%o1 + %o3] ASI_AIUS)
+	add		%o1, 0x4, %o1
+1:	cmp		%o2, 0
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
+	 nop
+
+80:	/* 0 < len <= 16 */
+	andcc		%o3, 0x3, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+82:
+	subcc		%o2, 4, %o2
+	EX(lduwa [%o1] %asi, %g1)
+	EX(stwa %g1, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 82b
+	 add		%o1, 4, %o1
+
+85:	retl
+	 clr		%o0
+
+	.align	32
+90:
+	subcc		%o2, 1, %o2
+	EX(lduba [%o1] %asi, %g1)
+	EX(stba %g1, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 90b
+	 add		%o1, 1, %o1
+	retl
+	 clr		%o0
+
+	/* Act like copy_{to,in}_user(), ie. return zero instead
+	 * of original destination pointer.  This is invoked when
+	 * copy_{to,in}_user() finds that %asi is kernel space.
+	 */
+	.globl		memcpy_user_stub
+memcpy_user_stub:
+	save		%sp, -192, %sp
+	mov		%i0, %o0
+	mov		%i1, %o1
+	call		memcpy
+	 mov		%i2, %o2
+	ret
+	 restore	%g0, %g0, %o0
diff -Nru a/arch/sparc64/lib/memmove.S b/arch/sparc64/lib/memmove.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/memmove.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,31 @@
+/* memmove.S: Simple memmove implementation.
+ *
+ * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+
+	.text
+	.align	32
+	.globl	memmove
+memmove:
+	mov		%o0, %g1
+	cmp		%o0, %o1
+	blu,pt		%xcc, memcpy
+	 sub		%o0, %o1, %g5
+	add		%o1, %o2, %g3
+	cmp		%g3, %o0
+	bleu,pt		%xcc, memcpy
+	 add		%o1, %o2, %g5
+	add		%o0, %o2, %o5
+
+	sub		%g5, 1, %o1
+	sub		%o5, 1, %o0
+1:	ldub		[%o1], %g5
+	subcc		%o2, 1, %o2
+	sub		%o1, 1, %o1
+	stb		%g5, [%o0]
+	bne,pt		%icc, 1b
+	 sub		%o0, 1, %o0
+
+	retl
+	 mov		%g1, %o0
diff -Nru a/arch/sparc64/lib/user_fixup.c b/arch/sparc64/lib/user_fixup.c
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/user_fixup.c	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,68 @@
+/* user_fixup.c: Fix up user copy faults.
+ *
+ * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+ */
+
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <asm/uaccess.h>
+
+/* Calculating the exact fault address when using
+ * block loads and stores can be very complicated.
+ * Instead of trying to be clever and handling all
+ * of the cases, just fix things up simply here.
+ */
+
+unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
+{
+	char *dst = to;
+	const char __user *src = from;
+
+	while (size--) {
+		if (__get_user(*dst, src))
+			break;
+		dst++;
+		src++;
+	}
+
+	if (size)
+		memset(dst, 0, size);
+
+	return size;
+}
+
+unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
+{
+	char __user *dst = to;
+	const char *src = from;
+
+	while (size--) {
+		if (__put_user(*src, dst))
+			break;
+		dst++;
+		src++;
+	}
+
+	return size;
+}
+
+unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
+{
+	char __user *dst = to;
+	char __user *src = from;
+
+	while (size--) {
+		char tmp;
+
+		if (__get_user(tmp, src))
+			break;
+		if (__put_user(tmp, dst))
+			break;
+		dst++;
+		src++;
+	}
+
+	return size;
+}
diff -Nru a/drivers/char/synclinkmp.c b/drivers/char/synclinkmp.c
--- a/drivers/char/synclinkmp.c	2003-09-09 09:09:31 -07:00
+++ b/drivers/char/synclinkmp.c	2004-08-26 11:02:57 -07:00
@@ -1,5 +1,5 @@
 /*
- * $Id: synclinkmp.c,v 3.22 2003/09/05 14:04:26 paulkf Exp $
+ * $Id: synclinkmp.c,v 3.23 2004/08/24 19:49:48 paulkf Exp $
  *
  * Device driver for Microgate SyncLink Multiport
  * high speed multiprotocol serial adapter.
@@ -504,7 +504,7 @@
 MODULE_PARM(dosyncppp,"1-" __MODULE_STRING(MAX_DEVICES) "i");
 
 static char *driver_name = "SyncLink MultiPort driver";
-static char *driver_version = "$Revision: 3.22 $";
+static char *driver_version = "$Revision: 3.23 $";
 
 static int __devinit synclinkmp_init_one(struct pci_dev *dev,const struct pci_device_id *ent);
 static void __devexit synclinkmp_remove_one(struct pci_dev *dev);
@@ -681,7 +681,7 @@
 static unsigned char tx_negate_fifo_level = 32;	// tx request FIFO negation level in bytes
 
 static u32 misc_ctrl_value = 0x007e4040;
-static u32 lcr1_brdr_value = 0x0080002d;
+static u32 lcr1_brdr_value = 0x00800029;
 
 static u32 read_ahead_count = 8;
 
@@ -2036,16 +2036,15 @@
 {
  	struct tty_struct *tty = info->tty;
  	struct	mgsl_icount *icount = &info->icount;
-	unsigned char status = read_reg(info, SR1);
-	unsigned char status2 = read_reg(info, SR2);
+	unsigned char status = read_reg(info, SR1) & info->ie1_value & (FLGD + IDLD + CDCD + BRKD);
+	unsigned char status2 = read_reg(info, SR2) & info->ie2_value & OVRN;
 
 	/* clear status bits */
-	if ( status & (FLGD + IDLD + CDCD + BRKD) )
-		write_reg(info, SR1, 
-				(unsigned char)(status & (FLGD + IDLD + CDCD + BRKD)));
+	if (status)
+		write_reg(info, SR1, status);
 
-	if ( status2 & OVRN )
-		write_reg(info, SR2, (unsigned char)(status2 & OVRN));
+	if (status2)
+		write_reg(info, SR2, status2);
 	
 	if ( debug_level >= DEBUG_LEVEL_ISR )
 		printk("%s(%d):%s isr_rxint status=%02X %02x\n",
@@ -2182,15 +2181,22 @@
 		printk("%s(%d):%s isr_txeom status=%02x\n",
 			__FILE__,__LINE__,info->device_name,status);
 
-	/* disable and clear MSCI interrupts */
-	info->ie1_value &= ~(IDLE + UDRN);
-	write_reg(info, IE1, info->ie1_value);
-	write_reg(info, SR1, (unsigned char)(UDRN + IDLE));
-
 	write_reg(info, TXDMA + DIR, 0x00); /* disable Tx DMA IRQs */
 	write_reg(info, TXDMA + DSR, 0xc0); /* clear IRQs and disable DMA */
 	write_reg(info, TXDMA + DCMD, SWABORT);	/* reset/init DMA channel */
 
+	if (status & UDRN) {
+		write_reg(info, CMD, TXRESET);
+		write_reg(info, CMD, TXENABLE);
+	} else
+		write_reg(info, CMD, TXBUFCLR);
+
+	/* disable and clear tx interrupts */
+	info->ie0_value &= ~TXRDYE;
+	info->ie1_value &= ~(IDLE + UDRN);
+	write_reg16(info, IE0, (unsigned short)((info->ie1_value << 8) + info->ie0_value));
+	write_reg(info, SR1, (unsigned char)(UDRN + IDLE));
+
 	if ( info->tx_active ) {
 		if (info->params.mode != MGSL_MODE_ASYNC) {
 			if (status & UDRN)
@@ -2231,10 +2237,10 @@
  */
 void isr_txint(SLMP_INFO * info)
 {
-	unsigned char status = read_reg(info, SR1);
+	unsigned char status = read_reg(info, SR1) & info->ie1_value & (UDRN + IDLE + CCTS);
 
 	/* clear status bits */
-	write_reg(info, SR1, (unsigned char)(status & (UDRN + IDLE + CCTS)));
+	write_reg(info, SR1, status);
 
 	if ( debug_level >= DEBUG_LEVEL_ISR )
 		printk("%s(%d):%s isr_txint status=%02x\n",
@@ -2263,6 +2269,14 @@
 		printk("%s(%d):%s isr_txrdy() tx_count=%d\n",
 			__FILE__,__LINE__,info->device_name,info->tx_count);
 
+	if (info->params.mode != MGSL_MODE_ASYNC) {
+		/* disable TXRDY IRQ, enable IDLE IRQ */
+		info->ie0_value &= ~TXRDYE;
+		info->ie1_value |= IDLE;
+		write_reg16(info, IE0, (unsigned short)((info->ie1_value << 8) + info->ie0_value));
+		return;
+	}
+
 	if (info->tty && (info->tty->stopped || info->tty->hw_stopped)) {
 		tx_stop(info);
 		return;
@@ -2317,13 +2331,6 @@
 
 void isr_txdmaok(SLMP_INFO * info)
 {
-	/* BIT7 = EOT (end of transfer, used for async mode)
-	 * BIT6 = EOM (end of message/frame, used for sync mode)
-	 *
-	 * We don't look at DMA status because only EOT is enabled
-	 * and we always clear and disable all tx DMA IRQs.
-	 */
-//	unsigned char dma_status = read_reg(info,TXDMA + DSR) & 0xc0;
 	unsigned char status_reg1 = read_reg(info, SR1);
 
 	write_reg(info, TXDMA + DIR, 0x00);	/* disable Tx DMA IRQs */
@@ -2334,19 +2341,10 @@
 		printk("%s(%d):%s isr_txdmaok(), status=%02x\n",
 			__FILE__,__LINE__,info->device_name,status_reg1);
 
-	/* If transmitter already idle, do end of frame processing,
-	 * otherwise enable interrupt for tx IDLE.
-	 */
-	if (status_reg1 & IDLE)
-		isr_txeom(info, IDLE);
-	else {
-		/* disable and clear underrun IRQ, enable IDLE interrupt */
-		info->ie1_value |= IDLE;
-		info->ie1_value &= ~UDRN;
-		write_reg(info, IE1, info->ie1_value);
-
-		write_reg(info, SR1, UDRN);
-	}
+	/* program TXRDY as FIFO empty flag, enable TXRDY IRQ */
+	write_reg16(info, TRC0, 0);
+	info->ie0_value |= TXRDYE;
+	write_reg(info, IE0, info->ie0_value);
 }
 
 void isr_txdmaerror(SLMP_INFO * info)
@@ -3037,7 +3035,7 @@
 		unsigned char oldval = info->ie1_value;
 		unsigned char newval = oldval +
 			 (mask & MgslEvent_ExitHuntMode ? FLGD:0) +
-			 (mask & MgslEvent_IdleReceived ? IDLE:0);
+			 (mask & MgslEvent_IdleReceived ? IDLD:0);
 		if ( oldval != newval ) {
 			info->ie1_value = newval;
 			write_reg(info, IE1, info->ie1_value);
@@ -3104,7 +3102,7 @@
 		spin_lock_irqsave(&info->lock,flags);
 		if (!waitqueue_active(&info->event_wait_q)) {
 			/* disable enable exit hunt mode/idle rcvd IRQs */
-			info->ie1_value &= ~(FLGD|IDLE);
+			info->ie1_value &= ~(FLGD|IDLD);
 			write_reg(info, IE1, info->ie1_value);
 		}
 		spin_unlock_irqrestore(&info->lock,flags);
@@ -3554,9 +3552,10 @@
 
 int claim_resources(SLMP_INFO *info)
 {
-	if (request_mem_region(info->phys_memory_base,0x40000,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_memory_base,SCA_MEM_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_memory_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
@@ -3565,22 +3564,25 @@
 	if (request_mem_region(info->phys_lcr_base + info->lcr_offset,128,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s lcr mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_lcr_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
 		info->lcr_mem_requested = 1;
 
-	if (request_mem_region(info->phys_sca_base + info->sca_offset,512,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_sca_base + info->sca_offset,SCA_BASE_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s sca mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_sca_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
 		info->sca_base_requested = 1;
 
-	if (request_mem_region(info->phys_statctrl_base + info->statctrl_offset,16,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_statctrl_base + info->statctrl_offset,SCA_REG_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s stat/ctrl mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_statctrl_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
@@ -3590,33 +3592,41 @@
 	if (!info->memory_base) {
 		printk( "%s(%d):%s Cant map shared memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
 
-	if ( !memory_test(info) ) {
-		printk( "%s(%d):Shared Memory Test failed for device %s MemAddr=%08X\n",
-			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
-		goto errout;
-	}
-
-	info->lcr_base = ioremap(info->phys_lcr_base,PAGE_SIZE) + info->lcr_offset;
+	info->lcr_base = ioremap(info->phys_lcr_base,PAGE_SIZE);
 	if (!info->lcr_base) {
 		printk( "%s(%d):%s Cant map LCR memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_lcr_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
+	info->lcr_base += info->lcr_offset;
 
-	info->sca_base = ioremap(info->phys_sca_base,PAGE_SIZE) + info->sca_offset;
+	info->sca_base = ioremap(info->phys_sca_base,PAGE_SIZE);
 	if (!info->sca_base) {
 		printk( "%s(%d):%s Cant map SCA memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_sca_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
+	info->sca_base += info->sca_offset;
 
-	info->statctrl_base = ioremap(info->phys_statctrl_base,PAGE_SIZE) + info->statctrl_offset;
+	info->statctrl_base = ioremap(info->phys_statctrl_base,PAGE_SIZE);
 	if (!info->statctrl_base) {
 		printk( "%s(%d):%s Cant map SCA Status/Control memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_statctrl_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
+		goto errout;
+	}
+	info->statctrl_base += info->statctrl_offset;
+
+	if ( !memory_test(info) ) {
+		printk( "%s(%d):Shared Memory Test failed for device %s MemAddr=%08X\n",
+			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
+		info->init_error = DiagStatus_MemoryError;
 		goto errout;
 	}
 
@@ -3639,7 +3649,7 @@
 	}
 
 	if ( info->shared_mem_requested ) {
-		release_mem_region(info->phys_memory_base,0x40000);
+		release_mem_region(info->phys_memory_base,SCA_MEM_SIZE);
 		info->shared_mem_requested = 0;
 	}
 	if ( info->lcr_mem_requested ) {
@@ -3647,11 +3657,11 @@
 		info->lcr_mem_requested = 0;
 	}
 	if ( info->sca_base_requested ) {
-		release_mem_region(info->phys_sca_base + info->sca_offset,512);
+		release_mem_region(info->phys_sca_base + info->sca_offset,SCA_BASE_SIZE);
 		info->sca_base_requested = 0;
 	}
 	if ( info->sca_statctrl_requested ) {
-		release_mem_region(info->phys_statctrl_base + info->statctrl_offset,16);
+		release_mem_region(info->phys_statctrl_base + info->statctrl_offset,SCA_REG_SIZE);
 		info->sca_statctrl_requested = 0;
 	}
 
@@ -3982,34 +3992,25 @@
 		       __FILE__,__LINE__,rc);
 	restore_flags(flags);
 
+	/* reset devices */
 	info = synclinkmp_device_list;
 	while(info) {
-#ifdef CONFIG_SYNCLINK_SYNCPPP
-		if (info->dosyncppp)
-			sppp_delete(info);
-#endif
 		reset_port(info);
-		if ( info->port_num == 0 ) {
-			if ( info->irq_requested ) {
-				free_irq(info->irq_level, info);
-				info->irq_requested = 0;
-			}
-		}
 		info = info->next_device;
 	}
 
-	/* port 0 of each adapter originally claimed
-	 * all resources, release those now
-	 */
+	/* release devices */
 	info = synclinkmp_device_list;
 	while(info) {
+#ifdef CONFIG_SYNCLINK_SYNCPPP
+		if (info->dosyncppp)
+			sppp_delete(info);
+#endif
 		free_dma_bufs(info);
 		free_tmp_rx_buf(info);
 		if ( info->port_num == 0 ) {
-			spin_lock_irqsave(&info->lock,flags);
-			reset_adapter(info);
-			write_reg(info, LPR, 1);		/* set low power mode */
-			spin_unlock_irqrestore(&info->lock,flags);
+			if (info->sca_base)
+				write_reg(info, LPR, 1); /* set low power mode */
 			release_resources(info);
 		}
 		tmp = info;
@@ -4229,6 +4230,9 @@
 				}
 			}
 
+			write_reg16(info, TRC0,
+				(unsigned short)(((tx_negate_fifo_level-1)<<8) + tx_active_fifo_level));
+
 			write_reg(info, TXDMA + DSR, 0); 		/* disable DMA channel */
 			write_reg(info, TXDMA + DCMD, SWABORT);	/* reset/init DMA channel */
 	
@@ -4240,11 +4244,10 @@
 			write_reg16(info, TXDMA + EDA,
 				info->tx_buf_list_ex[info->last_tx_buf].phys_entry);
 	
-			/* clear IDLE and UDRN status bit */
-			info->ie1_value &= ~(IDLE + UDRN);
-			if (info->params.mode != MGSL_MODE_ASYNC)
-				info->ie1_value |= UDRN;     		/* HDLC, IRQ on underrun */
-			write_reg(info, IE1, info->ie1_value);	/* enable MSCI interrupts */
+			/* enable underrun IRQ */
+			info->ie1_value &= ~IDLE;
+			info->ie1_value |= UDRN;
+			write_reg(info, IE1, info->ie1_value);
 			write_reg(info, SR1, (unsigned char)(IDLE + UDRN));
 	
 			write_reg(info, TXDMA + DIR, 0x40);		/* enable Tx DMA interrupts (EOM) */
diff -Nru a/drivers/char/vc_screen.c b/drivers/char/vc_screen.c
--- a/drivers/char/vc_screen.c	2004-07-30 06:29:39 -07:00
+++ b/drivers/char/vc_screen.c	2004-08-26 06:29:53 -07:00
@@ -136,10 +136,12 @@
 	if (!vc_cons_allocated(currcons))
 		goto unlock_out;
 
-	read = 0;
-	ret = 0;
+	ret = -EINVAL;
 	if (pos != n)
 		goto unlock_out;
+
+	read = 0;
+	ret = 0;
 	while (count) {
 		char *con_buf0, *con_buf_start;
 		long this_round, size;
diff -Nru a/drivers/hotplug/ibmphp_res.c b/drivers/hotplug/ibmphp_res.c
--- a/drivers/hotplug/ibmphp_res.c	2003-08-28 04:21:26 -07:00
+++ b/drivers/hotplug/ibmphp_res.c	2004-08-26 08:10:16 -07:00
@@ -42,7 +42,7 @@
 static int update_bridge_ranges (struct bus_node **);
 static int add_range (int type, struct range_node *, struct bus_node *);
 static void fix_resources (struct bus_node *);
-static inline struct bus_node *find_bus_wprev (u8, struct bus_node **, u8);
+static struct bus_node *find_bus_wprev (u8, struct bus_node **, u8);
 
 static LIST_HEAD(gbuses);
 LIST_HEAD(ibmphp_res_head);
@@ -1757,7 +1757,7 @@
 	return find_bus_wprev (bus_number, NULL, 0);
 }
 
-static inline struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u8 flag)
+static struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u8 flag)
 {
 	struct bus_node *bus_cur;
 	struct list_head *tmp;
diff -Nru a/drivers/isdn/hysdn/hysdn_procconf.c b/drivers/isdn/hysdn/hysdn_procconf.c
--- a/drivers/isdn/hysdn/hysdn_procconf.c	2004-07-30 06:29:39 -07:00
+++ b/drivers/isdn/hysdn/hysdn_procconf.c	2004-08-26 06:42:04 -07:00
@@ -219,20 +219,25 @@
 	if (off != &file->f_pos)	/* fs error check */
 		return -ESPIPE;
 
+	if (!(file->f_mode & FMODE_READ))
+		return -EPERM;
+
 	if (!(cp = file->private_data))
 		return (-EFAULT);	/* should never happen */
+
 	i = strlen(cp);	/* get total string length */
-	if (pos == (unsigned)pos && pos < i) {
-		/* still bytes to transfer */
-		cp += pos;	/* point to desired data offset */
-		i -= pos;	/* remaining length */
-		if (i > count)
-			i = count;	/* limit length to transfer */
-		if (copy_to_user(buf, cp, i))
-			return (-EFAULT);	/* copy error */
-		*off = pos + i;	/* adjust offset */
-	} else
-		return (0);
+
+	if (pos != (unsigned)pos || pos >= i)
+		return 0;
+
+	/* still bytes to transfer */
+	cp += pos;	/* point to desired data offset */
+	i -= pos;	/* remaining length */
+	if (i > count)
+		i = count;	/* limit length to transfer */
+	if (copy_to_user(buf, cp, i))
+		return (-EFAULT);	/* copy error */
+	*off = pos + i;	/* adjust offset */
 
 	return (i);
 }				/* hysdn_conf_read */
diff -Nru a/drivers/net/ne2k-pci.c b/drivers/net/ne2k-pci.c
--- a/drivers/net/ne2k-pci.c	2003-11-18 12:16:28 -08:00
+++ b/drivers/net/ne2k-pci.c	2004-08-26 16:50:38 -07:00
@@ -505,8 +505,12 @@
 		insl(NE_BASE + NE_DATAPORT, buf, count>>2);
 		if (count & 3) {
 			buf += count & ~3;
-			if (count & 2)
-				*((u16*)buf)++ = le16_to_cpu(inw(NE_BASE + NE_DATAPORT));
+			if (count & 2) {
+				u16 *b = (u16 *)buf;
+
+				*b++ = le16_to_cpu(inw(NE_BASE + NE_DATAPORT));
+				buf = (char *)b;
+			}
 			if (count & 1)
 				*buf = inb(NE_BASE + NE_DATAPORT);
 		}
@@ -566,8 +570,12 @@
 		outsl(NE_BASE + NE_DATAPORT, buf, count>>2);
 		if (count & 3) {
 			buf += count & ~3;
-			if (count & 2)
-				outw(cpu_to_le16(*((u16*)buf)++), NE_BASE + NE_DATAPORT);
+			if (count & 2) {
+				u16 *b = (u16 *)buf;
+
+				outw(cpu_to_le16(*b++), NE_BASE + NE_DATAPORT);
+				buf = (char *)b;
+			}
 		}
 	}
 
diff -Nru a/drivers/net/wan/dscc4.c b/drivers/net/wan/dscc4.c
--- a/drivers/net/wan/dscc4.c	2003-10-04 06:02:06 -07:00
+++ b/drivers/net/wan/dscc4.c	2004-08-26 11:18:53 -07:00
@@ -351,8 +351,8 @@
 #endif
 
 /* Functions prototypes */
-static inline void dscc4_rx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
-static inline void dscc4_tx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
+static void dscc4_rx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
+static void dscc4_tx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
 static int dscc4_found1(struct pci_dev *, unsigned long ioaddr);
 static int dscc4_init_one(struct pci_dev *, const struct pci_device_id *ent);
 static int dscc4_open(struct net_device *);
@@ -366,7 +366,6 @@
 static void dscc4_irq(int irq, void *dev_id, struct pt_regs *ptregs);
 static int dscc4_hdlc_attach(hdlc_device *, unsigned short, unsigned short);
 static int dscc4_set_iface(struct dscc4_dev_priv *, struct net_device *);
-static inline int dscc4_set_quartz(struct dscc4_dev_priv *, int);
 #ifdef DSCC4_POLLING
 static int dscc4_tx_poll(struct dscc4_dev_priv *, struct net_device *);
 #endif
@@ -857,6 +856,18 @@
 	//scc_writel(0x00250008 & ~RxActivate, dpriv, dev, CCR2);
 }
 
+static inline int dscc4_set_quartz(struct dscc4_dev_priv *dpriv, int hz)
+{
+	int ret = 0;
+
+	if ((hz < 0) || (hz > DSCC4_HZ_MAX))
+		ret = -EOPNOTSUPP;
+	else
+		dpriv->pci_priv->xtal_hz = hz;
+
+	return ret;
+}
+
 static int dscc4_found1(struct pci_dev *pdev, unsigned long ioaddr)
 {
 	struct dscc4_pci_priv *ppriv;
@@ -1325,18 +1336,6 @@
 	return ret;
 }
 
-static inline int dscc4_set_quartz(struct dscc4_dev_priv *dpriv, int hz)
-{
-	int ret = 0;
-
-	if ((hz < 0) || (hz > DSCC4_HZ_MAX))
-		ret = -EOPNOTSUPP;
-	else
-		dpriv->pci_priv->xtal_hz = hz;
-
-	return ret;
-}
-
 static int dscc4_match(struct thingie *p, int value)
 {
 	int i;
@@ -1512,7 +1511,7 @@
 	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
-static inline void dscc4_tx_irq(struct dscc4_pci_priv *ppriv,
+static void dscc4_tx_irq(struct dscc4_pci_priv *ppriv,
 				struct dscc4_dev_priv *dpriv)
 {
 	struct net_device *dev = hdlc_to_dev(&dpriv->hdlc);
@@ -1681,7 +1680,7 @@
 	goto try;
 }
 
-static inline void dscc4_rx_irq(struct dscc4_pci_priv *priv,
+static void dscc4_rx_irq(struct dscc4_pci_priv *priv,
 				    struct dscc4_dev_priv *dpriv)
 {
 	struct net_device *dev = hdlc_to_dev(&dpriv->hdlc);
diff -Nru a/drivers/net/wan/lmc/lmc_debug.c b/drivers/net/wan/lmc/lmc_debug.c
--- a/drivers/net/wan/lmc/lmc_debug.c	2002-03-11 06:13:28 -08:00
+++ b/drivers/net/wan/lmc/lmc_debug.c	2004-08-26 09:59:47 -07:00
@@ -66,7 +66,7 @@
 #endif
 }
 
-inline void lmc_trace(struct net_device *dev, char *msg){
+void lmc_trace(struct net_device *dev, char *msg){
 #ifdef LMC_TRACE
     unsigned long j = jiffies + 3; /* Wait for 50 ms */
 
diff -Nru a/drivers/net/wan/lmc/lmc_debug.h b/drivers/net/wan/lmc/lmc_debug.h
--- a/drivers/net/wan/lmc/lmc_debug.h	2002-02-05 09:40:01 -08:00
+++ b/drivers/net/wan/lmc/lmc_debug.h	2004-08-26 09:58:32 -07:00
@@ -47,6 +47,6 @@
 
 void lmcConsoleLog(char *type, unsigned char *ucData, int iLen);
 void lmcEventLog (u_int32_t EventNum, u_int32_t arg2, u_int32_t arg3);
-inline void lmc_trace(struct net_device *dev, char *msg);
+void lmc_trace(struct net_device *dev, char *msg);
 
 #endif
diff -Nru a/drivers/scsi/53c7,8xx.c b/drivers/scsi/53c7,8xx.c
--- a/drivers/scsi/53c7,8xx.c	2003-09-02 11:27:25 -07:00
+++ b/drivers/scsi/53c7,8xx.c	2004-08-26 16:50:38 -07:00
@@ -3926,7 +3926,7 @@
 	restore_flags (flags);
 	cmd->result = le32_to_cpu(0xffff);	/* The NCR will overwrite message
 				       and status with valid data */
-	cmd->host_scribble = (unsigned char *) tmp = create_cmd (cmd);
+	cmd->host_scribble = (unsigned char *) create_cmd (cmd);
     }
     cli();
     /*
diff -Nru a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
--- a/drivers/scsi/advansys.c	2004-02-01 07:56:53 -08:00
+++ b/drivers/scsi/advansys.c	2004-08-26 16:50:38 -07:00
@@ -6118,8 +6118,8 @@
     } else {
         /* Append to 'done_scp' at the end with 'last_scp'. */
         ASC_ASSERT(last_scp != NULL);
-        REQPNEXT(last_scp) = asc_dequeue_list(&boardp->active,
-            &new_last_scp, ASC_TID_ALL);
+        last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->active, &new_last_scp, ASC_TID_ALL);
         if (new_last_scp != NULL) {
             ASC_ASSERT(REQPNEXT(last_scp) != NULL);
             for (tscp = REQPNEXT(last_scp); tscp; tscp = REQPNEXT(tscp)) {
@@ -6141,8 +6141,8 @@
     } else {
         /* Append to 'done_scp' at the end with 'last_scp'. */
         ASC_ASSERT(last_scp != NULL);
-        REQPNEXT(last_scp) = asc_dequeue_list(&boardp->waiting,
-            &new_last_scp, ASC_TID_ALL);
+        last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->waiting, &new_last_scp, ASC_TID_ALL);
         if (new_last_scp != NULL) {
             ASC_ASSERT(REQPNEXT(last_scp) != NULL);
             for (tscp = REQPNEXT(last_scp); tscp; tscp = REQPNEXT(tscp)) {
@@ -6394,8 +6394,8 @@
                     ASC_TID_ALL);
             } else {
                 ASC_ASSERT(last_scp != NULL);
-                REQPNEXT(last_scp) = asc_dequeue_list(&boardp->done,
-                    &new_last_scp, ASC_TID_ALL);
+                last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->done, &new_last_scp, ASC_TID_ALL);
                 if (new_last_scp != NULL) {
                     ASC_ASSERT(REQPNEXT(last_scp) != NULL);
                     last_scp = new_last_scp;
@@ -6466,7 +6466,7 @@
     while (scp != NULL) {
         ASC_DBG1(3, "asc_scsi_done_list: scp 0x%lx\n", (ulong) scp);
         tscp = REQPNEXT(scp);
-        REQPNEXT(scp) = NULL;
+        scp->host_scribble = NULL;
         ASC_STATS(scp->host, done);
         ASC_ASSERT(scp->scsi_done != NULL);
         scp->scsi_done(scp);
@@ -7511,7 +7511,7 @@
     tid = REQPTID(reqp);
     ASC_ASSERT(tid >= 0 && tid <= ADV_MAX_TID);
     if (flag == ASC_FRONT) {
-        REQPNEXT(reqp) = ascq->q_first[tid];
+        reqp->host_scribble = (unsigned char *)ascq->q_first[tid];
         ascq->q_first[tid] = reqp;
         /* If the queue was empty, set the last pointer. */
         if (ascq->q_last[tid] == NULL) {
@@ -7519,10 +7519,10 @@
         }
     } else { /* ASC_BACK */
         if (ascq->q_last[tid] != NULL) {
-            REQPNEXT(ascq->q_last[tid]) = reqp;
+            ascq->q_last[tid]->host_scribble = (unsigned char *)reqp;
         }
         ascq->q_last[tid] = reqp;
-        REQPNEXT(reqp) = NULL;
+        reqp->host_scribble = NULL;
         /* If the queue was empty, set the first pointer. */
         if (ascq->q_first[tid] == NULL) {
             ascq->q_first[tid] = reqp;
@@ -7643,7 +7643,7 @@
                     lastp = ascq->q_last[i];
                 } else {
                     ASC_ASSERT(lastp != NULL);
-                    REQPNEXT(lastp) = ascq->q_first[i];
+                    lastp->host_scribble = (unsigned char *)ascq->q_first[i];
                     lastp = ascq->q_last[i];
                 }
                 ascq->q_first[i] = ascq->q_last[i] = NULL;
@@ -7721,8 +7721,8 @@
              currp; prevp = currp, currp = REQPNEXT(currp)) {
             if (currp == reqp) {
                 ret = ASC_TRUE;
-                REQPNEXT(prevp) = REQPNEXT(currp);
-                REQPNEXT(reqp) = NULL;
+                prevp->host_scribble = (unsigned char *)REQPNEXT(currp);
+                reqp->host_scribble = NULL;
                 if (ascq->q_last[tid] == reqp) {
                     ascq->q_last[tid] = prevp;
                 }
diff -Nru a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
--- a/drivers/scsi/dpt_i2o.c	2003-03-31 05:20:33 -08:00
+++ b/drivers/scsi/dpt_i2o.c	2004-08-26 16:50:38 -07:00
@@ -434,7 +434,7 @@
 			cmd->scsi_done(cmd);
 			return 0;
 		}
-		(struct adpt_device*)(cmd->device->hostdata) = pDev;
+		cmd->device->hostdata = pDev;
 	}
 	pDev->pScsi_dev = cmd->device;
 
@@ -2194,7 +2194,7 @@
 		printk ("%s: scsi_register returned NULL\n",pHba->name);
 		return -1;
 	}
-	(adpt_hba*)(host->hostdata[0]) = pHba;
+	host->hostdata[0] = (unsigned long)pHba;
 	pHba->host = host;
 
 	host->irq = pHba->pDev->irq;;
diff -Nru a/drivers/scsi/ips.c b/drivers/scsi/ips.c
--- a/drivers/scsi/ips.c	2004-08-16 14:09:52 -07:00
+++ b/drivers/scsi/ips.c	2004-08-20 00:18:04 -07:00
@@ -133,6 +133,10 @@
 /* 6.10.00  - Remove 1G Addressing Limitations                               */
 /* 6.11.xx  - Get VersionInfo buffer off the stack !              DDTS 60401 */
 /* 6.11.xx  - Make Logical Drive Info structure safe for DMA      DDTS 60639 */
+/* 7.10.xx  - Add highmem_io flag in SCSI Templete for 2.4 kernels           */
+/*          - Fix path/name for scsi_hosts.h include for 2.6 kernels         */
+/*          - Fix sort order of 7k                                           */
+/*          - Remove 3 unused "inline" functions                             */
 /*****************************************************************************/
 
 /*
@@ -176,7 +180,13 @@
 #include <scsi/sg.h>
 
 #include "scsi.h"
+
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(2,5,0)
 #include "hosts.h"
+#else
+#include <scsi/scsi_host.h>
+#endif
+
 #include "ips.h"
 
 #include <linux/module.h>
@@ -197,8 +207,8 @@
 /*
  * DRIVER_VER
  */
-#define IPS_VERSION_HIGH        "7.00"
-#define IPS_VERSION_LOW         ".15 "
+#define IPS_VERSION_HIGH        "7.10"
+#define IPS_VERSION_LOW         ".18 "
 
 #if !defined(__i386__) && !defined(__ia64__) && !defined(__x86_64__)
 #warning "This driver has only been tested on the x86/ia64/x86_64 platforms"
@@ -277,9 +287,9 @@
 	.use_clustering		= ENABLE_CLUSTERING,
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
 	.use_new_eh_code	= 1,
-#ifdef RHEL3
-    .vary_io            = 1,
 #endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,20)  &&  LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
+    .highmem_io          = 1,   
 #endif
 };
 
@@ -477,21 +487,17 @@
 static uint32_t ips_statupd_copperhead_memio(ips_ha_t *);
 static uint32_t ips_statupd_morpheus(ips_ha_t *);
 static ips_scb_t *ips_getscb(ips_ha_t *);
-static inline void ips_putq_scb_head(ips_scb_queue_t *, ips_scb_t *);
-static inline void ips_putq_scb_tail(ips_scb_queue_t *, ips_scb_t *);
-static inline void ips_putq_wait_head(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline void ips_putq_wait_tail(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline void ips_putq_copp_head(ips_copp_queue_t *,
-				      ips_copp_wait_item_t *);
-static inline void ips_putq_copp_tail(ips_copp_queue_t *,
+static void ips_putq_scb_head(ips_scb_queue_t *, ips_scb_t *);
+static void ips_putq_wait_tail(ips_wait_queue_t *, Scsi_Cmnd *);
+static void ips_putq_copp_tail(ips_copp_queue_t *,
 				      ips_copp_wait_item_t *);
-static inline ips_scb_t *ips_removeq_scb_head(ips_scb_queue_t *);
-static inline ips_scb_t *ips_removeq_scb(ips_scb_queue_t *, ips_scb_t *);
-static inline Scsi_Cmnd *ips_removeq_wait_head(ips_wait_queue_t *);
-static inline Scsi_Cmnd *ips_removeq_wait(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline ips_copp_wait_item_t *ips_removeq_copp(ips_copp_queue_t *,
+static ips_scb_t *ips_removeq_scb_head(ips_scb_queue_t *);
+static ips_scb_t *ips_removeq_scb(ips_scb_queue_t *, ips_scb_t *);
+static Scsi_Cmnd *ips_removeq_wait_head(ips_wait_queue_t *);
+static Scsi_Cmnd *ips_removeq_wait(ips_wait_queue_t *, Scsi_Cmnd *);
+static ips_copp_wait_item_t *ips_removeq_copp(ips_copp_queue_t *,
 						     ips_copp_wait_item_t *);
-static inline ips_copp_wait_item_t *ips_removeq_copp_head(ips_copp_queue_t *);
+static ips_copp_wait_item_t *ips_removeq_copp_head(ips_copp_queue_t *);
 
 static int ips_is_passthru(Scsi_Cmnd *);
 static int ips_make_passthru(ips_ha_t *, Scsi_Cmnd *, ips_scb_t *, int);
@@ -802,168 +808,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_removeq_copp_head                                      */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_copp_wait_item_t *
-ips_removeq_copp_head(ips_copp_queue_t * queue)
-{
-	ips_copp_wait_item_t *item;
-
-	METHOD_TRACE("ips_removeq_copp_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = item->next;
-	item->next = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_copp                                           */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_copp_wait_item_t *
-ips_removeq_copp(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
-{
-	ips_copp_wait_item_t *p;
-
-	METHOD_TRACE("ips_removeq_copp", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_copp_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != p->next))
-		p = p->next;
-
-	if (p) {
-		/* found a match */
-		p->next = item->next;
-
-		if (!item->next)
-			queue->tail = p;
-
-		item->next = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_wait_head                                      */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline Scsi_Cmnd *
-ips_removeq_wait_head(ips_wait_queue_t * queue)
-{
-	Scsi_Cmnd *item;
-
-	METHOD_TRACE("ips_removeq_wait_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = (Scsi_Cmnd *) item->host_scribble;
-	item->host_scribble = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_wait                                           */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline Scsi_Cmnd *
-ips_removeq_wait(ips_wait_queue_t * queue, Scsi_Cmnd * item)
-{
-	Scsi_Cmnd *p;
-
-	METHOD_TRACE("ips_removeq_wait", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_wait_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != (Scsi_Cmnd *) p->host_scribble))
-		p = (Scsi_Cmnd *) p->host_scribble;
-
-	if (p) {
-		/* found a match */
-		p->host_scribble = item->host_scribble;
-
-		if (!item->host_scribble)
-			queue->tail = p;
-
-		item->host_scribble = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_eh_abort                                               */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -1021,87 +865,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_removeq_scb_head                                       */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_scb_t *
-ips_removeq_scb_head(ips_scb_queue_t * queue)
-{
-	ips_scb_t *item;
-
-	METHOD_TRACE("ips_removeq_scb_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = item->q_next;
-	item->q_next = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_scb                                            */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_scb_t *
-ips_removeq_scb(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	ips_scb_t *p;
-
-	METHOD_TRACE("ips_removeq_scb", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_scb_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != p->q_next))
-		p = p->q_next;
-
-	if (p) {
-		/* found a match */
-		p->q_next = item->q_next;
-
-		if (!item->q_next)
-			queue->tail = p;
-
-		item->q_next = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_eh_reset                                               */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -1301,70 +1064,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_copp_tail                                         */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_copp_tail(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
-{
-	METHOD_TRACE("ips_putq_copp_tail", 1);
-
-	if (!item)
-		return;
-
-	item->next = NULL;
-
-	if (queue->tail)
-		queue->tail->next = item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_putq_wait_tail                                         */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_wait_tail(ips_wait_queue_t * queue, Scsi_Cmnd * item)
-{
-	METHOD_TRACE("ips_putq_wait_tail", 1);
-
-	if (!item)
-		return;
-
-	item->host_scribble = NULL;
-
-	if (queue->tail)
-		queue->tail->host_scribble = (char *) item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_queue                                                  */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -2195,7 +1894,7 @@
 /*   Fill in a single scb sg_list element from an address                   */
 /*   return a -1 if a breakup occurred                                      */
 /****************************************************************************/
-static inline int
+static int
 ips_fill_scb_sg_single(ips_ha_t * ha, dma_addr_t busaddr,
 		       ips_scb_t * scb, int indx, unsigned int e_len)
 {
@@ -3000,66 +2699,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_scb_head                                          */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the head of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_scb_head(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	METHOD_TRACE("ips_putq_scb_head", 1);
-
-	if (!item)
-		return;
-
-	item->q_next = queue->head;
-	queue->head = item;
-
-	if (!queue->tail)
-		queue->tail = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_putq_scb_tail                                          */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_scb_tail(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	METHOD_TRACE("ips_putq_scb_tail", 1);
-
-	if (!item)
-		return;
-
-	item->q_next = NULL;
-
-	if (queue->tail)
-		queue->tail->q_next = item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_next                                                   */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -3311,7 +2950,7 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_wait_head                                         */
+/* Routine Name: ips_putq_scb_head                                          */
 /*                                                                          */
 /* Routine Description:                                                     */
 /*                                                                          */
@@ -3320,15 +2959,15 @@
 /* ASSUMED to be called from within the HA lock                             */
 /*                                                                          */
 /****************************************************************************/
-static inline void
-ips_putq_wait_head(ips_wait_queue_t * queue, Scsi_Cmnd * item)
+static void
+ips_putq_scb_head(ips_scb_queue_t * queue, ips_scb_t * item)
 {
-	METHOD_TRACE("ips_putq_wait_head", 1);
+	METHOD_TRACE("ips_putq_scb_head", 1);
 
 	if (!item)
 		return;
 
-	item->host_scribble = (char *) queue->head;
+	item->q_next = queue->head;
 	queue->head = item;
 
 	if (!queue->tail)
@@ -3339,34 +2978,313 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_copp_head                                         */
+/* Routine Name: ips_removeq_scb_head                                       */
 /*                                                                          */
 /* Routine Description:                                                     */
 /*                                                                          */
-/*   Add an item to the head of the queue                                   */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_scb_t *
+ips_removeq_scb_head(ips_scb_queue_t * queue)
+{
+	ips_scb_t *item;
+
+	METHOD_TRACE("ips_removeq_scb_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = item->q_next;
+	item->q_next = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_scb                                            */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_scb_t *
+ips_removeq_scb(ips_scb_queue_t * queue, ips_scb_t * item)
+{
+	ips_scb_t *p;
+
+	METHOD_TRACE("ips_removeq_scb", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_scb_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != p->q_next))
+		p = p->q_next;
+
+	if (p) {
+		/* found a match */
+		p->q_next = item->q_next;
+
+		if (!item->q_next)
+			queue->tail = p;
+
+		item->q_next = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_putq_wait_tail                                         */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Add an item to the tail of the queue                                   */
 /*                                                                          */
 /* ASSUMED to be called from within the HA lock                             */
 /*                                                                          */
 /****************************************************************************/
-static inline void
-ips_putq_copp_head(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+static void
+ips_putq_wait_tail(ips_wait_queue_t * queue, Scsi_Cmnd * item)
 {
-	METHOD_TRACE("ips_putq_copp_head", 1);
+	METHOD_TRACE("ips_putq_wait_tail", 1);
 
 	if (!item)
 		return;
 
-	item->next = queue->head;
-	queue->head = item;
+	item->host_scribble = NULL;
 
-	if (!queue->tail)
-		queue->tail = item;
+	if (queue->tail)
+		queue->tail->host_scribble = (char *) item;
+
+	queue->tail = item;
+
+	if (!queue->head)
+		queue->head = item;
 
 	queue->count++;
 }
 
 /****************************************************************************/
 /*                                                                          */
+/* Routine Name: ips_removeq_wait_head                                      */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static Scsi_Cmnd *
+ips_removeq_wait_head(ips_wait_queue_t * queue)
+{
+	Scsi_Cmnd *item;
+
+	METHOD_TRACE("ips_removeq_wait_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = (Scsi_Cmnd *) item->host_scribble;
+	item->host_scribble = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_wait                                           */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static Scsi_Cmnd *
+ips_removeq_wait(ips_wait_queue_t * queue, Scsi_Cmnd * item)
+{
+	Scsi_Cmnd *p;
+
+	METHOD_TRACE("ips_removeq_wait", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_wait_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != (Scsi_Cmnd *) p->host_scribble))
+		p = (Scsi_Cmnd *) p->host_scribble;
+
+	if (p) {
+		/* found a match */
+		p->host_scribble = item->host_scribble;
+
+		if (!item->host_scribble)
+			queue->tail = p;
+
+		item->host_scribble = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_putq_copp_tail                                         */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Add an item to the tail of the queue                                   */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static void
+ips_putq_copp_tail(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+{
+	METHOD_TRACE("ips_putq_copp_tail", 1);
+
+	if (!item)
+		return;
+
+	item->next = NULL;
+
+	if (queue->tail)
+		queue->tail->next = item;
+
+	queue->tail = item;
+
+	if (!queue->head)
+		queue->head = item;
+
+	queue->count++;
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_copp_head                                      */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_copp_wait_item_t *
+ips_removeq_copp_head(ips_copp_queue_t * queue)
+{
+	ips_copp_wait_item_t *item;
+
+	METHOD_TRACE("ips_removeq_copp_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = item->next;
+	item->next = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_copp                                           */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_copp_wait_item_t *
+ips_removeq_copp(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+{
+	ips_copp_wait_item_t *p;
+
+	METHOD_TRACE("ips_removeq_copp", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_copp_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != p->next))
+		p = p->next;
+
+	if (p) {
+		/* found a match */
+		p->next = item->next;
+
+		if (!item->next)
+			queue->tail = p;
+
+		item->next = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
 /* Routine Name: ipsintr_blocking                                           */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -4014,7 +3932,7 @@
 			scb->cmd.logical_info.reserved2 = 0;
 			scb->cmd.logical_info.reserved3 = 0;
 			scb->data_len = sizeof (IPS_LD_INFO);
-            scb->data_busaddr = ha->logical_drive_info_dma_addr;
+			scb->data_busaddr = ha->logical_drive_info_dma_addr;
 			scb->flags = 0;
 			scb->cmd.logical_info.buffer_addr = scb->data_busaddr;
 			ret = IPS_SUCCESS;
@@ -4340,7 +4258,7 @@
 
 	if ((scb->basic_status & IPS_GSC_STATUS_MASK) > 1) {
 		memset(ha->logical_drive_info, 0, sizeof (IPS_LD_INFO));
-        return (0);
+		return (0);
 	}
 
 	if (ha->logical_drive_info->drive_info[scb->target_id].state !=
@@ -4579,7 +4497,7 @@
 		if (ha->logical_drive_info) {
 			pci_free_consistent(ha->pcidev,
 					    sizeof (IPS_LD_INFO),
-                        ha->logical_drive_info,
+					    ha->logical_drive_info,
 					    ha->logical_drive_info_dma_addr);
 			ha->logical_drive_info = NULL;
 		}
@@ -4773,7 +4691,7 @@
 	}
 
 	ha->scb_freelist = scb->q_next;
-    scb->flags = 0;
+	scb->flags = 0;
 	scb->q_next = NULL;
 
 	ips_init_scb(ha, scb);
@@ -6831,7 +6749,7 @@
 
 	METHOD_TRACE("ips_version_check", 1);
 
-    VersionInfo = ( IPS_VERSION_DATA * ) ha->ioctl_data;
+	VersionInfo = ( IPS_VERSION_DATA * ) ha->ioctl_data;
 
 	memset(FirmwareVersion, 0, IPS_COMPAT_ID_LENGTH + 1);
 	memset(BiosVersion, 0, IPS_COMPAT_ID_LENGTH + 1);
@@ -6843,7 +6761,7 @@
 	rc = IPS_FAILURE;
 	if (ha->subsys->param[4] & IPS_GET_VERSION_SUPPORT) {	/* If Versioning is Supported */
 		/* Get the Version Info with a Get Version Command */
-        memset( VersionInfo, 0, sizeof (IPS_VERSION_DATA));
+		memset( VersionInfo, 0, sizeof (IPS_VERSION_DATA));
 		rc = ips_get_version_info(ha, ha->ioctl_busaddr, intr);
 		if (rc == IPS_SUCCESS)
 			memcpy(FirmwareVersion, VersionInfo->compatibilityId,
@@ -6999,7 +6917,6 @@
 			for (j = position; j < ips_num_controllers; j++) {
 				switch (ips_ha[j]->ad_type) {
 				case IPS_ADTYPE_SERVERAID6M:
-				case IPS_ADTYPE_SERVERAID7k:
 				case IPS_ADTYPE_SERVERAID7M:
 					if (nvram->adapter_order[i] == 'M') {
 						ips_shift_controllers(position,
@@ -7020,6 +6937,7 @@
 				case IPS_ADTYPE_SERVERAID6I:
 				case IPS_ADTYPE_SERVERAID5I2:
 				case IPS_ADTYPE_SERVERAID5I1:
+				case IPS_ADTYPE_SERVERAID7k:
 					if (nvram->adapter_order[i] == 'S') {
 						ips_shift_controllers(position,
 								      j);
@@ -7547,6 +7465,15 @@
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,9)
 MODULE_LICENSE("GPL");
 #endif
+
+#ifdef MODULE_DESCRIPTION
+MODULE_DESCRIPTION("IBM ServeRAID Adapter Driver " IPS_VER_STRING);
+#endif
+
+#ifdef MODULE_VERSION
+MODULE_VERSION(IPS_VER_STRING);
+#endif
+
 
 /*
  * Overrides for Emacs so that we almost follow Linus's tabbing style.
diff -Nru a/drivers/scsi/ips.h b/drivers/scsi/ips.h
--- a/drivers/scsi/ips.h	2004-05-19 12:42:01 -07:00
+++ b/drivers/scsi/ips.h	2004-08-20 00:18:04 -07:00
@@ -95,14 +95,15 @@
       #define scsi_set_pci_device(sh,dev) (0)
    #endif
 
-   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,23)
-      typedef void irqreturn_t;
+   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
+   
+      #ifndef irqreturn_t
+         typedef void irqreturn_t;
+      #endif 
+      
       #define IRQ_NONE
       #define IRQ_HANDLED
       #define IRQ_RETVAL(x)
-   #endif
-
-   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
       #define IPS_REGISTER_HOSTS(SHT)      scsi_register_module(MODULE_SCSI_HA,SHT)
       #define IPS_UNREGISTER_HOSTS(SHT)    scsi_unregister_module(MODULE_SCSI_HA,SHT)
       #define IPS_ADD_HOST(shost,device)
@@ -1213,13 +1214,13 @@
 
 #define IPS_VER_MAJOR 7
 #define IPS_VER_MAJOR_STRING "7"
-#define IPS_VER_MINOR 00
-#define IPS_VER_MINOR_STRING "00"
-#define IPS_VER_BUILD 15
-#define IPS_VER_BUILD_STRING "15"
-#define IPS_VER_STRING "7.00.15"
+#define IPS_VER_MINOR 10
+#define IPS_VER_MINOR_STRING "10"
+#define IPS_VER_BUILD 18
+#define IPS_VER_BUILD_STRING "18"
+#define IPS_VER_STRING "7.10.18"
 #define IPS_RELEASE_ID 0x00020000
-#define IPS_BUILD_IDENT 625
+#define IPS_BUILD_IDENT 731
 #define IPS_LEGALCOPYRIGHT_STRING "(C) Copyright IBM Corp. 1994, 2002. All Rights Reserved."
 #define IPS_ADAPTECCOPYRIGHT_STRING "(c) Copyright Adaptec, Inc. 2002 to 2004. All Rights Reserved."
 #define IPS_DELLCOPYRIGHT_STRING "(c) Copyright Dell 2004. All Rights Reserved."
@@ -1230,32 +1231,35 @@
 #define IPS_VER_SERVERAID2 "2.88.13"
 #define IPS_VER_NAVAJO "2.88.13"
 #define IPS_VER_SERVERAID3 "6.10.24"
-#define IPS_VER_SERVERAID4H "7.00.15"
-#define IPS_VER_SERVERAID4MLx "7.00.15"
-#define IPS_VER_SARASOTA "7.00.15"
-#define IPS_VER_MARCO "7.00.15"
-#define IPS_VER_SEBRING "7.00.15"
+#define IPS_VER_SERVERAID4H "7.10.11"
+#define IPS_VER_SERVERAID4MLx "7.10.18"
+#define IPS_VER_SARASOTA "7.10.18"
+#define IPS_VER_MARCO "7.10.18"
+#define IPS_VER_SEBRING "7.10.18"
+#define IPS_VER_KEYWEST "7.10.18"
 
 /* Compatability IDs for various adapters */
 #define IPS_COMPAT_UNKNOWN ""
-#define IPS_COMPAT_CURRENT "SB610"
+#define IPS_COMPAT_CURRENT "KW710"
 #define IPS_COMPAT_SERVERAID1 "2.25.01"
 #define IPS_COMPAT_SERVERAID2 "2.88.13"
 #define IPS_COMPAT_NAVAJO  "2.88.13"
 #define IPS_COMPAT_KIOWA "2.88.13"
 #define IPS_COMPAT_SERVERAID3H  "SB610"
 #define IPS_COMPAT_SERVERAID3L  "SB610"
-#define IPS_COMPAT_SERVERAID4H  "SB610"
-#define IPS_COMPAT_SERVERAID4M  "SB610"
-#define IPS_COMPAT_SERVERAID4L  "SB610"
-#define IPS_COMPAT_SERVERAID4Mx "SB610"
-#define IPS_COMPAT_SERVERAID4Lx "SB610"
-#define IPS_COMPAT_SARASOTA     "SB610"
-#define IPS_COMPAT_MARCO        "SB610"
-#define IPS_COMPAT_SEBRING      "SB610"
-#define IPS_COMPAT_BIOS "SB610"
+#define IPS_COMPAT_SERVERAID4H  "KW710"
+#define IPS_COMPAT_SERVERAID4M  "KW710"
+#define IPS_COMPAT_SERVERAID4L  "KW710"
+#define IPS_COMPAT_SERVERAID4Mx "KW710"
+#define IPS_COMPAT_SERVERAID4Lx "KW710"
+#define IPS_COMPAT_SARASOTA     "KW710"
+#define IPS_COMPAT_MARCO        "KW710"
+#define IPS_COMPAT_SEBRING      "KW710"
+#define IPS_COMPAT_TAMPA        "KW710"
+#define IPS_COMPAT_KEYWEST      "KW710"
+#define IPS_COMPAT_BIOS "KW710"
 
-#define IPS_COMPAT_MAX_ADAPTER_TYPE 16
+#define IPS_COMPAT_MAX_ADAPTER_TYPE 18
 #define IPS_COMPAT_ID_LENGTH 8
 
 #define IPS_DEFINE_COMPAT_TABLE(tablename) \
@@ -1275,7 +1279,9 @@
       IPS_COMPAT_SARASOTA,         /* one-channel variety of SARASOTA */  \
       IPS_COMPAT_SARASOTA,         /* two-channel variety of SARASOTA */  \
       IPS_COMPAT_MARCO, \
-      IPS_COMPAT_SEBRING \
+      IPS_COMPAT_SEBRING, \
+	  IPS_COMPAT_TAMPA, \
+      IPS_COMPAT_KEYWEST \
    }
 
 
diff -Nru a/drivers/scsi/seagate.c b/drivers/scsi/seagate.c
--- a/drivers/scsi/seagate.c	2003-06-29 08:10:06 -07:00
+++ b/drivers/scsi/seagate.c	2004-08-26 16:50:38 -07:00
@@ -698,7 +698,7 @@
 	done_fn = done;
 	current_target = SCpnt->target;
 	current_lun = SCpnt->lun;
-	(const void *) current_cmnd = SCpnt->cmnd;
+	current_cmnd = SCpnt->cmnd;
 	current_data = (unsigned char *) SCpnt->request_buffer;
 	current_bufflen = SCpnt->request_bufflen;
 	SCint = SCpnt;
diff -Nru a/fs/affs/super.c b/fs/affs/super.c
--- a/fs/affs/super.c	2004-06-18 02:07:09 -07:00
+++ b/fs/affs/super.c	2004-08-26 16:50:38 -07:00
@@ -130,7 +130,7 @@
 					printk("AFFS: Argument for set[ug]id option missing\n");
 					return 0;
 				} else {
-					(f ? *uid : *gid) = simple_strtoul(value,&value,0);
+					*(f ? uid : gid) = simple_strtoul(value,&value,0);
 					if (*value) {
 						printk("AFFS: Bad set[ug]id argument\n");
 						return 0;
diff -Nru a/fs/hfs/file_hdr.c b/fs/hfs/file_hdr.c
--- a/fs/hfs/file_hdr.c	2004-08-08 01:59:53 -07:00
+++ b/fs/hfs/file_hdr.c	2004-08-26 06:28:10 -07:00
@@ -642,17 +642,18 @@
         int left, lcv, written = 0;
 	struct hdr_hdr meta;
 	int built_meta = 0;
-        off_t pos;
+        loff_t pos;
 
 	if (!S_ISREG(inode->i_mode)) {
 		hfs_warn("hfs_hdr_write: mode = %07o\n", inode->i_mode);
 		return -EINVAL;
 	}
+
+	pos = (filp->f_flags & O_APPEND) ? inode->i_size : *ppos;
+
 	if (count <= 0 || pos != (unsigned)pos) {
 		return 0;
 	}
-
-	pos = (filp->f_flags & O_APPEND) ? inode->i_size : *ppos;
 
 	if (!HFS_I(inode)->layout) {
 		HFS_I(inode)->layout = dup_layout(HFS_I(inode)->default_layout);
diff -Nru a/include/asm-i386/smpboot.h b/include/asm-i386/smpboot.h
--- a/include/asm-i386/smpboot.h	2004-04-20 16:00:02 -07:00
+++ b/include/asm-i386/smpboot.h	2004-08-26 10:48:47 -07:00
@@ -130,8 +130,8 @@
 			cpu = (cpu+1)%smp_num_cpus;
 			return cpu_to_physical_apicid(cpu);
 		default:
+			return cpu_online_map;
 	}
-	return cpu_online_map;
 }
 #else
 #define target_cpus() (cpu_online_map)
diff -Nru a/include/asm-sparc/sigcontext.h b/include/asm-sparc/sigcontext.h
--- a/include/asm-sparc/sigcontext.h	2002-02-05 09:39:47 -08:00
+++ b/include/asm-sparc/sigcontext.h	2004-08-24 17:28:32 -07:00
@@ -57,20 +57,6 @@
 	} si_fpqueue [16];
 } __siginfo_fpu_t;
 
-#ifdef __KERNEL__
-
-/* This magic should be in g_upper[0] for all upper parts
-   to be valid.
-   This is generated by sparc64 only, but for 32bit processes,
-   so we define it here as well.  */
-#define SIGINFO_EXTRA_V8PLUS_MAGIC      0x130e269
-typedef struct {
-	unsigned   int g_upper[8];
-	unsigned   int o_upper[8];
-} siginfo_extra_v8plus_t;
-
-#endif
-
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(__SPARC_SIGCONTEXT_H) */
diff -Nru a/include/asm-sparc64/pbm.h b/include/asm-sparc64/pbm.h
--- a/include/asm-sparc64/pbm.h	2003-08-02 17:45:01 -07:00
+++ b/include/asm-sparc64/pbm.h	2004-08-18 13:58:50 -07:00
@@ -70,6 +70,13 @@
 	 */
 	u32		lowest_consistent_map;
 
+	/* In order to deal with some buggy third-party PCI bridges that
+	 * do wrong prefetching, we never mark valid mappings as invalid.
+	 * Instead we point them at this dummy page.
+	 */
+	unsigned long	dummy_page;
+	unsigned long	dummy_page_pa;
+
 	/* If PBM_NCLUSTERS is ever decreased to 4 or lower,
 	 * or if largest supported page_table_sz * 8K goes above
 	 * 2GB, you must increase the size of the type of
@@ -92,6 +99,8 @@
 	 */
 	u32 dma_addr_mask;
 };
+
+extern void pci_iommu_table_init(struct pci_iommu *, int);
 
 /* This describes a PCI bus module's streaming buffer. */
 struct pci_strbuf {
diff -Nru a/include/asm-sparc64/sigcontext.h b/include/asm-sparc64/sigcontext.h
--- a/include/asm-sparc64/sigcontext.h	2002-02-05 09:39:50 -08:00
+++ b/include/asm-sparc64/sigcontext.h	2004-08-24 17:28:32 -07:00
@@ -83,18 +83,6 @@
 	unsigned long		sigc_mask;
 };
 
-#ifdef __KERNEL__
-
-/* This magic should be in g_upper[0] for all upper parts
-   to be valid.  */
-#define SIGINFO_EXTRA_V8PLUS_MAGIC	0x130e269
-typedef struct {
-	unsigned   int g_upper[8];
-	unsigned   int o_upper[8];
-} siginfo_extra_v8plus_t;
-
-#endif
-
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(__SPARC64_SIGCONTEXT_H) */
diff -Nru a/include/asm-sparc64/string.h b/include/asm-sparc64/string.h
--- a/include/asm-sparc64/string.h	2004-07-27 13:08:37 -07:00
+++ b/include/asm-sparc64/string.h	2004-08-26 14:25:45 -07:00
@@ -15,35 +15,25 @@
 
 #include <asm/asi.h>
 
-extern void __memmove(void *,const void *,__kernel_size_t);
 extern void *__memset(void *,int,__kernel_size_t);
-extern void *__builtin_memset(void *,int,__kernel_size_t);
 
 #ifndef EXPORT_SYMTAB_STROPS
 
 /* First the mem*() things. */
-#define __HAVE_ARCH_BCOPY
 #define __HAVE_ARCH_MEMMOVE
-
-#undef memmove
-#define memmove(_to, _from, _n) \
-({ \
-	void *_t = (_to); \
-	__memmove(_t, (_from), (_n)); \
-	_t; \
-})
+extern void *memmove(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMCPY
-
-extern void * memcpy(void *,const void *,__kernel_size_t);
+extern void *memcpy(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMSET
+extern void *__builtin_memset(void *,int,__kernel_size_t);
 
 static inline void *__constant_memset(void *s, int c, __kernel_size_t count)
 {
 	extern __kernel_size_t __bzero(void *, __kernel_size_t);
 
-	if(!c) {
+	if (!c) {
 		__bzero(s, count);
 		return s;
 	} else
@@ -61,19 +51,19 @@
 #define __HAVE_ARCH_MEMSCAN
 
 #undef memscan
-#define memscan(__arg0, __char, __arg2)						\
-({										\
-	extern void *__memscan_zero(void *, size_t);				\
-	extern void *__memscan_generic(void *, int, size_t);			\
-	void *__retval, *__addr = (__arg0);					\
-	size_t __size = (__arg2);						\
-										\
-	if(__builtin_constant_p(__char) && !(__char))				\
-		__retval = __memscan_zero(__addr, __size);			\
-	else									\
-		__retval = __memscan_generic(__addr, (__char), __size);		\
-										\
-	__retval;								\
+#define memscan(__arg0, __char, __arg2)					\
+({									\
+	extern void *__memscan_zero(void *, size_t);			\
+	extern void *__memscan_generic(void *, int, size_t);		\
+	void *__retval, *__addr = (__arg0);				\
+	size_t __size = (__arg2);					\
+									\
+	if(__builtin_constant_p(__char) && !(__char))			\
+		__retval = __memscan_zero(__addr, __size);		\
+	else								\
+		__retval = __memscan_generic(__addr, (__char), __size);	\
+									\
+	__retval;							\
 })
 
 #define __HAVE_ARCH_MEMCMP
diff -Nru a/include/asm-sparc64/uaccess.h b/include/asm-sparc64/uaccess.h
--- a/include/asm-sparc64/uaccess.h	2002-02-04 23:49:27 -08:00
+++ b/include/asm-sparc64/uaccess.h	2004-08-26 14:25:45 -07:00
@@ -249,26 +249,50 @@
 
 extern int __get_user_bad(void);
 
-extern __kernel_size_t __copy_from_user(void *to, const void *from,
-					__kernel_size_t size);
+extern unsigned long ___copy_from_user(void *to, const void *from,
+				       unsigned long size);
+extern unsigned long copy_from_user_fixup(void *to, const void *from,
+					  unsigned long size);
+static inline unsigned long copy_from_user(void *to, const void *from,
+					   unsigned long size)
+{
+	unsigned long ret = ___copy_from_user(to, from, size);
 
-extern __kernel_size_t __copy_to_user(void *to, const void *from,
-				      __kernel_size_t size);
+	if (ret)
+		ret = copy_from_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_from_user copy_from_user
 
-extern __kernel_size_t __copy_in_user(void *to, const void *from,
-				      __kernel_size_t size);
+extern unsigned long ___copy_to_user(void *to, const void *from,
+				     unsigned long size);
+extern unsigned long copy_to_user_fixup(void *to, const void *from,
+					unsigned long size);
+static inline unsigned long copy_to_user(void *to, const void *from,
+					 unsigned long size)
+{
+	unsigned long ret = ___copy_to_user(to, from, size);
 
-#define copy_from_user(to,from,n)		\
-	__copy_from_user((void *)(to),	\
-		    (void *)(from), (__kernel_size_t)(n))
+	if (ret)
+		ret = copy_to_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_to_user copy_to_user
 
-#define copy_to_user(to,from,n) \
-	__copy_to_user((void *)(to), \
-	(void *) (from), (__kernel_size_t)(n))
+extern unsigned long ___copy_in_user(void *to, const void *from,
+				     unsigned long size);
+extern unsigned long copy_in_user_fixup(void *to, void *from,
+					unsigned long size);
+static inline unsigned long copy_in_user(void *to, void *from,
+					 unsigned long size)
+{
+	unsigned long ret = ___copy_in_user(to, from, size);
 
-#define copy_in_user(to,from,n) \
-	__copy_in_user((void *)(to), \
-	(void *) (from), (__kernel_size_t)(n))
+	if (ret)
+		ret = copy_in_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_in_user copy_in_user
 
 extern __inline__ __kernel_size_t __clear_user(void *addr, __kernel_size_t size)
 {
diff -Nru a/include/linux/mm.h b/include/linux/mm.h
--- a/include/linux/mm.h	2004-08-18 09:08:54 -07:00
+++ b/include/linux/mm.h	2004-08-27 05:11:55 -07:00
@@ -309,7 +309,7 @@
 #define UnlockPage(page)	unlock_page(page)
 #define Page_Uptodate(page)	test_bit(PG_uptodate, &(page)->flags)
 #ifndef SetPageUptodate
-#define SetPageUptodate(page)	set_bit(PG_uptodate, &(page)->flags);
+#define SetPageUptodate(page)	set_bit(PG_uptodate, &(page)->flags)
 #endif
 #define ClearPageUptodate(page)	clear_bit(PG_uptodate, &(page)->flags)
 #define PageDirty(page)		test_bit(PG_dirty, &(page)->flags)
diff -Nru a/net/irda/ircomm/ircomm_param.c b/net/irda/ircomm/ircomm_param.c
--- a/net/irda/ircomm/ircomm_param.c	2003-10-11 12:19:28 -07:00
+++ b/net/irda/ircomm/ircomm_param.c	2004-08-26 10:29:46 -07:00
@@ -198,7 +198,7 @@
 		IRDA_DEBUG(2, "%s(), No common service type to use!\n", __FUNCTION__);
 		return -1;
 	}
-	IRDA_DEBUG(0, __FUNCTION__ "%s(), services in common=%02x\n", __FUNCTION__,
+	IRDA_DEBUG(0, "%s(), services in common=%02x\n", __FUNCTION__ ,
 		   service_type);
 
 	/*
diff -Nru a/net/irda/irlmp.c b/net/irda/irlmp.c
--- a/net/irda/irlmp.c	2002-12-18 09:56:13 -08:00
+++ b/net/irda/irlmp.c	2004-08-26 10:38:48 -07:00
@@ -1241,7 +1241,7 @@
 	/* Get the number of lsap. That's the only safe way to know
 	 * that we have looped around... - Jean II */
 	lsap_todo = HASHBIN_GET_SIZE(self->lsaps);
-	IRDA_DEBUG(4, __FUNCTION__ "() : %d lsaps to scan\n", lsap_todo);
+	IRDA_DEBUG(4, "%s() : %d lsaps to scan\n", __FUNCTION__ , lsap_todo);
 
 	/* Poll lsap in order until the queue is full or until we
 	 * tried them all.
@@ -1255,7 +1255,7 @@
 			/* Note that if there is only one LSAP on the LAP
 			 * (most common case), self->flow_next is always NULL,
 			 * so we always avoid this loop. - Jean II */
-			IRDA_DEBUG(4, __FUNCTION__ "() : searching my LSAP\n");
+			IRDA_DEBUG(4, "%s() : searching my LSAP\n", __FUNCTION__ );
 
 			/* We look again in hashbins, because the lsap
 			 * might have gone away... - Jean II */
@@ -1274,14 +1274,14 @@
 
 		/* Next time, we will get the next one (or the first one) */
 		self->flow_next = (struct lsap_cb *) hashbin_get_next(self->lsaps);
-		IRDA_DEBUG(4, __FUNCTION__ "() : curr is %p, next was %p and is now %p, still %d to go - queue len = %d\n", curr, next, self->flow_next, lsap_todo, IRLAP_GET_TX_QUEUE_LEN(self->irlap));
+		IRDA_DEBUG(4, "%s() : curr is %p, next was %p and is now %p, still %d to go - queue len = %d\n", __FUNCTION__ , curr, next, self->flow_next, lsap_todo, IRLAP_GET_TX_QUEUE_LEN(self->irlap));
 
 		/* Inform lsap user that it can send one more packet. */
 		if (curr->notify.flow_indication != NULL)
 			curr->notify.flow_indication(curr->notify.instance, 
 						     curr, flow);
 		else
-			IRDA_DEBUG(1, __FUNCTION__ "(), no handler\n");
+			IRDA_DEBUG(1, "%s(), no handler\n", __FUNCTION__ );
 	}
 }
 
