diff -Nru a/CREDITS b/CREDITS
--- a/CREDITS	2004-07-27 01:23:09 -07:00
+++ b/CREDITS	2004-08-28 02:19:18 -07:00
@@ -2130,14 +2130,13 @@
 S: Netherlands
 
 N: David S. Miller
-E: davem@redhat.com
+E: davem@davemloft.net
 D: Sparc and blue box hacker
 D: Vger Linux mailing list co-maintainer
 D: Linux Emacs elf/qmagic support + other libc/gcc things
 D: Yee bore de yee bore! ;-)
-S: 750 N. Shoreline Blvd.
-S: Apt. #111
-S: Mountain View, California 94043
+S: 575 Harrison St. #103
+S: San Francisco, California 94105
 S: USA
 
 N: Rick Miller
diff -Nru a/Documentation/Configure.help b/Documentation/Configure.help
--- a/Documentation/Configure.help	2004-08-23 20:35:40 -07:00
+++ b/Documentation/Configure.help	2004-08-27 19:14:02 -07:00
@@ -28961,6 +28961,18 @@
   This code also includes SHA-384, a 384 bit hash with 192 bits
   of security against collision attacks.
 
+CONFIG_CRYPTO_WHIRLPOOL
+  Whirlpool hash algorithm.
+
+  Whirlpool is part of the NESSIE cryptographic primtives.
+  Whirlpool works on messages shorter than 2^256 bits and
+  produces a 512 bit hash.
+
+  Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard
+
+  See also:
+  http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html
+
 CONFIG_CRYPTO_DES
   DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
 
diff -Nru a/Documentation/crypto/api-intro.txt b/Documentation/crypto/api-intro.txt
--- a/Documentation/crypto/api-intro.txt	2004-08-10 18:27:14 -07:00
+++ b/Documentation/crypto/api-intro.txt	2004-08-24 11:28:59 -07:00
@@ -225,6 +225,9 @@
 Khazad algorithm contributors:
   Aaron Grothe
 
+Whirlpool algorithm contributors:
+  Aaron Grothe
+
 Generic scatterwalk code by Adam J. Richter <adam@yggdrasil.com>
 
 Please send any credits updates or corrections to:
diff -Nru a/Documentation/filesystems/xfs.txt b/Documentation/filesystems/xfs.txt
--- a/Documentation/filesystems/xfs.txt	2004-04-28 23:12:12 -07:00
+++ b/Documentation/filesystems/xfs.txt	2004-08-18 15:35:53 -07:00
@@ -120,18 +120,18 @@
 
   fs.xfs.stats_clear		(Min: 0  Default: 0  Max: 1)
 	Setting this to "1" clears accumulated XFS statistics 
-	in /proc/fs/xfs/stat.  It then immediately reset to "0".
+	in /proc/fs/xfs/stat.  It then immediately resets to "0".
   	
-  fs.xfs.sync_interval		(Min: HZ  Default: 30*HZ  Max: 60*HZ)
+  fs.xfs.xfssyncd_centisecs	(Min: 100  Default: 3000  Max: 6000)
 	The interval at which the xfssyncd thread flushes metadata
 	out to disk.  This thread will flush log activity out, and
 	do some processing on unlinked inodes.
 
-  fs.xfs.age_buffer             (Min: 1*HZ  Default: 15*HZ  Max: 300*HZ)
-	The age at which xfsbufd flushes dirty metadata buffers to disk.
-
-  fs.xfs.flush_interval         (Min: HZ/2  Default: HZ  Max: 30*HZ)
+  fs.xfs.xfsbufd_centisecs	(Min: 50  Default: 100  Max: 3000)
 	The interval at which xfsbufd scans the dirty metadata buffers list.
+
+  fs.xfs.age_buffer_centisecs	(Min: 100  Default: 1500  Max: 30000)
+	The age at which xfsbufd flushes dirty metadata buffers to disk.
 
   fs.xfs.error_level		(Min: 0  Default: 3  Max: 11)
 	A volume knob for error reporting when internal errors occur.
diff -Nru a/MAINTAINERS b/MAINTAINERS
--- a/MAINTAINERS	2004-08-23 15:01:27 -07:00
+++ b/MAINTAINERS	2004-08-28 02:18:52 -07:00
@@ -534,15 +534,7 @@
 P:	James Morris
 M:	jmorris@intercode.com.au
 P:	David S. Miller
-M:	davem@redhat.com
-MARVELL YUKON / SYSKONNECT DRIVER
-P:	Mirko Lindner
-M: 	mlindner@syskonnect.de
-P:	Ralph Roesler
-M: 	rroesler@syskonnect.de
-W: 	http://www.syskonnect.com
-S: 	Supported
-M:	davem@redhat.com
+M:	davem@davemloft.net
 W	http://samba.org/~jamesm/crypto/
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
@@ -1192,6 +1184,14 @@
 L:	netdev@oss.sgi.com
 S:	Supported
 
+MARVELL YUKON / SYSKONNECT DRIVER
+P:	Mirko Lindner
+M: 	mlindner@syskonnect.de
+P:	Ralph Roesler
+M: 	rroesler@syskonnect.de
+W: 	http://www.syskonnect.com
+S: 	Supported
+
 MATROX FRAMEBUFFER DRIVER
 P:	Petr Vandrovec
 M:	vandrove@vc.cvut.cz
@@ -1316,7 +1316,7 @@
 
 NETWORKING [IPv4/IPv6]
 P:	David S. Miller
-M:	davem@redhat.com
+M:	davem@davemloft.net
 P:	Alexey Kuznetsov
 M:	kuznet@ms2.inr.ac.ru
 P:	Pekka Savola (ipv6)
@@ -1726,7 +1726,7 @@
 
 UltraSPARC (sparc64):
 P:	David S. Miller
-M:	davem@redhat.com
+M:	davem@davemloft.net
 P:	Eddie C. Dost
 M:	ecd@skynet.be
 P:	Jakub Jelinek
diff -Nru a/Makefile b/Makefile
--- a/Makefile	2004-08-25 04:10:06 -07:00
+++ b/Makefile	2004-09-04 02:48:43 -07:00
@@ -1,7 +1,7 @@
 VERSION = 2
 PATCHLEVEL = 4
 SUBLEVEL = 28
-EXTRAVERSION = -pre2
+EXTRAVERSION = -pre2-bk3
 
 KERNELRELEASE=$(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
 
diff -Nru a/arch/sparc64/defconfig b/arch/sparc64/defconfig
--- a/arch/sparc64/defconfig	2004-05-19 16:32:32 -07:00
+++ b/arch/sparc64/defconfig	2004-08-26 14:42:36 -07:00
@@ -375,7 +375,7 @@
 CONFIG_NET_SCH_TEQL=m
 CONFIG_NET_SCH_TBF=m
 CONFIG_NET_SCH_GRED=m
-CONFIG_NET_SCH_DELAY=m
+CONFIG_NET_SCH_NETEM=m
 CONFIG_NET_SCH_DSMARK=m
 CONFIG_NET_SCH_INGRESS=m
 CONFIG_NET_QOS=y
@@ -410,6 +410,7 @@
 #
 # CONFIG_BLK_DEV_HD_IDE is not set
 # CONFIG_BLK_DEV_HD is not set
+# CONFIG_BLK_DEV_IDE_SATA is not set
 CONFIG_BLK_DEV_IDEDISK=y
 # CONFIG_IDEDISK_MULTI_MODE is not set
 # CONFIG_IDEDISK_STROKE is not set
@@ -1134,6 +1135,8 @@
 CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_TEA=m
+CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_ARC4=m
 CONFIG_CRYPTO_DEFLATE=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
diff -Nru a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
--- a/arch/sparc64/kernel/head.S	2004-02-04 11:44:03 -08:00
+++ b/arch/sparc64/kernel/head.S	2004-09-02 14:47:07 -07:00
@@ -50,7 +50,7 @@
  */
         .global root_flags, ram_flags, root_dev
         .global sparc_ramdisk_image, sparc_ramdisk_size
-	.globl	silo_args
+	.global sparc_ramdisk_image64
 
         .ascii  "HdrS"
         .word   LINUX_VERSION_CODE
@@ -61,7 +61,7 @@
 	 * 0x0202 : Supports kernel params string
 	 * 0x0201 : Supports reboot_command
 	 */
-	.half   0x0300          /* HdrS version */
+	.half   0x0301          /* HdrS version */
 
 root_flags:
         .half   1
@@ -75,6 +75,8 @@
         .word   0
         .xword  reboot_command
 	.xword	bootstr_info
+sparc_ramdisk_image64:
+	.xword	0
 	.word	_end
 
 	/* We must be careful, 32-bit OpenBOOT will get confused if it
diff -Nru a/arch/sparc64/kernel/ioctl32.c b/arch/sparc64/kernel/ioctl32.c
--- a/arch/sparc64/kernel/ioctl32.c	2004-04-27 11:45:56 -07:00
+++ b/arch/sparc64/kernel/ioctl32.c	2004-09-02 13:11:19 -07:00
@@ -2948,7 +2948,7 @@
 	case LV_REMOVE:
 	case LV_RENAME:
 	case LV_STATUS_BYNAME:
-	        err = copy_from_user(&u.pv_status, arg, sizeof(u.pv_status.pv_name));
+	        err = copy_from_user(&u.lv_req, arg, sizeof(u.lv_req.lv_name));
 		if (err)
 			return -EFAULT;
 		if (cmd != LV_REMOVE) {
@@ -2991,7 +2991,7 @@
 
 	case PV_CHANGE:
 	case PV_STATUS:
-		err = copy_from_user(&u.pv_status, arg, sizeof(u.lv_req.lv_name));
+		err = copy_from_user(&u.pv_status, arg, sizeof(u.pv_status.pv_name));
 		if (err)
 			return -EFAULT;
 		err = __get_user(ptr, &((pv_status_req32_t *)arg)->pv);
@@ -3063,7 +3063,7 @@
 	        if (u.lv_bydev.lv) {
 			if (!err)
 				err = copy_lv_t(ptr, u.lv_bydev.lv);
-			put_lv_t(u.lv_byindex.lv);
+			put_lv_t(u.lv_bydev.lv);
 	        }
 	        break;
 
diff -Nru a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
--- a/arch/sparc64/kernel/pci_iommu.c	2003-07-27 21:19:59 -07:00
+++ b/arch/sparc64/kernel/pci_iommu.c	2004-08-18 13:58:50 -07:00
@@ -56,6 +56,39 @@
 	}
 }
 
+#define IOPTE_CONSISTENT(CTX) \
+	(IOPTE_VALID | IOPTE_CACHE | \
+	 (((CTX) << 47) & IOPTE_CONTEXT))
+
+#define IOPTE_STREAMING(CTX) \
+	(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
+
+/* Existing mappings are never marked invalid, instead they
+ * are pointed to a dummy page.
+ */
+#define IOPTE_IS_DUMMY(iommu, iopte)	\
+	((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
+
+static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
+{
+	unsigned long val = iopte_val(*iopte);
+
+	val &= ~IOPTE_PAGE;
+	val |= iommu->dummy_page_pa;
+
+	iopte_val(*iopte) = val;
+}
+
+void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize)
+{
+	int i;
+
+	tsbsize /= sizeof(iopte_t);
+
+	for (i = 0; i < tsbsize; i++)
+		iopte_make_dummy(iommu, &iommu->page_table[i]);
+}
+
 static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
 {
 	iopte_t *iopte, *limit, *first;
@@ -79,7 +112,7 @@
 	
 	first = iopte;
 	for (;;) {
-		if (iopte_val(*iopte) == 0UL) {
+		if (IOPTE_IS_DUMMY(iommu, iopte)) {
 			if ((iopte + (1 << cnum)) >= limit)
 				ent = 0;
 			else
@@ -142,12 +175,12 @@
 	iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
 	while (iopte > iommu->page_table) {
 		iopte--;
-		if (!(iopte_val(*iopte) & IOPTE_VALID)) {
+		if (IOPTE_IS_DUMMY(iommu, iopte)) {
 			unsigned long tmp = npages;
 
 			while (--tmp) {
 				iopte--;
-				if (iopte_val(*iopte) & IOPTE_VALID)
+				if (!IOPTE_IS_DUMMY(iommu, iopte))
 					break;
 			}
 			if (tmp == 0) {
@@ -162,15 +195,6 @@
 	return NULL;
 }
 
-#define IOPTE_CONSISTENT(CTX) \
-	(IOPTE_VALID | IOPTE_CACHE | \
-	 (((CTX) << 47) & IOPTE_CONTEXT))
-
-#define IOPTE_STREAMING(CTX) \
-	(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
-
-#define IOPTE_INVALID	0UL
-
 /* Allocate and map kernel buffer of size SIZE using consistent mode
  * DMA for PCI device PDEV.  Return non-NULL cpu-side address if
  * successful and set *DMA_ADDRP to the PCI side dma address.
@@ -261,7 +285,7 @@
 		limit = (iommu->page_table +
 			 (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
 		while (walk < limit) {
-			if (iopte_val(*walk) != IOPTE_INVALID)
+			if (!IOPTE_IS_DUMMY(iommu, walk))
 				break;
 			walk++;
 		}
@@ -280,7 +304,7 @@
 		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
 
 	for (i = 0; i < npages; i++, iopte++)
-		iopte_val(*iopte) = IOPTE_INVALID;
+		iopte_make_dummy(iommu, iopte);
 
 	if (iommu->iommu_ctxflush) {
 		pci_iommu_write(iommu->iommu_ctxflush, ctx);
@@ -377,7 +401,7 @@
 	base = iommu->page_table +
 		((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 #ifdef DEBUG_PCI_IOMMU
-	if (iopte_val(*base) == IOPTE_INVALID)
+	if (IOPTE_IS_DUMMY(iommu, base))
 		printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
 		       bus_addr, sz, __builtin_return_address(0));
 #endif
@@ -416,7 +440,7 @@
 	}
 
 	/* Step 2: Clear out first TSB entry. */
-	iopte_val(*base) = IOPTE_INVALID;
+	iopte_make_dummy(iommu, base);
 
 	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
 			       npages, ctx);
@@ -617,7 +641,7 @@
 		((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 
 #ifdef DEBUG_PCI_IOMMU
-	if (iopte_val(*base) == IOPTE_INVALID)
+	if (IOPTE_IS_DUMMY(iommu, base))
 		printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
 #endif
 
@@ -654,7 +678,7 @@
 	}
 
 	/* Step 2: Clear out first TSB entry. */
-	iopte_val(*base) = IOPTE_INVALID;
+	iopte_make_dummy(iommu, base);
 
 	free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
 			       npages, ctx);
diff -Nru a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
--- a/arch/sparc64/kernel/pci_psycho.c	2003-08-24 02:38:14 -07:00
+++ b/arch/sparc64/kernel/pci_psycho.c	2004-08-18 13:58:50 -07:00
@@ -1373,6 +1373,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	/* Using assumed page size 8K with 128K entries we need 1MB iommu page
 	 * table (128K ioptes * 8 bytes per iopte).  This is
 	 * page order 7 on UltraSparc.
@@ -1386,7 +1394,7 @@
 	iommu->page_table_sz_bits = 17;
 	iommu->page_table_map_base = 0xc0000000;
 	iommu->dma_addr_mask = 0xffffffff;
-	memset((char *)tsbbase, 0, IO_TSB_SIZE);
+	pci_iommu_table_init(iommu, IO_TSB_SIZE);
 
 	/* We start with no consistent mappings. */
 	iommu->lowest_consistent_map =
diff -Nru a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
--- a/arch/sparc64/kernel/pci_sabre.c	2003-12-18 12:24:22 -08:00
+++ b/arch/sparc64/kernel/pci_sabre.c	2004-08-18 13:58:50 -07:00
@@ -1315,6 +1315,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
 	if (!tsbbase) {
 		prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n");
@@ -1323,7 +1331,7 @@
 	iommu->page_table = (iopte_t *)tsbbase;
 	iommu->page_table_map_base = dvma_offset;
 	iommu->dma_addr_mask = dma_mask;
-	memset((char *)tsbbase, 0, PAGE_SIZE << order);
+	pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
 	sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
 
diff -Nru a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c
--- a/arch/sparc64/kernel/pci_schizo.c	2003-08-24 02:38:14 -07:00
+++ b/arch/sparc64/kernel/pci_schizo.c	2004-08-18 13:58:50 -07:00
@@ -1843,6 +1843,14 @@
 	 * in pci_iommu.c
 	 */
 
+	iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+	if (!iommu->dummy_page) {
+		prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+		prom_halt();
+	}
+	memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+	iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
 	/* Using assumed page size 8K with 128K entries we need 1MB iommu page
 	 * table (128K ioptes * 8 bytes per iopte).  This is
 	 * page order 7 on UltraSparc.
@@ -1857,7 +1865,7 @@
 	iommu->page_table = (iopte_t *)tsbbase;
 	iommu->page_table_map_base = vdma[0];
 	iommu->dma_addr_mask = dma_mask;
-	memset((char *)tsbbase, 0, PAGE_SIZE << order);
+	pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
 	switch (tsbsize) {
 	case 64:
diff -Nru a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
--- a/arch/sparc64/kernel/process.c	2004-04-28 13:35:32 -07:00
+++ b/arch/sparc64/kernel/process.c	2004-08-26 14:58:34 -07:00
@@ -503,7 +503,7 @@
 
 	distance = fp - psp;
 	rval = (csp - distance);
-	if (copy_in_user(rval, psp, distance))
+	if (copy_in_user((void *)rval, (void *)psp, distance))
 		rval = 0;
 	else if (current->thread.flags & SPARC_FLAG_32BIT) {
 		if (put_user(((u32)csp), &(((struct reg_window32 *)rval)->ins[6])))
diff -Nru a/arch/sparc64/kernel/signal.c b/arch/sparc64/kernel/signal.c
--- a/arch/sparc64/kernel/signal.c	2003-06-29 15:00:05 -07:00
+++ b/arch/sparc64/kernel/signal.c	2004-08-24 17:28:32 -07:00
@@ -116,8 +116,8 @@
 	regs->tnpc = npc;
 	err |= __get_user(regs->y, &((*grp)[MC_Y]));
 	err |= __get_user(tstate, &((*grp)[MC_TSTATE]));
-	regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC);
-	regs->tstate |= (tstate & (TSTATE_ICC | TSTATE_XCC));
+	regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC);
+	regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC));
 	err |= __get_user(regs->u_regs[UREG_G1], (&(*grp)[MC_G1]));
 	err |= __get_user(regs->u_regs[UREG_G2], (&(*grp)[MC_G2]));
 	err |= __get_user(regs->u_regs[UREG_G3], (&(*grp)[MC_G3]));
@@ -437,9 +437,9 @@
 	err |= __get_user(tstate, &sf->regs.tstate);
 	err |= copy_from_user(regs->u_regs, sf->regs.u_regs, sizeof(regs->u_regs));
 
-	/* User can only change condition codes in %tstate. */
-	regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC);
-	regs->tstate |= (tstate & (TSTATE_ICC | TSTATE_XCC));
+	/* User can only change condition codes and %asi in %tstate. */
+	regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC);
+	regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC));
 
 	err |= __get_user(fpu_save, &sf->fpu_save);
 	if (fpu_save)
diff -Nru a/arch/sparc64/kernel/signal32.c b/arch/sparc64/kernel/signal32.c
--- a/arch/sparc64/kernel/signal32.c	2004-03-26 21:08:52 -08:00
+++ b/arch/sparc64/kernel/signal32.c	2004-08-24 17:28:32 -07:00
@@ -62,6 +62,16 @@
 	unsigned extramask[_NSIG_WORDS32 - 1];
 };
 
+/* This magic should be in g_upper[0] for all upper parts
+ * to be valid.
+ */
+#define SIGINFO_EXTRA_V8PLUS_MAGIC	0x130e269
+typedef struct {
+	unsigned int g_upper[8];
+	unsigned int o_upper[8];
+	unsigned int asi;
+} siginfo_extra_v8plus_t;
+
 /* 
  * And the new one, intended to be used for Linux applications only
  * (we have enough in there to work with clone).
@@ -292,8 +302,13 @@
 	if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) {
 		err |= __get_user(i, &sf->v8plus.g_upper[0]);
 		if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) {
+			unsigned long asi;
+
 			for (i = UREG_G1; i <= UREG_I7; i++)
 				err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+			err |= __get_user(asi, &sf->v8plus.asi);
+			regs->tstate &= ~TSTATE_ASI;
+			regs->tstate |= ((asi & 0xffUL) << 24UL);
 		}
 	}
 
@@ -431,8 +446,13 @@
 	if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) {
 		err |= __get_user(i, &sf->v8plus.g_upper[0]);
 		if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) {
+			unsigned long asi;
+
 			for (i = UREG_G1; i <= UREG_I7; i++)
 				err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+			err |= __get_user(asi, &sf->v8plus.asi);
+			regs->tstate &= ~TSTATE_ASI;
+			regs->tstate |= ((asi & 0xffUL) << 24UL);
 		}
 	}
 
@@ -728,7 +748,10 @@
 	err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size);
 	err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]);
 	for (i = 1; i < 16; i++)
-		err |= __put_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+		err |= __put_user(((u32 *)regs->u_regs)[2*i],
+				  &sf->v8plus.g_upper[i]);
+	err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL,
+			  &sf->v8plus.asi);
 
 	if (psr & PSR_EF) {
 		err |= save_fpu_state32(regs, &sf->fpu_state);
@@ -1175,7 +1198,10 @@
 	err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size);
 	err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]);
 	for (i = 1; i < 16; i++)
-		err |= __put_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]);
+		err |= __put_user(((u32 *)regs->u_regs)[2*i],
+				  &sf->v8plus.g_upper[i]);
+	err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL,
+			  &sf->v8plus.asi);
 
 	if (psr & PSR_EF) {
 		err |= save_fpu_state32(regs, &sf->fpu_state);
diff -Nru a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
--- a/arch/sparc64/kernel/sparc64_ksyms.c	2004-07-27 13:08:37 -07:00
+++ b/arch/sparc64/kernel/sparc64_ksyms.c	2004-08-26 14:25:45 -07:00
@@ -324,7 +324,6 @@
 #endif
 
 /* Special internal versions of library functions. */
-EXPORT_SYMBOL(__memset);
 EXPORT_SYMBOL(_clear_page);
 EXPORT_SYMBOL(clear_user_page);
 EXPORT_SYMBOL(copy_user_page);
@@ -333,14 +332,17 @@
 EXPORT_SYMBOL(__memscan_generic);
 EXPORT_SYMBOL(__memcmp);
 EXPORT_SYMBOL(__strncmp);
-EXPORT_SYMBOL(__memmove);
+EXPORT_SYMBOL(__memset);
 
 EXPORT_SYMBOL(csum_partial_copy_sparc64);
 
 /* Moving data to/from userspace. */
-EXPORT_SYMBOL(__copy_to_user);
-EXPORT_SYMBOL(__copy_from_user);
-EXPORT_SYMBOL(__strncpy_from_user);
+EXPORT_SYMBOL(___copy_to_user);
+EXPORT_SYMBOL(___copy_from_user);
+EXPORT_SYMBOL(___copy_in_user);
+EXPORT_SYMBOL(copy_to_user_fixup);
+EXPORT_SYMBOL(copy_from_user_fixup);
+EXPORT_SYMBOL(copy_in_user_fixup);
 EXPORT_SYMBOL(__bzero_noasi);
 
 /* Various address conversion macros use this. */
diff -Nru a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c
--- a/arch/sparc64/kernel/time.c	2003-07-23 08:51:40 -07:00
+++ b/arch/sparc64/kernel/time.c	2004-08-26 14:57:34 -07:00
@@ -432,18 +432,12 @@
 		extern int _stext;
 		extern int rwlock_impl_begin, rwlock_impl_end;
 		extern int atomic_impl_begin, atomic_impl_end;
-		extern int __memcpy_begin, __memcpy_end;
-		extern int __bzero_begin, __bzero_end;
 		extern int __bitops_begin, __bitops_end;
 
 		if ((pc >= (unsigned long) &atomic_impl_begin &&
 		     pc < (unsigned long) &atomic_impl_end) ||
 		    (pc >= (unsigned long) &rwlock_impl_begin &&
 		     pc < (unsigned long) &rwlock_impl_end) ||
-		    (pc >= (unsigned long) &__memcpy_begin &&
-		     pc < (unsigned long) &__memcpy_end) ||
-		    (pc >= (unsigned long) &__bzero_begin &&
-		     pc < (unsigned long) &__bzero_end) ||
 		    (pc >= (unsigned long) &__bitops_begin &&
 		     pc < (unsigned long) &__bitops_end))
 			pc = o7;
diff -Nru a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
--- a/arch/sparc64/lib/Makefile	2002-05-08 23:18:22 -07:00
+++ b/arch/sparc64/lib/Makefile	2004-08-26 14:25:45 -07:00
@@ -8,9 +8,11 @@
 L_TARGET = lib.a
 obj-y := PeeCeeI.o blockops.o debuglocks.o strlen.o strncmp.o \
 	 memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \
-	 VIScopy.o VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
+	 VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
 	 VIScsumcopyusr.o VISsave.o atomic.o rwlock.o bitops.o \
-	 dec_and_lock.o U3memcpy.o U3copy_from_user.o U3copy_to_user.o \
-	 U3copy_in_user.o mcount.o
+	 U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
+	 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
+	 copy_in_user.o user_fixup.o memmove.o \
+	 dec_and_lock.o mcount.o
 
 include $(TOPDIR)/Rules.make
diff -Nru a/arch/sparc64/lib/U1copy_from_user.S b/arch/sparc64/lib/U1copy_from_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1copy_from_user.S	2004-08-26 14:25:45 -07:00
@@ -0,0 +1,33 @@
+/* U1copy_from_user.S: UltraSparc-I/II/IIi/IIe optimized copy from userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#define EX_LD(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		___copy_from_user
+#define LOAD(type,addr,dest)	type##a [addr] %asi, dest
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_AIUS, dest
+#define EX_RETVAL(x)		0
+
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
+
+#include "U1memcpy.S"
diff -Nru a/arch/sparc64/lib/U1copy_to_user.S b/arch/sparc64/lib/U1copy_to_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1copy_to_user.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,33 @@
+/* U1copy_to_user.S: UltraSparc-I/II/IIi/IIe optimized copy to userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#define EX_ST(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		___copy_to_user
+#define STORE(type,src,addr)	type##a src, [addr] ASI_AIUS
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_AIUS
+#define EX_RETVAL(x)		0
+
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
+
+#include "U1memcpy.S"
diff -Nru a/arch/sparc64/lib/U1memcpy.S b/arch/sparc64/lib/U1memcpy.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U1memcpy.S	2004-08-26 14:25:45 -07:00
@@ -0,0 +1,552 @@
+/* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
+ *
+ * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+
+#ifdef __KERNEL__
+#include <asm/visasm.h>
+#include <asm/asi.h>
+#else
+#define ASI_BLK_P 0xf0
+#define FPRS_FEF  0x04
+#ifdef MEMCPY_DEBUG
+#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
+		 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#else
+#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
+#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#endif
+#endif
+
+#ifndef EX_LD
+#define EX_LD(x)	x
+#endif
+
+#ifndef EX_ST
+#define EX_ST(x)	x
+#endif
+
+#ifndef EX_RETVAL
+#define EX_RETVAL(x)	x
+#endif
+
+#ifndef LOAD
+#define LOAD(type,addr,dest)	type [addr], dest
+#endif
+
+#ifndef LOAD_BLK
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_P, dest
+#endif
+
+#ifndef STORE
+#define STORE(type,src,addr)	type src, [addr]
+#endif
+
+#ifndef STORE_BLK
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
+#endif
+
+#ifndef FUNC_NAME
+#define FUNC_NAME	memcpy
+#endif
+
+#ifndef PREAMBLE
+#define PREAMBLE
+#endif
+
+#ifndef XCC
+#define XCC xcc
+#endif
+
+#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
+	faligndata		%f1, %f2, %f48;			\
+	faligndata		%f2, %f3, %f50;			\
+	faligndata		%f3, %f4, %f52;			\
+	faligndata		%f4, %f5, %f54;			\
+	faligndata		%f5, %f6, %f56;			\
+	faligndata		%f6, %f7, %f58;			\
+	faligndata		%f7, %f8, %f60;			\
+	faligndata		%f8, %f9, %f62;
+
+#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt)	\
+	EX_LD(LOAD_BLK(%src, %fdest));				\
+	EX_ST(STORE_BLK(%fsrc, %dest));				\
+	add			%src, 0x40, %src;		\
+	subcc			%len, 0x40, %len;		\
+	be,pn			%xcc, jmptgt;			\
+	 add			%dest, 0x40, %dest;		\
+
+#define LOOP_CHUNK1(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f0,  f48, len, branch_dest)
+#define LOOP_CHUNK2(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
+#define LOOP_CHUNK3(src, dest, len, branch_dest)		\
+	MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
+
+#define STORE_SYNC(dest, fsrc)				\
+	EX_ST(STORE_BLK(%fsrc, %dest));			\
+	add			%dest, 0x40, %dest;
+
+#define STORE_JUMP(dest, fsrc, target)			\
+	EX_ST(STORE_BLK(%fsrc, %dest));			\
+	add			%dest, 0x40, %dest;	\
+	ba,pt			%xcc, target;
+
+#define FINISH_VISCHUNK(dest, f0, f1, left)	\
+	subcc			%left, 8, %left;\
+	bl,pn			%xcc, 95f;	\
+	 faligndata		%f0, %f1, %f48;	\
+	EX_ST(STORE(std, %f48, %dest));		\
+	add			%dest, 8, %dest;
+
+#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
+	subcc			%left, 8, %left;	\
+	bl,pn			%xcc, 95f;		\
+	 fsrc1			%f0, %f1;
+
+#define UNEVEN_VISCHUNK(dest, f0, f1, left)		\
+	UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
+	ba,a,pt			%xcc, 93f;
+
+	.register	%g2,#scratch
+	.register	%g3,#scratch
+
+	.text
+	.align		64
+
+	.globl		FUNC_NAME
+FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+	PREAMBLE
+	mov		%o0, %g5
+	cmp		%o2, 0
+	be,pn		%XCC, 85f
+	 or		%o0, %o1, %o3
+	cmp		%o2, 16
+	blu,a,pn	%XCC, 80f
+	 or		%o3, %o2, %o3
+
+	cmp		%o2, (5 * 64)
+	blu,pt		%XCC, 70f
+	 andcc		%o3, 0x7, %g0
+
+	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  */
+	VISEntry
+
+	/* Is 'dst' already aligned on an 64-byte boundary? */
+	andcc		%o0, 0x3f, %g2
+	be,pt		%XCC, 2f
+
+	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
+	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
+	 * subtract this from 'len'.
+	 */
+	 sub		%o0, %o1, %o4
+	sub		%g2, 0x40, %g2
+	sub		%g0, %g2, %g2
+	sub		%o2, %g2, %o2
+	andcc		%g2, 0x7, %g1
+	be,pt		%icc, 2f
+	 and		%g2, 0x38, %g2
+
+1:	subcc		%g1, 0x1, %g1
+	EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
+	EX_ST(STORE(stb, %o3, %o1 + %o4))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x1, %o1
+
+	add		%o1, %o4, %o0
+
+2:	cmp		%g2, 0x0
+	and		%o1, 0x7, %g1
+	be,pt		%icc, 3f
+	 alignaddr	%o1, %g0, %o1
+
+	EX_LD(LOAD(ldd, %o1, %f4))
+1:	EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f4, %f6, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	be,pn		%icc, 3f
+	 add		%o0, 0x8, %o0
+
+	EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f6, %f4, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	bne,pt		%icc, 1b
+	 add		%o0, 0x8, %o0
+
+	/* Destination is 64-byte aligned.  */
+3:	
+	membar		  #LoadStore | #StoreStore | #StoreLoad
+
+	subcc		%o2, 0x40, %o4
+	add		%o1, %g1, %g1
+	andncc		%o4, (0x40 - 1), %o4
+	srl		%g1, 3, %g2
+	sub		%o2, %o4, %g3
+	andn		%o1, (0x40 - 1), %o1
+	and		%g2, 7, %g2
+	andncc		%g3, 0x7, %g3
+	fmovd		%f0, %f2
+	sub		%g3, 0x8, %g3
+	sub		%o2, %o4, %o2
+
+	add		%g1, %o4, %g1
+	subcc		%o2, %g3, %o2
+
+	EX_LD(LOAD_BLK(%o1, %f0))
+	add		%o1, 0x40, %o1
+	add		%g1, %g3, %g1
+	EX_LD(LOAD_BLK(%o1, %f16))
+	add		%o1, 0x40, %o1
+	sub		%o4, 0x80, %o4
+	EX_LD(LOAD_BLK(%o1, %f32))
+	add		%o1, 0x40, %o1
+
+	/* There are 8 instances of the unrolled loop,
+	 * one for each possible alignment of the
+	 * source buffer.  Each loop instance is 452
+	 * bytes.
+	 */
+	sll		%g2, 3, %o3
+	sub		%o3, %g2, %o3
+	sllx		%o3, 4, %o3
+	add		%o3, %g2, %o3
+	sllx		%o3, 2, %g2
+1:	rd		%pc, %o3
+	add		%o3, %lo(1f - 1b), %o3
+	jmpl		%o3 + %g2, %g0
+	 nop
+
+	.align		64
+1:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f0, %f2, %f48
+1:	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	STORE_JUMP(o0, f48, 40f) membar #Sync
+2:	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	STORE_JUMP(o0, f48, 48f) membar #Sync
+3:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+	STORE_JUMP(o0, f48, 56f) membar #Sync
+
+1:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f2, %f4, %f48
+1:	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	STORE_JUMP(o0, f48, 41f) membar #Sync
+2:	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	STORE_JUMP(o0, f48, 49f) membar #Sync
+3:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+	STORE_JUMP(o0, f48, 57f) membar #Sync
+
+1:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f4, %f6, %f48
+1:	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	STORE_JUMP(o0, f48, 42f) membar #Sync
+2:	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	STORE_JUMP(o0, f48, 50f) membar #Sync
+3:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+	STORE_JUMP(o0, f48, 58f) membar #Sync
+
+1:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f6, %f8, %f48
+1:	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
+	STORE_JUMP(o0, f48, 43f) membar #Sync
+2:	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	STORE_JUMP(o0, f48, 51f) membar #Sync
+3:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+	STORE_JUMP(o0, f48, 59f) membar #Sync
+
+1:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f8, %f10, %f48
+1:	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	STORE_JUMP(o0, f48, 44f) membar #Sync
+2:	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	STORE_JUMP(o0, f48, 52f) membar #Sync
+3:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+	STORE_JUMP(o0, f48, 60f) membar #Sync
+
+1:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f10, %f12, %f48
+1:	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	STORE_JUMP(o0, f48, 45f) membar #Sync
+2:	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	STORE_JUMP(o0, f48, 53f) membar #Sync
+3:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+	STORE_JUMP(o0, f48, 61f) membar #Sync
+
+1:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f12, %f14, %f48
+1:	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	STORE_JUMP(o0, f48, 46f) membar #Sync
+2:	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	STORE_JUMP(o0, f48, 54f) membar #Sync
+3:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+	STORE_JUMP(o0, f48, 62f) membar #Sync
+
+1:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	LOOP_CHUNK1(o1, o0, o4, 1f)
+	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	LOOP_CHUNK2(o1, o0, o4, 2f)
+	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	LOOP_CHUNK3(o1, o0, o4, 3f)
+	ba,pt		%xcc, 1b+4
+	 faligndata	%f14, %f16, %f48
+1:	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	STORE_JUMP(o0, f48, 47f) membar #Sync
+2:	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	STORE_JUMP(o0, f48, 55f) membar #Sync
+3:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+	STORE_SYNC(o0, f48) membar #Sync
+	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+	STORE_JUMP(o0, f48, 63f) membar #Sync
+
+40:	FINISH_VISCHUNK(o0, f0,  f2,  g3)
+41:	FINISH_VISCHUNK(o0, f2,  f4,  g3)
+42:	FINISH_VISCHUNK(o0, f4,  f6,  g3)
+43:	FINISH_VISCHUNK(o0, f6,  f8,  g3)
+44:	FINISH_VISCHUNK(o0, f8,  f10, g3)
+45:	FINISH_VISCHUNK(o0, f10, f12, g3)
+46:	FINISH_VISCHUNK(o0, f12, f14, g3)
+47:	UNEVEN_VISCHUNK(o0, f14, f0,  g3)
+48:	FINISH_VISCHUNK(o0, f16, f18, g3)
+49:	FINISH_VISCHUNK(o0, f18, f20, g3)
+50:	FINISH_VISCHUNK(o0, f20, f22, g3)
+51:	FINISH_VISCHUNK(o0, f22, f24, g3)
+52:	FINISH_VISCHUNK(o0, f24, f26, g3)
+53:	FINISH_VISCHUNK(o0, f26, f28, g3)
+54:	FINISH_VISCHUNK(o0, f28, f30, g3)
+55:	UNEVEN_VISCHUNK(o0, f30, f0,  g3)
+56:	FINISH_VISCHUNK(o0, f32, f34, g3)
+57:	FINISH_VISCHUNK(o0, f34, f36, g3)
+58:	FINISH_VISCHUNK(o0, f36, f38, g3)
+59:	FINISH_VISCHUNK(o0, f38, f40, g3)
+60:	FINISH_VISCHUNK(o0, f40, f42, g3)
+61:	FINISH_VISCHUNK(o0, f42, f44, g3)
+62:	FINISH_VISCHUNK(o0, f44, f46, g3)
+63:	UNEVEN_VISCHUNK_LAST(o0, f46, f0,  g3)
+
+93:	EX_LD(LOAD(ldd, %o1, %f2))
+	add		%o1, 8, %o1
+	subcc		%g3, 8, %g3
+	faligndata	%f0, %f2, %f8
+	EX_ST(STORE(std, %f8, %o0))
+	bl,pn		%xcc, 95f
+	 add		%o0, 8, %o0
+	EX_LD(LOAD(ldd, %o1, %f0))
+	add		%o1, 8, %o1
+	subcc		%g3, 8, %g3
+	faligndata	%f2, %f0, %f8
+	EX_ST(STORE(std, %f8, %o0))
+	bge,pt		%xcc, 93b
+	 add		%o0, 8, %o0
+
+95:	brz,pt		%o2, 2f
+	 mov		%g1, %o1
+
+1:	EX_LD(LOAD(ldub, %o1, %o3))
+	add		%o1, 1, %o1
+	subcc		%o2, 1, %o2
+	EX_ST(STORE(stb, %o3, %o0))
+	bne,pt		%xcc, 1b
+	 add		%o0, 1, %o0
+
+2:	membar		#StoreLoad | #StoreStore
+	VISExit
+	retl
+	 mov		EX_RETVAL(%g5), %o0
+
+	.align		64
+70:	/* 16 < len <= (5 * 64) */
+	bne,pn		%XCC, 75f
+	 sub		%o0, %o1, %o3
+
+72:	andn		%o2, 0xf, %o4
+	and		%o2, 0xf, %o2
+1:	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
+	subcc		%o4, 0x10, %o4
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+	EX_ST(STORE(stx, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x8, %o1
+73:	andcc		%o2, 0x8, %g0
+	be,pt		%XCC, 1f
+	 nop
+	EX_LD(LOAD(ldx, %o1, %o5))
+	sub		%o2, 0x8, %o2
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+1:	andcc		%o2, 0x4, %g0
+	be,pt		%XCC, 1f
+	 nop
+	EX_LD(LOAD(lduw, %o1, %o5))
+	sub		%o2, 0x4, %o2
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
+	add		%o1, 0x4, %o1
+1:	cmp		%o2, 0
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
+	 nop
+
+75:	andcc		%o0, 0x7, %g1
+	sub		%g1, 0x8, %g1
+	be,pn		%icc, 2f
+	 sub		%g0, %g1, %g1
+	sub		%o2, %g1, %o2
+
+1:	EX_LD(LOAD(ldub, %o1, %o5))
+	subcc		%g1, 1, %g1
+	EX_ST(STORE(stb, %o5, %o1 + %o3))
+	bgu,pt		%icc, 1b
+	 add		%o1, 1, %o1
+
+2:	add		%o1, %o3, %o0
+	andcc		%o1, 0x7, %g1
+	bne,pt		%icc, 8f
+	 sll		%g1, 3, %g1
+
+	cmp		%o2, 16
+	bgeu,pt		%icc, 72b
+	 nop
+	ba,a,pt		%xcc, 73b
+
+8:	mov		64, %o3
+	andn		%o1, 0x7, %o1
+	EX_LD(LOAD(ldx, %o1, %g2))
+	sub		%o3, %g1, %o3
+	andn		%o2, 0x7, %o4
+	sllx		%g2, %g1, %g2
+1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
+	subcc		%o4, 0x8, %o4
+	add		%o1, 0x8, %o1
+	srlx		%g3, %o3, %o5
+	or		%o5, %g2, %o5
+	EX_ST(STORE(stx, %o5, %o0))
+	add		%o0, 0x8, %o0
+	bgu,pt		%icc, 1b
+	 sllx		%g3, %g1, %g2
+
+	srl		%g1, 3, %g1
+	andcc		%o2, 0x7, %o2
+	be,pn		%icc, 85f
+	 add		%o1, %g1, %o1
+	ba,pt		%xcc, 90f
+	 sub		%o0, %o1, %o3
+
+	.align		64
+80:	/* 0 < len <= 16 */
+	andcc		%o3, 0x3, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+1:	EX_LD(LOAD(lduw, %o1, %g1))
+	subcc		%o2, 4, %o2
+	EX_ST(STORE(stw, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 4, %o1
+
+85:	retl
+	 mov		EX_RETVAL(%g5), %o0
+
+	.align		32
+90:	EX_LD(LOAD(ldub, %o1, %g1))
+	subcc		%o2, 1, %o2
+	EX_ST(STORE(stb, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 90b
+	 add		%o1, 1, %o1
+	retl
+	 mov		EX_RETVAL(%g5), %o0
diff -Nru a/arch/sparc64/lib/U3copy_from_user.S b/arch/sparc64/lib/U3copy_from_user.S
--- a/arch/sparc64/lib/U3copy_from_user.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3copy_from_user.S	2004-08-26 14:25:45 -07:00
@@ -3,428 +3,20 @@
  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
  */
 
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
+#define EX_LD(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		U3copy_from_user
+#define LOAD(type,addr,dest)	type##a [addr] %asi, dest
+#define EX_RETVAL(x)		0
 
-#define XCC xcc
-
-#define EXNV_RAW(x,y,a,b)		\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	a, b, %o1;			\
-	ba U3cfu_fixup;			\
-	 add %o1, 4, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV8(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	add %o1, %o3, %o0;		\
-	a, b, %o1;			\
-	ba U3cfu_fixup;			\
-	 add %o1, 8, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	ba U3cfu_fixup;			\
-	 a, b, %o1;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX2(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	add %o1, %o4, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, 0x1c0, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX3(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	sll %g3, 6, %g3;		\
-	add %o1, 0x80, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, %g3, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX4(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o1;	\
-	add %o1, 0x40, %o1;		\
-	ba U3cfu_fixup;			\
-	 add %o1, %g3, %o1;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	/* Special/non-trivial issues of this code:
-	 *
-	 * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
-	 * 2) Only low 32 FPU registers are used so that only the
-	 *    lower half of the FPU register set is dirtied by this
-	 *    code.  This is especially important in the kernel.
-	 * 3) This code never prefetches cachelines past the end
-	 *    of the source buffer.
-	 */
-
-	.text
-	.align	32
-
-	/* The cheetah's flexible spine, oversized liver, enlarged heart,
-	 * slender muscular body, and claws make it the swiftest hunter
-	 * in Africa and the fastest animal on land.  Can reach speeds
-	 * of up to 2.4GB per second.
-	 */
-
-	.globl	U3copy_from_user
-U3copy_from_user:	/* %o0=dst, %o1=src, %o2=len */
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
-	 andcc		%o3, 0x7, %g0
-
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
-	 */
-	.align		64
-enter:
-	/* Is 'dst' already aligned on an 64-byte boundary? */
-	be,pt		%XCC, 2f
-
-	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
-	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
-	 * subtract this from 'len'.
-	 */
-	 sub		%g2, 0x40, %g2
-	sub		%g0, %g2, %g2
-	sub		%o2, %g2, %o2
-
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	EXNV_RAW(lduba [%o1 + 0x00] %asi, %o3, add %o2, %g2)
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 stb		%o3, [%o0 + -1]
-
-2:	VISEntryHalf
-	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
-	 alignaddr	%o1, %g0, %o1
-
-	.align		64
-begin:
-
-	.globl		U3copy_from_user_nop_1_6
-U3copy_from_user_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-
-	prefetcha	[%o1 + 0x000] %asi, #one_read
-	prefetcha	[%o1 + 0x040] %asi, #one_read
-	andn		%o2, (0x40 - 1), %o4
-	prefetcha	[%o1 + 0x080] %asi, #one_read
-	prefetcha	[%o1 + 0x0c0] %asi, #one_read
-	EX(ldda [%o1 + 0x000] %asi, %f0, add %o2, %g0)
-	prefetcha	[%o1 + 0x100] %asi, #one_read
-	EX(ldda [%o1 + 0x008] %asi, %f2, add %o2, %g0)
-	prefetcha	[%o1 + 0x140] %asi, #one_read
-	EX(ldda [%o1 + 0x010] %asi, %f4, add %o2, %g0)
-	prefetcha	[%o1 + 0x180] %asi, #one_read
-	faligndata	%f0, %f2, %f16
-	EX(ldda [%o1 + 0x018] %asi, %f6, add %o2, %g0)
-	faligndata	%f2, %f4, %f18
-	EX(ldda [%o1 + 0x020] %asi, %f8, add %o2, %g0)
-	faligndata	%f4, %f6, %f20
-	EX(ldda [%o1 + 0x028] %asi, %f10, add %o2, %g0)
-	faligndata	%f6, %f8, %f22
-
-	EX(ldda [%o1 + 0x030] %asi, %f12, add %o2, %g0)
-	faligndata	%f8, %f10, %f24
-	EX(ldda [%o1 + 0x038] %asi, %f14, add %o2, %g0)
-	faligndata	%f10, %f12, %f26
-	EX(ldda [%o1 + 0x040] %asi, %f0, add %o2, %g0)
-
-	sub		%o4, 0x80, %o4
-	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
-	 srl		%o4, 6, %o3
-
-	.align		64
-loop:
-	EX3(ldda [%o1 + 0x008] %asi, %f2)
-	faligndata	%f12, %f14, %f28
-	EX3(ldda [%o1 + 0x010] %asi, %f4)
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	EX3(ldda [%o1 + 0x018] %asi, %f6)
-	faligndata	%f0, %f2, %f16
-
-	EX3(ldda [%o1 + 0x020] %asi, %f8)
-	faligndata	%f2, %f4, %f18
-	EX3(ldda [%o1 + 0x028] %asi, %f10)
-	faligndata	%f4, %f6, %f20
-	EX3(ldda [%o1 + 0x030] %asi, %f12)
-	faligndata	%f6, %f8, %f22
-	EX3(ldda [%o1 + 0x038] %asi, %f14)
-	faligndata	%f8, %f10, %f24
-
-	EX3(ldda [%o1 + 0x040] %asi, %f0)
-	prefetcha	[%o1 + 0x180] %asi, #one_read
-	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
-
-	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	EX3(ldda [%o1 + 0x008] %asi, %f2)
-	faligndata	%f12, %f14, %f28
-	EX3(ldda [%o1 + 0x010] %asi, %f4)
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	EX3(ldda [%o1 + 0x018] %asi, %f6)
-	faligndata	%f0, %f2, %f16
-	EX3(ldda [%o1 + 0x020] %asi, %f8)
-	faligndata	%f2, %f4, %f18
-	EX3(ldda [%o1 + 0x028] %asi, %f10)
-	faligndata	%f4, %f6, %f20
-	EX3(ldda [%o1 + 0x030] %asi, %f12)
-	faligndata	%f6, %f8, %f22
-	EX3(ldda [%o1 + 0x038] %asi, %f14)
-	faligndata	%f8, %f10, %f24
-	cmp		%g1, 0
-	be,pt		%XCC, 1f
-	 add		%o0, 0x40, %o0
-	EX4(ldda [%o1 + 0x040] %asi, %f0)
-1:	faligndata	%f10, %f12, %f26
-	faligndata	%f12, %f14, %f28
-	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	add		%o0, 0x40, %o0
-	add		%o1, 0x40, %o1
-
-	.globl		U3copy_from_user_nop_2_3
-U3copy_from_user_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-
-	membar		#Sync
-
-	/* Now we copy the (len modulo 64) bytes at the end.
-	 * Note how we borrow the %f0 loaded above.
-	 *
-	 * Also notice how this code is careful not to perform a
-	 * load past the end of the src buffer.
-	 */
-loopend:
-	and		%o2, 0x3f, %o2
-	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
-	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
-	 cmp		%g1, 0
-
-	be,a,pt		%XCC, 1f
-	 EX(ldda [%o1 + 0x00] %asi, %f0, add %o2, %g0)
-
-1:	EX(ldda [%o1 + 0x08] %asi, %f2, add %o2, %g0)
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f0, %f2, %f8
-	std		%f8, [%o0 + 0x00]
-	be,pn		%XCC, endcruft
-	 add		%o0, 0x8, %o0
-	EX(ldda [%o1 + 0x08] %asi, %f0, add %o2, %g0)
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f2, %f0, %f8
-	std		%f8, [%o0 + 0x00]
-	bne,pn		%XCC, 1b
-	 add		%o0, 0x8, %o0
-
-	/* If anything is left, we copy it one byte at a time.
-	 * Note that %g1 is (src & 0x3) saved above before the
-	 * alignaddr was performed.
-	 */
-endcruft:
-	cmp		%o2, 0
-	add		%o1, %g1, %o1
-	VISExitHalf
-	be,pn		%XCC, out
-	 sub		%o0, %o1, %o3
-
-	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
-	 andcc		%o2, 0x8, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(ldxa [%o1] %asi, %o5, add %o2, %g0)
-	stx		%o5, [%o1 + %o3]
-	add		%o1, 0x8, %o1
-
-1:	andcc		%o2, 0x4, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(lduwa [%o1] %asi, %o5, and %o2, 0x7)
-	stw		%o5, [%o1 + %o3]
-	add		%o1, 0x4, %o1
-
-1:	andcc		%o2, 0x2, %g0
-	be,pt		%icc, 1f
-	 nop
-	EXNV(lduha [%o1] %asi, %o5, and %o2, 0x3)
-	sth		%o5, [%o1 + %o3]
-	add		%o1, 0x2, %o1
-
-1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
-	 nop
-	EXNV(lduba [%o1] %asi, %o5, and %o2, 0x1)
-	ba,pt		%xcc, out
-	 stb		%o5, [%o1 + %o3]
-
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	EXNV8(ldxa [%o1] %asi, %o5, add %o2, %o4)
-	stx		%o5, [%o1 + %o3]
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	EXNV4(lduwa [%o1] %asi, %o5, add %o2, %g0)
-	stw		%o5, [%o1 + %o3]
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	EXNV(lduwa [%o1] %asi, %g1, add %o2, %g0)
-	stw		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
-
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	EXNV(lduba [%o1] %asi, %g1, add %o2, %g0)
-	stb		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
-
-U3cfu_fixup:
-	/* Since this is copy_from_user(), zero out the rest of the
-	 * kernel buffer.
-	 */
-	cmp		%o1, 0
-	ble,pn		%icc, 2f
-	 mov		%o1, %g2
-
-1:	subcc		%g2, 1, %g2
-	stb		%g0, [%o0]
-	bne,pt		%icc, 1b
-	 add		%o0, 1, %o0
-
-2:	retl
-	 mov		%o1, %o0
+#include "U3memcpy.S"
diff -Nru a/arch/sparc64/lib/U3copy_in_user.S b/arch/sparc64/lib/U3copy_in_user.S
--- a/arch/sparc64/lib/U3copy_in_user.S	2004-07-29 16:29:51 -07:00
+++ /dev/null	Wed Dec 31 16:00:00 196900
@@ -1,140 +0,0 @@
-/* U3copy_in_user.S: UltraSparc-III optimized memcpy.
- *
- * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
- */
-
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
-
-#define XCC xcc
-
-#define EXNV(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV1(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 1, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 4, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV8(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 8, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	.text
-	.align	32
-
-	/* Don't try to get too fancy here, just nice and
-	 * simple.  This is predominantly used for well aligned
-	 * small copies in the compat layer.  It is also used
-	 * to copy register windows around during thread cloning.
-	 */
-
-	.globl	U3copy_in_user
-U3copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
-	/* Writing to %asi is _expensive_ so we hardcode it.
-	 * Reading %asi to check for KERNEL_DS is comparatively
-	 * cheap.
-	 */
-	rd		%asi, %g1
-	cmp		%g1, ASI_AIUS
-	bne,pn		%icc, U3memcpy_user_stub
-	 nop
-
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-medium_copy: /* 16 < len <= 64 */
-	andcc		%o3, 0x7, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	EXNV8(ldxa [%o1] %asi, %o5, add %o4, %o2)
-	EXNV8(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	EXNV4(lduwa [%o1] %asi, %o5, add %o4, %o2)
-	EXNV4(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o4, %o2)
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	EXNV4(lduwa [%o1] %asi, %g1, add %o2, %g0)
-	EXNV4(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
-
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	EXNV1(lduba [%o1] %asi, %g1, add %o2, %g0)
-	EXNV1(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
diff -Nru a/arch/sparc64/lib/U3copy_to_user.S b/arch/sparc64/lib/U3copy_to_user.S
--- a/arch/sparc64/lib/U3copy_to_user.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3copy_to_user.S	2004-08-26 14:25:45 -07:00
@@ -1,433 +1,33 @@
-/* U3copy_to_user.S: UltraSparc-III optimized memcpy.
+/* U3copy_to_user.S: UltraSparc-III optimized copy to userspace.
  *
  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
  */
 
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
+#define EX_ST(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#define FUNC_NAME		U3copy_to_user
+#define STORE(type,src,addr)	type##a src, [addr] ASI_AIUS
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_AIUS
+#define EX_RETVAL(x)		0
 
-#define XCC xcc
-
-#define EXNV(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV2(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 1, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV3(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 4, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXNV4(x,y,a,b)	\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	a, b, %o0;			\
-	retl;				\
-	 add %o0, 8, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EX(x,y,a,b)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	retl;				\
-	 a, b, %o0;			\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK1(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	add %o4, 0x1c0, %o1;		\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o1, %o2, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK2(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	sll %o3, 6, %o3;		\
-	and %o2, (0x40 - 1), %o2;	\
-	add %o3, 0x80, %o1;		\
-	retl;				\
-	 add %o1, %o2, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK3(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o2, 0x80, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-#define EXBLK4(x,y)			\
-98:	x,y;				\
-	.section .fixup;		\
-	.align 4;			\
-99:	VISExitHalf;			\
-	and %o2, (0x40 - 1), %o2;	\
-	retl;				\
-	 add %o2, 0x40, %o0;		\
-	.section __ex_table;		\
-	.align 4;			\
-	.word 98b, 99b;			\
-	.text;				\
-	.align 4;
-
-	.register	%g2,#scratch
-	.register	%g3,#scratch
-
-	/* Special/non-trivial issues of this code:
-	 *
-	 * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
-	 * 2) Only low 32 FPU registers are used so that only the
-	 *    lower half of the FPU register set is dirtied by this
-	 *    code.  This is especially important in the kernel.
-	 * 3) This code never prefetches cachelines past the end
-	 *    of the source buffer.
-	 */
-
-	.text
-	.align	32
-
-	/* The cheetah's flexible spine, oversized liver, enlarged heart,
-	 * slender muscular body, and claws make it the swiftest hunter
-	 * in Africa and the fastest animal on land.  Can reach speeds
-	 * of up to 2.4GB per second.
-	 */
-
-	.globl	U3copy_to_user
-U3copy_to_user:	/* %o0=dst, %o1=src, %o2=len */
 	/* Writing to %asi is _expensive_ so we hardcode it.
 	 * Reading %asi to check for KERNEL_DS is comparatively
 	 * cheap.
 	 */
-	rd		%asi, %g1
-	cmp		%g1, ASI_AIUS
-	bne,pn		%icc, U3memcpy_user_stub
-	 nop
-
-	cmp		%o2, 0
-	be,pn		%XCC, out
-	 or		%o0, %o1, %o3
-	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
-	 or		%o3, %o2, %o3
-
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
-	 andcc		%o3, 0x7, %g0
-
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
-	 */
-	.align		64
-enter:
-	/* Is 'dst' already aligned on an 64-byte boundary? */
-	be,pt		%XCC, 2f
-
-	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
-	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
-	 * subtract this from 'len'.
-	 */
-	 sub		%g2, 0x40, %g2
-	sub		%g0, %g2, %g2
-	sub		%o2, %g2, %o2
-
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	ldub		[%o1 + 0x00], %o3
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 EXNV2(stba %o3, [%o0 + -1] %asi, add %o2, %g2)
-
-2:	VISEntryHalf
-	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
-	 alignaddr	%o1, %g0, %o1
-
-	.align		64
-begin:
-
-	.globl		U3copy_to_user_nop_1_6
-U3copy_to_user_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-
-	prefetch	[%o1 + 0x000], #one_read
-	prefetch	[%o1 + 0x040], #one_read
-	andn		%o2, (0x40 - 1), %o4
-	prefetch	[%o1 + 0x080], #one_read
-	prefetch	[%o1 + 0x0c0], #one_read
-	ldd		[%o1 + 0x000], %f0
-	prefetch	[%o1 + 0x100], #one_read
-	ldd		[%o1 + 0x008], %f2
-	prefetch	[%o1 + 0x140], #one_read
-	ldd		[%o1 + 0x010], %f4
-	prefetch	[%o1 + 0x180], #one_read
-	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f6, %f8, %f22
-
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f8, %f10, %f24
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f10, %f12, %f26
-	ldd		[%o1 + 0x040], %f0
-
-	sub		%o4, 0x80, %o4
-	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
-	 srl		%o4, 6, %o3
-
-	.align		64
-loop:
-	ldd		[%o1 + 0x008], %f2
-	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
-	faligndata	%f14, %f0, %f30
-	EXBLK2(stda %f16, [%o0] ASI_BLK_AIUS)
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f0, %f2, %f16
-
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
-
-	ldd		[%o1 + 0x040], %f0
-	prefetch	[%o1 + 0x180], #one_read
-	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
-
-	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	ldd		[%o1 + 0x008], %f2
-	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
-	faligndata	%f14, %f0, %f30
-	EXBLK3(stda %f16, [%o0] ASI_BLK_AIUS)
-	ldd		[%o1 + 0x018], %f6
-	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x020], %f8
-	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
-	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
-	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
-	cmp		%g1, 0
-	be,pt		%XCC, 1f
-	 add		%o0, 0x40, %o0
-	ldd		[%o1 + 0x040], %f0
-1:	faligndata	%f10, %f12, %f26
-	faligndata	%f12, %f14, %f28
-	faligndata	%f14, %f0, %f30
-	EXBLK4(stda %f16, [%o0] ASI_BLK_AIUS)
-	add		%o0, 0x40, %o0
-	add		%o1, 0x40, %o1
-
-	.globl		U3copy_to_user_nop_2_3
-U3copy_to_user_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-
-	membar		#Sync
-
-	/* Now we copy the (len modulo 64) bytes at the end.
-	 * Note how we borrow the %f0 loaded above.
-	 *
-	 * Also notice how this code is careful not to perform a
-	 * load past the end of the src buffer.
-	 */
-loopend:
-	and		%o2, 0x3f, %o2
-	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
-	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
-	 cmp		%g1, 0
-
-	be,a,pt		%XCC, 1f
-	 ldd		[%o1 + 0x00], %f0
-
-1:	ldd		[%o1 + 0x08], %f2
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f0, %f2, %f8
-	EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
-	be,pn		%XCC, endcruft
-	 add		%o0, 0x8, %o0
-	ldd		[%o1 + 0x08], %f0
-	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
-	subcc		%g2, 0x8, %g2
-	faligndata	%f2, %f0, %f8
-	EX(stda %f8, [%o0 + 0x00] %asi, add %o2, 0x8)
-	bne,pn		%XCC, 1b
-	 add		%o0, 0x8, %o0
-
-	/* If anything is left, we copy it one byte at a time.
-	 * Note that %g1 is (src & 0x3) saved above before the
-	 * alignaddr was performed.
-	 */
-endcruft:
-	cmp		%o2, 0
-	add		%o1, %g1, %o1
-	VISExitHalf
-	be,pn		%XCC, out
-	 sub		%o0, %o1, %o3
-
-	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
-	 andcc		%o2, 0x8, %g0
-	be,pt		%icc, 1f
-	 nop
-	ldx		[%o1], %o5
-	EXNV(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	add		%o1, 0x8, %o1
-
-1:	andcc		%o2, 0x4, %g0
-	be,pt		%icc, 1f
-	 nop
-	lduw		[%o1], %o5
-	EXNV(stwa %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x7)
-	add		%o1, 0x4, %o1
-
-1:	andcc		%o2, 0x2, %g0
-	be,pt		%icc, 1f
-	 nop
-	lduh		[%o1], %o5
-	EXNV(stha %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x3)
-	add		%o1, 0x2, %o1
-
-1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
-	 nop
-	ldub		[%o1], %o5
-	ba,pt		%xcc, out
-	 EXNV(stba %o5, [%o1 + %o3] ASI_AIUS, and %o2, 0x1)
-
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	ldx		[%o1], %o5
-	EXNV4(stxa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %o4)
-	bgu,pt		%XCC, 1b
-	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
-	be,pt		%XCC, 1f
-	 nop
-	sub		%o2, 0x4, %o2
-	lduw		[%o1], %o5
-	EXNV3(stwa %o5, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	add		%o1, 0x4, %o1
-1:	cmp		%o2, 0
-	be,pt		%XCC, out
-	 nop
-	ba,pt		%xcc, small_copy_unaligned
-	 nop
-
-small_copy: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
-	 sub		%o0, %o1, %o3
-
-small_copy_aligned:
-	subcc		%o2, 4, %o2
-	lduw		[%o1], %g1
-	EXNV3(stwa %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_aligned
-	 add		%o1, 4, %o1
-
-out:	retl
-	 clr		%o0
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
 
-	.align	32
-small_copy_unaligned:
-	subcc		%o2, 1, %o2
-	ldub		[%o1], %g1
-	EXNV2(stba %g1, [%o1 + %o3] ASI_AIUS, add %o2, %g0)
-	bgu,pt		%XCC, small_copy_unaligned
-	 add		%o1, 1, %o1
-	retl
-	 clr		%o0
+#include "U3memcpy.S"
diff -Nru a/arch/sparc64/lib/U3memcpy.S b/arch/sparc64/lib/U3memcpy.S
--- a/arch/sparc64/lib/U3memcpy.S	2004-07-29 16:29:51 -07:00
+++ b/arch/sparc64/lib/U3memcpy.S	2004-08-26 14:25:45 -07:00
@@ -6,14 +6,50 @@
 #ifdef __KERNEL__
 #include <asm/visasm.h>
 #include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
 #else
 #define ASI_BLK_P 0xf0
 #define FPRS_FEF  0x04
+#ifdef MEMCPY_DEBUG
+#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
+		     clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#else
 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
 #endif
+#endif
+
+#ifndef EX_LD
+#define EX_LD(x)	x
+#endif
+
+#ifndef EX_ST
+#define EX_ST(x)	x
+#endif
+
+#ifndef EX_RETVAL
+#define EX_RETVAL(x)	x
+#endif
+
+#ifndef LOAD
+#define LOAD(type,addr,dest)	type [addr], dest
+#endif
+
+#ifndef STORE
+#define STORE(type,src,addr)	type src, [addr]
+#endif
+
+#ifndef STORE_BLK
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
+#endif
+
+#ifndef FUNC_NAME
+#define FUNC_NAME	U3memcpy
+#endif
+
+#ifndef PREAMBLE
+#define PREAMBLE
+#endif
 
 #ifndef XCC
 #define XCC xcc
@@ -33,7 +69,7 @@
 	 */
 
 	.text
-	.align	32
+	.align		64
 
 	/* The cheetah's flexible spine, oversized liver, enlarged heart,
 	 * slender muscular body, and claws make it the swiftest hunter
@@ -41,156 +77,159 @@
 	 * of up to 2.4GB per second.
 	 */
 
-	.globl	U3memcpy
-U3memcpy:	/* %o0=dst, %o1=src, %o2=len */
+	.globl	FUNC_NAME
+FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+	PREAMBLE
 	mov		%o0, %g5
 	cmp		%o2, 0
-	be,pn		%XCC, out
+	be,pn		%XCC, 85f
 	 or		%o0, %o1, %o3
 	cmp		%o2, 16
-	bleu,a,pn	%XCC, small_copy
+	blu,a,pn	%XCC, 80f
 	 or		%o3, %o2, %o3
 
-	cmp		%o2, 256
-	blu,pt		%XCC, medium_copy
+	cmp		%o2, (3 * 64)
+	blu,pt		%XCC, 70f
 	 andcc		%o3, 0x7, %g0
 
-	ba,pt		%xcc, enter
-	 andcc		%o0, 0x3f, %g2
-
-	/* Here len >= 256 and condition codes reflect execution
-	 * of "andcc %o0, 0x7, %g2", done by caller.
+	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  We must preserve
+	 * o5 from here until we hit VISExitHalf.
 	 */
-	.align		64
-enter:
+	VISEntryHalf
+
 	/* Is 'dst' already aligned on an 64-byte boundary? */
+	andcc		%o0, 0x3f, %g2
 	be,pt		%XCC, 2f
 
 	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
 	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
 	 * subtract this from 'len'.
 	 */
-	 sub		%g2, 0x40, %g2
+	 sub		%o0, %o1, %o4
+	sub		%g2, 0x40, %g2
 	sub		%g0, %g2, %g2
 	sub		%o2, %g2, %o2
+	andcc		%g2, 0x7, %g1
+	be,pt		%icc, 2f
+	 and		%g2, 0x38, %g2
+
+1:	subcc		%g1, 0x1, %g1
+	EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
+	EX_ST(STORE(stb, %o3, %o1 + %o4))
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x1, %o1
 
-	/* Copy %g2 bytes from src to dst, one byte at a time. */
-1:	ldub		[%o1 + 0x00], %o3
-	add		%o1, 0x1, %o1
-	add		%o0, 0x1, %o0
-	subcc		%g2, 0x1, %g2
-
-	bg,pt		%XCC, 1b
-	 stb		%o3, [%o0 + -1]
+	add		%o1, %o4, %o0
 
-2:	VISEntryHalf
+2:	cmp		%g2, 0x0
 	and		%o1, 0x7, %g1
-	ba,pt		%xcc, begin
+	be,pt		%icc, 3f
 	 alignaddr	%o1, %g0, %o1
 
-	.align		64
-begin:
-#ifdef __KERNEL__
-	.globl		U3memcpy_nop_1_6
-U3memcpy_nop_1_6:
-	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi		%uhi(DCU_PE), %o3
-	sllx		%o3, 32, %o3
-	or		%g3, %o3, %o3
-	stxa		%o3, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
-	membar		#Sync
-#endif
-	prefetch	[%o1 + 0x000], #one_read
-	prefetch	[%o1 + 0x040], #one_read
+	EX_LD(LOAD(ldd, %o1, %f4))
+1:	EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f4, %f6, %f0
+	EX_ST(STORE(std, %f0, %o0))
+	be,pn		%icc, 3f
+	 add		%o0, 0x8, %o0
+
+	EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
+	add		%o1, 0x8, %o1
+	subcc		%g2, 0x8, %g2
+	faligndata	%f6, %f4, %f2
+	EX_ST(STORE(std, %f2, %o0))
+	bne,pt		%icc, 1b
+	 add		%o0, 0x8, %o0
+
+3:	LOAD(prefetch, %o1 + 0x000, #one_read)
+	LOAD(prefetch, %o1 + 0x040, #one_read)
 	andn		%o2, (0x40 - 1), %o4
-	prefetch	[%o1 + 0x080], #one_read
-	prefetch	[%o1 + 0x0c0], #one_read
-	ldd		[%o1 + 0x000], %f0
-	prefetch	[%o1 + 0x100], #one_read
-	ldd		[%o1 + 0x008], %f2
-	prefetch	[%o1 + 0x140], #one_read
-	ldd		[%o1 + 0x010], %f4
-	prefetch	[%o1 + 0x180], #one_read
+	LOAD(prefetch, %o1 + 0x080, #one_read)
+	LOAD(prefetch, %o1 + 0x0c0, #one_read)
+	LOAD(prefetch, %o1 + 0x100, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x000, %f0))
+	LOAD(prefetch, %o1 + 0x140, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
+	LOAD(prefetch, %o1 + 0x180, #one_read)
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
+	LOAD(prefetch, %o1 + 0x1c0, #one_read)
 	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x018], %f6
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f6, %f8, %f22
 
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
 	faligndata	%f8, %f10, %f24
-	ldd		[%o1 + 0x038], %f14
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 	faligndata	%f10, %f12, %f26
-	ldd		[%o1 + 0x040], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
 
-	sub		%o4, 0x80, %o4
+	subcc		%o4, 0x80, %o4
 	add		%o1, 0x40, %o1
-	ba,pt		%xcc, loop
+	bgu,pt		%XCC, 1f
 	 srl		%o4, 6, %o3
+	ba,pt		%xcc, 2f
+	 nop
 
 	.align		64
-loop:
-	ldd		[%o1 + 0x008], %f2
+1:
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
 	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	ldd		[%o1 + 0x018], %f6
+	EX_ST(STORE_BLK(%f16, %o0))
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f0, %f2, %f16
+	add		%o0, 0x40, %o0
 
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
+	subcc		%o3, 0x01, %o3
 	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
-	faligndata	%f8, %f10, %f24
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 
-	ldd		[%o1 + 0x040], %f0
-	prefetch	[%o1 + 0x180], #one_read
+	faligndata	%f8, %f10, %f24
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
+	LOAD(prefetch, %o1 + 0x1c0, #one_read)
 	faligndata	%f10, %f12, %f26
-	subcc		%o3, 0x01, %o3
-	add		%o1, 0x40, %o1
-	bg,pt		%XCC, loop
-	 add		%o0, 0x40, %o0
+	bg,pt		%XCC, 1b
+	 add		%o1, 0x40, %o1
 
 	/* Finally we copy the last full 64-byte block. */
-loopfini:
-	ldd		[%o1 + 0x008], %f2
+2:
+	EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
 	faligndata	%f12, %f14, %f28
-	ldd		[%o1 + 0x010], %f4
+	EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
-	ldd		[%o1 + 0x018], %f6
+	EX_ST(STORE_BLK(%f16, %o0))
+	EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
 	faligndata	%f0, %f2, %f16
-	ldd		[%o1 + 0x020], %f8
+	EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
 	faligndata	%f2, %f4, %f18
-	ldd		[%o1 + 0x028], %f10
+	EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
 	faligndata	%f4, %f6, %f20
-	ldd		[%o1 + 0x030], %f12
+	EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
 	faligndata	%f6, %f8, %f22
-	ldd		[%o1 + 0x038], %f14
+	EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
 	faligndata	%f8, %f10, %f24
 	cmp		%g1, 0
 	be,pt		%XCC, 1f
 	 add		%o0, 0x40, %o0
-	ldd		[%o1 + 0x040], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
 1:	faligndata	%f10, %f12, %f26
 	faligndata	%f12, %f14, %f28
 	faligndata	%f14, %f0, %f30
-	stda		%f16, [%o0] ASI_BLK_P
+	EX_ST(STORE_BLK(%f16, %o0))
 	add		%o0, 0x40, %o0
 	add		%o1, 0x40, %o1
-#ifdef __KERNEL__
-	.globl		U3memcpy_nop_2_3
-U3memcpy_nop_2_3:
-	mov		PRIMARY_CONTEXT, %o3
-	stxa		%g0, [%o3] ASI_DMMU		! Flush P-cache
-	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
-#endif
 	membar		#Sync
 
 	/* Now we copy the (len modulo 64) bytes at the end.
@@ -199,31 +238,29 @@
 	 * Also notice how this code is careful not to perform a
 	 * load past the end of the src buffer.
 	 */
-loopend:
 	and		%o2, 0x3f, %o2
 	andcc		%o2, 0x38, %g2
-	be,pn		%XCC, endcruft
+	be,pn		%XCC, 2f
 	 subcc		%g2, 0x8, %g2
-	be,pn		%XCC, endcruft
+	be,pn		%XCC, 2f
 	 cmp		%g1, 0
 
+	sub		%o2, %g2, %o2
 	be,a,pt		%XCC, 1f
-	 ldd		[%o1 + 0x00], %f0
+	 EX_LD(LOAD(ldd, %o1 + 0x00, %f0))
 
-1:	ldd		[%o1 + 0x08], %f2
+1:	EX_LD(LOAD(ldd, %o1 + 0x08, %f2))
 	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
 	subcc		%g2, 0x8, %g2
 	faligndata	%f0, %f2, %f8
-	std		%f8, [%o0 + 0x00]
-	be,pn		%XCC, endcruft
+	EX_ST(STORE(std, %f8, %o0))
+	be,pn		%XCC, 2f
 	 add		%o0, 0x8, %o0
-	ldd		[%o1 + 0x08], %f0
+	EX_LD(LOAD(ldd, %o1 + 0x08, %f0))
 	add		%o1, 0x8, %o1
-	sub		%o2, 0x8, %o2
 	subcc		%g2, 0x8, %g2
 	faligndata	%f2, %f0, %f8
-	std		%f8, [%o0 + 0x00]
+	EX_ST(STORE(std, %f8, %o0))
 	bne,pn		%XCC, 1b
 	 add		%o0, 0x8, %o0
 
@@ -231,103 +268,147 @@
 	 * Note that %g1 is (src & 0x3) saved above before the
 	 * alignaddr was performed.
 	 */
-endcruft:
+2:
 	cmp		%o2, 0
 	add		%o1, %g1, %o1
 	VISExitHalf
-	be,pn		%XCC, out
+	be,pn		%XCC, 85f
 	 sub		%o0, %o1, %o3
 
 	andcc		%g1, 0x7, %g0
-	bne,pn		%icc, small_copy_unaligned
+	bne,pn		%icc, 90f
 	 andcc		%o2, 0x8, %g0
 	be,pt		%icc, 1f
 	 nop
-	ldx		[%o1], %o5
-	stx		%o5, [%o1 + %o3]
+	EX_LD(LOAD(ldx, %o1, %o5))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
 	add		%o1, 0x8, %o1
 
 1:	andcc		%o2, 0x4, %g0
 	be,pt		%icc, 1f
 	 nop
-	lduw		[%o1], %o5
-	stw		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduw, %o1, %o5))
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
 	add		%o1, 0x4, %o1
 
 1:	andcc		%o2, 0x2, %g0
 	be,pt		%icc, 1f
 	 nop
-	lduh		[%o1], %o5
-	sth		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduh, %o1, %o5))
+	EX_ST(STORE(sth, %o5, %o1 + %o3))
 	add		%o1, 0x2, %o1
 
 1:	andcc		%o2, 0x1, %g0
-	be,pt		%icc, out
+	be,pt		%icc, 85f
 	 nop
-	ldub		[%o1], %o5
-	ba,pt		%xcc, out
-	 stb		%o5, [%o1 + %o3]
+	EX_LD(LOAD(ldub, %o1, %o5))
+	ba,pt		%xcc, 85f
+	 EX_ST(STORE(stb, %o5, %o1 + %o3))
 
-medium_copy: /* 16 < len <= 64 */
-	bne,pn		%XCC, small_copy_unaligned
+	.align		64
+70: /* 16 < len <= 64 */
+	bne,pn		%XCC, 75f
 	 sub		%o0, %o1, %o3
 
-medium_copy_aligned:
-	andn		%o2, 0x7, %o4
-	and		%o2, 0x7, %o2
-1:	subcc		%o4, 0x8, %o4
-	ldx		[%o1], %o5
-	stx		%o5, [%o1 + %o3]
+72:
+	andn		%o2, 0xf, %o4
+	and		%o2, 0xf, %o2
+1:	subcc		%o4, 0x10, %o4
+	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+	EX_ST(STORE(stx, %g1, %o1 + %o3))
 	bgu,pt		%XCC, 1b
 	 add		%o1, 0x8, %o1
-	andcc		%o2, 0x4, %g0
+73:	andcc		%o2, 0x8, %g0
+	be,pt		%XCC, 1f
+	 nop
+	sub		%o2, 0x8, %o2
+	EX_LD(LOAD(ldx, %o1, %o5))
+	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	add		%o1, 0x8, %o1
+1:	andcc		%o2, 0x4, %g0
 	be,pt		%XCC, 1f
 	 nop
 	sub		%o2, 0x4, %o2
-	lduw		[%o1], %o5
-	stw		%o5, [%o1 + %o3]
+	EX_LD(LOAD(lduw, %o1, %o5))
+	EX_ST(STORE(stw, %o5, %o1 + %o3))
 	add		%o1, 0x4, %o1
 1:	cmp		%o2, 0
-	be,pt		%XCC, out
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
 	 nop
-	ba,pt		%xcc, small_copy_unaligned
+
+75:
+	andcc		%o0, 0x7, %g1
+	sub		%g1, 0x8, %g1
+	be,pn		%icc, 2f
+	 sub		%g0, %g1, %g1
+	sub		%o2, %g1, %o2
+
+1:	subcc		%g1, 1, %g1
+	EX_LD(LOAD(ldub, %o1, %o5))
+	EX_ST(STORE(stb, %o5, %o1 + %o3))
+	bgu,pt		%icc, 1b
+	 add		%o1, 1, %o1
+
+2:	add		%o1, %o3, %o0
+	andcc		%o1, 0x7, %g1
+	bne,pt		%icc, 8f
+	 sll		%g1, 3, %g1
+
+	cmp		%o2, 16
+	bgeu,pt		%icc, 72b
 	 nop
+	ba,a,pt		%xcc, 73b
+
+8:	mov		64, %o3
+	andn		%o1, 0x7, %o1
+	EX_LD(LOAD(ldx, %o1, %g2))
+	sub		%o3, %g1, %o3
+	andn		%o2, 0x7, %o4
+	sllx		%g2, %g1, %g2
+1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
+	subcc		%o4, 0x8, %o4
+	add		%o1, 0x8, %o1
+	srlx		%g3, %o3, %o5
+	or		%o5, %g2, %o5
+	EX_ST(STORE(stx, %o5, %o0))
+	add		%o0, 0x8, %o0
+	bgu,pt		%icc, 1b
+	 sllx		%g3, %g1, %g2
+
+	srl		%g1, 3, %g1
+	andcc		%o2, 0x7, %o2
+	be,pn		%icc, 85f
+	 add		%o1, %g1, %o1
+	ba,pt		%xcc, 90f
+	 sub		%o0, %o1, %o3
 
-small_copy: /* 0 < len <= 16 */
+	.align		64
+80: /* 0 < len <= 16 */
 	andcc		%o3, 0x3, %g0
-	bne,pn		%XCC, small_copy_unaligned
+	bne,pn		%XCC, 90f
 	 sub		%o0, %o1, %o3
 
-small_copy_aligned:
+1:
 	subcc		%o2, 4, %o2
-	lduw		[%o1], %g1
-	stw		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_aligned
+	EX_LD(LOAD(lduw, %o1, %g1))
+	EX_ST(STORE(stw, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 1b
 	 add		%o1, 4, %o1
 
-out:	retl
-	 mov		%g5, %o0
+85:	retl
+	 mov		EX_RETVAL(%g5), %o0
 
-	.align	32
-small_copy_unaligned:
+	.align		32
+90:
 	subcc		%o2, 1, %o2
-	ldub		[%o1], %g1
-	stb		%g1, [%o1 + %o3]
-	bgu,pt		%XCC, small_copy_unaligned
+	EX_LD(LOAD(ldub, %o1, %g1))
+	EX_ST(STORE(stb, %g1, %o1 + %o3))
+	bgu,pt		%XCC, 90b
 	 add		%o1, 1, %o1
 	retl
-	 mov		%g5, %o0
-
-	/* Act like copy_{to,in}_user(), ie. return zero instead
-	 * of original destination pointer.  This is invoked when
-	 * copy_{to,in}_user() finds that %asi is kernel space.
-	 */
-	.globl	U3memcpy_user_stub
-U3memcpy_user_stub:
-	save		%sp, -192, %sp
-	mov		%i0, %o0
-	mov		%i1, %o1
-	call		U3memcpy
-	 mov		%i2, %o2
-	ret
-	 restore	%g0, %g0, %o0
+	 mov		EX_RETVAL(%g5), %o0
diff -Nru a/arch/sparc64/lib/U3patch.S b/arch/sparc64/lib/U3patch.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/U3patch.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,30 @@
+/* U3patch.S: Patch Ultra-I routines with Ultra-III variant.
+ *
+ * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+ */
+
+#define BRANCH_ALWAYS	0x10680000
+#define NOP		0x01000000
+#define ULTRA3_DO_PATCH(OLD, NEW)	\
+	sethi	%hi(NEW), %g1; \
+	or	%g1, %lo(NEW), %g1; \
+	sethi	%hi(OLD), %g2; \
+	or	%g2, %lo(OLD), %g2; \
+	sub	%g1, %g2, %g1; \
+	sethi	%hi(BRANCH_ALWAYS), %g3; \
+	srl	%g1, 2, %g1; \
+	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
+	or	%g3, %g1, %g3; \
+	stw	%g3, [%g2]; \
+	sethi	%hi(NOP), %g3; \
+	or	%g3, %lo(NOP), %g3; \
+	stw	%g3, [%g2 + 0x4]; \
+	flush	%g2;
+
+	.globl	cheetah_patch_copyops
+cheetah_patch_copyops:
+	ULTRA3_DO_PATCH(memcpy, U3memcpy)
+	ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user)
+	ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user)
+	retl
+	 nop
diff -Nru a/arch/sparc64/lib/VIScopy.S b/arch/sparc64/lib/VIScopy.S
--- a/arch/sparc64/lib/VIScopy.S	2004-07-27 13:08:37 -07:00
+++ /dev/null	Wed Dec 31 16:00:00 196900
@@ -1,1183 +0,0 @@
-/* $Id: VIScopy.S,v 1.26 2001/09/27 04:36:24 kanoj Exp $
- * VIScopy.S: High speed copy operations utilizing the UltraSparc
- *            Visual Instruction Set.
- *
- * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
- * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- */
-
-#include "VIS.h"
-
-	/* VIS code can be used for numerous copy/set operation variants.
-	 * It can be made to work in the kernel, one single instance,
-	 * for all of memcpy, copy_to_user, and copy_from_user by setting
-	 * the ASI src/dest globals correctly.  Furthermore it can
-	 * be used for kernel-->kernel page copies as well, a hook label
-	 * is put in here just for this purpose.
-	 *
-	 * For userland, compiling this without __KERNEL__ defined makes
-	 * it work just fine as a generic libc bcopy and memcpy.
-	 * If for userland it is compiled with a 32bit gcc (but you need
-	 * -Wa,-Av9a for as), the code will just rely on lower 32bits of
-	 * IEU registers, if you compile it with 64bit gcc (ie. define
-	 * __sparc_v9__), the code will use full 64bit.
-	 */
-	 
-#ifdef __KERNEL__
-
-#include <asm/visasm.h>
-#include <asm/asm_offsets.h>
-
-#define FPU_CLEAN_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	VISExit									\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define FPU_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	VISExit									\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define NORMAL_RETL								\
-	ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o1;	\
-	clr		%o0;							\
-	retl;									\
-	 wr		%o1, %g0, %asi;
-#define EX(x,y,a,b) 				\
-98: 	x,y;					\
-	.section .fixup;			\
-	.align	4;				\
-99:	ba	VIScopyfixup_ret;		\
-	 a, b, %o1;				\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, 99b;			\
-	.text;					\
-	.align	4;
-#define EX2(x,y,c,d,e,a,b) 			\
-98: 	x,y;					\
-	.section .fixup;			\
-	.align	4;				\
-99:	c, d, e;				\
-	ba	VIScopyfixup_ret;		\
-	 a, b, %o1;				\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, 99b;			\
-	.text;					\
-	.align	4;
-#define EXO2(x,y) 				\
-98: 	x,y;					\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, VIScopyfixup_reto2;	\
-	.text;					\
-	.align	4;
-#define EXVISN(x,y,n) 				\
-98: 	x,y;					\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	98b, VIScopyfixup_vis##n;	\
-	.text;					\
-	.align	4;
-#define EXT(start,end,handler) 			\
-	.section __ex_table;			\
-	.align	4;				\
-	.word	start, 0, end, handler;		\
-	.text;					\
-	.align	4;
-#else
-#ifdef REGS_64BIT
-#define FPU_CLEAN_RETL				\
-	retl;					\
-	 mov	%g6, %o0;
-#define FPU_RETL				\
-	retl;					\
-	 mov	%g6, %o0;
-#else
-#define FPU_CLEAN_RETL				\
-	wr	%g0, FPRS_FEF, %fprs;		\
-	retl;					\
-	 mov	%g6, %o0;
-#define FPU_RETL				\
-	wr	%g0, FPRS_FEF, %fprs;		\
-	retl;					\
-	 mov	%g6, %o0;
-#endif
-#define NORMAL_RETL	\
-	retl;		\
-	 mov	%g6, %o0;
-#define EX(x,y,a,b)		x,y
-#define EX2(x,y,c,d,e,a,b)	x,y
-#define EXO2(x,y)		x,y
-#define EXVISN(x,y,n)		x,y
-#define EXT(a,b,c)
-#endif
-#define EXVIS(x,y) EXVISN(x,y,0)
-#define EXVIS1(x,y) EXVISN(x,y,1)
-#define EXVIS2(x,y) EXVISN(x,y,2)
-#define EXVIS3(x,y) EXVISN(x,y,3)
-#define EXVIS4(x,y) EXVISN(x,y,4)
-
-#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
-	faligndata		%f1, %f2, %f48;			\
-	faligndata		%f2, %f3, %f50;			\
-	faligndata		%f3, %f4, %f52;			\
-	faligndata		%f4, %f5, %f54;			\
-	faligndata		%f5, %f6, %f56;			\
-	faligndata		%f6, %f7, %f58;			\
-	faligndata		%f7, %f8, %f60;			\
-	faligndata		%f8, %f9, %f62;
-
-#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt)	\
-	EXVIS(LDBLK		[%src] ASIBLK, %fdest);		\
-	ASI_SETDST_BLK						\
-	EXVIS(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%src, 0x40, %src;		\
-	subcc			%len, 0x40, %len;		\
-	be,pn			%xcc, jmptgt;			\
-	 add			%dest, 0x40, %dest;		\
-	ASI_SETSRC_BLK
-
-#define LOOP_CHUNK1(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f0,  f48, len, branch_dest)
-#define LOOP_CHUNK2(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
-#define LOOP_CHUNK3(src, dest, len, branch_dest)		\
-	MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
-
-#define STORE_SYNC(dest, fsrc)					\
-	EXVIS(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%dest, 0x40, %dest;
-
-#ifdef __KERNEL__
-#define STORE_JUMP(dest, fsrc, target)				\
-	srl			asi_dest, 3, %g5;		\
-	EXVIS2(STBLK		%fsrc, [%dest] ASIBLK);		\
-	xor		       asi_dest, ASI_BLK_XOR1, asi_dest;\
-	add			%dest, 0x40, %dest;		\
-	xor			asi_dest, %g5, asi_dest;	\
-	ba,pt			%xcc, target;
-#else
-#define STORE_JUMP(dest, fsrc, target)				\
-	EXVIS2(STBLK		%fsrc, [%dest] ASIBLK);		\
-	add			%dest, 0x40, %dest;		\
-	ba,pt			%xcc, target;
-#endif
-
-#ifndef __KERNEL__
-#define VISLOOP_PAD nop; nop; nop; nop; \
-		    nop; nop; nop; nop; \
-		    nop; nop; nop; nop; \
-		    nop; nop; nop;
-#else
-#define VISLOOP_PAD
-#endif
-
-#define FINISH_VISCHUNK(dest, f0, f1, left)			\
-	ASI_SETDST_NOBLK					\
-	subcc			%left, 8, %left;		\
-	bl,pn			%xcc, vis_out;			\
-	 faligndata		%f0, %f1, %f48;			\
-	EXVIS3(STDF		%f48, [%dest] ASINORMAL);	\
-	add			%dest, 8, %dest;
-
-#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)		\
-	subcc			%left, 8, %left;		\
-	bl,pn			%xcc, vis_out;			\
-	 fsrc1			%f0, %f1;
-#define UNEVEN_VISCHUNK(dest, f0, f1, left)			\
-	UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)		\
-	ba,a,pt			%xcc, vis_out_slk;
-
-	/* Macros for non-VIS memcpy code. */
-#ifdef REGS_64BIT
-
-#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3)			\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x00] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x08] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x10] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x18] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STW			%t0, [%dst + offset + 0x04] ASINORMAL; 	\
-	srlx			%t0, 32, %t0;				\
-	STW			%t0, [%dst + offset + 0x00] ASINORMAL; 	\
-	STW			%t1, [%dst + offset + 0x0c] ASINORMAL; 	\
-	srlx			%t1, 32, %t1;				\
-	STW			%t1, [%dst + offset + 0x08] ASINORMAL; 	\
-	STW			%t2, [%dst + offset + 0x14] ASINORMAL; 	\
-	srlx			%t2, 32, %t2;				\
-	STW			%t2, [%dst + offset + 0x10] ASINORMAL; 	\
-	STW			%t3, [%dst + offset + 0x1c] ASINORMAL;	\
-	srlx			%t3, 32, %t3;				\
-	STW			%t3, [%dst + offset + 0x18] ASINORMAL;
-
-#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x00] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x08] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x10] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x18] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst + offset + 0x00] ASINORMAL; 	\
-	STX			%t1, [%dst + offset + 0x08] ASINORMAL; 	\
-	STX			%t2, [%dst + offset + 0x10] ASINORMAL; 	\
-	STX			%t3, [%dst + offset + 0x18] ASINORMAL; 	\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src + offset + 0x20] ASINORMAL, %t0; 	\
-	LDX			[%src + offset + 0x28] ASINORMAL, %t1; 	\
-	LDX			[%src + offset + 0x30] ASINORMAL, %t2; 	\
-	LDX			[%src + offset + 0x38] ASINORMAL, %t3; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst + offset + 0x20] ASINORMAL; 	\
-	STX			%t1, [%dst + offset + 0x28] ASINORMAL; 	\
-	STX			%t2, [%dst + offset + 0x30] ASINORMAL; 	\
-	STX			%t3, [%dst + offset + 0x38] ASINORMAL;
-
-#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src - offset - 0x10] ASINORMAL, %t0;	\
-	LDX			[%src - offset - 0x08] ASINORMAL, %t1; 	\
-	ASI_SETDST_NOBLK						\
-	STW			%t0, [%dst - offset - 0x0c] ASINORMAL; 	\
-	srlx			%t0, 32, %t2;				\
-	STW			%t2, [%dst - offset - 0x10] ASINORMAL; 	\
-	STW			%t1, [%dst - offset - 0x04] ASINORMAL; 	\
-	srlx			%t1, 32, %t3;				\
-	STW			%t3, [%dst - offset - 0x08] ASINORMAL;
-
-#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1)			\
-	ASI_SETSRC_NOBLK						\
-	LDX			[%src - offset - 0x10] ASINORMAL, %t0; 	\
-	LDX			[%src - offset - 0x08] ASINORMAL, %t1; 	\
-	ASI_SETDST_NOBLK						\
-	STX			%t0, [%dst - offset - 0x10] ASINORMAL; 	\
-	STX			%t1, [%dst - offset - 0x08] ASINORMAL;
-
-#else /* !REGS_64BIT */
-
-#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3)			\
-	lduw			[%src + offset + 0x00], %t0; 		\
-	lduw			[%src + offset + 0x04], %t1; 		\
-	lduw			[%src + offset + 0x08], %t2; 		\
-	lduw			[%src + offset + 0x0c], %t3; 		\
-	stw			%t0, [%dst + offset + 0x00]; 		\
-	stw			%t1, [%dst + offset + 0x04]; 		\
-	stw			%t2, [%dst + offset + 0x08]; 		\
-	stw			%t3, [%dst + offset + 0x0c]; 		\
-	lduw			[%src + offset + 0x10], %t0; 		\
-	lduw			[%src + offset + 0x14], %t1; 		\
-	lduw			[%src + offset + 0x18], %t2; 		\
-	lduw			[%src + offset + 0x1c], %t3; 		\
-	stw			%t0, [%dst + offset + 0x10]; 		\
-	stw			%t1, [%dst + offset + 0x14]; 		\
-	stw			%t2, [%dst + offset + 0x18]; 		\
-	stw			%t3, [%dst + offset + 0x1c];
-
-#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3)		\
-	lduw			[%src - offset - 0x10], %t0; 		\
-	lduw			[%src - offset - 0x0c], %t1; 		\
-	lduw			[%src - offset - 0x08], %t2; 		\
-	lduw			[%src - offset - 0x04], %t3; 		\
-	stw			%t0, [%dst - offset - 0x10]; 		\
-	stw			%t1, [%dst - offset - 0x0c]; 		\
-	stw			%t2, [%dst - offset - 0x08]; 		\
-	stw			%t3, [%dst - offset - 0x04];
-
-#endif /* !REGS_64BIT */
-
-#ifdef __KERNEL__
-		.section	__ex_table,#alloc
-		.section	.fixup,#alloc,#execinstr
-#endif
-
-		.text
-		.align			32
-		.globl			memcpy
-		.type			memcpy,@function
-
-		.globl			bcopy
-		.type			bcopy,@function
-
-#ifdef __KERNEL__
-		.globl			__memcpy_begin
-__memcpy_begin:
-
-memcpy_private:
-memcpy:		mov		ASI_P, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		ASI_P, asi_dest			! IEU1
-		retl
-		 clr		%o0
-
-		.align			32
-		.globl			__copy_from_user
-		.type			__copy_from_user,@function
-__copy_from_user:rd		%asi, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		ASI_P, asi_dest			! IEU1
-
-		.globl			__copy_to_user
-		.type			__copy_to_user,@function
-__copy_to_user:	mov		ASI_P, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 rd		%asi, asi_dest			! IEU1
-		retl						! CTI	Group
-		 clr		%o0				! IEU0	Group
-
-		.globl			__copy_in_user
-		.type			__copy_in_user,@function
-__copy_in_user:	rd		%asi, asi_src			! IEU0	Group
-		brnz,pt		%o2, __memcpy_entry		! CTI
-		 mov		asi_src, asi_dest		! IEU1
-		retl						! CTI	Group
-		 clr		%o0				! IEU0	Group
-#endif
-
-bcopy:		or		%o0, 0, %g3			! IEU0	Group
-		addcc		%o1, 0, %o0			! IEU1
-		brgez,pt	%o2, memcpy_private		! CTI
-		 or		%g3, 0, %o1			! IEU0	Group
-		retl						! CTI	Group brk forced
-		 clr		%o0				! IEU0
-
-
-#ifdef __KERNEL__
-#define BRANCH_ALWAYS	0x10680000
-#define NOP		0x01000000
-#define ULTRA3_DO_PATCH(OLD, NEW)	\
-	sethi	%hi(NEW), %g1; \
-	or	%g1, %lo(NEW), %g1; \
-	sethi	%hi(OLD), %g2; \
-	or	%g2, %lo(OLD), %g2; \
-	sub	%g1, %g2, %g1; \
-	sethi	%hi(BRANCH_ALWAYS), %g3; \
-	srl	%g1, 2, %g1; \
-	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
-	or	%g3, %g1, %g3; \
-	stw	%g3, [%g2]; \
-	sethi	%hi(NOP), %g3; \
-	or	%g3, %lo(NOP), %g3; \
-	stw	%g3, [%g2 + 0x4]; \
-	flush	%g2;
-#define ULTRA3_PCACHE_DO_NOP(symbol)	\
-	sethi	%hi(symbol##_nop_1_6), %g1; \
-	or	%g1, %lo(symbol##_nop_1_6), %g1; \
-	sethi	%hi(NOP), %g2; \
-	stw	%g2, [%g1 + 0x00]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x00; \
-	stw	%g2, [%g1 + 0x08]; \
-	stw	%g2, [%g1 + 0x0c]; \
-	flush	%g1 + 0x08; \
-	stw	%g2, [%g1 + 0x10]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x10; \
-	sethi	%hi(symbol##_nop_2_3), %g1; \
-	or	%g1, %lo(symbol##_nop_2_3), %g1; \
-	stw	%g2, [%g1 + 0x00]; \
-	stw	%g2, [%g1 + 0x04]; \
-	flush	%g1 + 0x00; \
-	stw	%g2, [%g1 + 0x08]; \
-	flush	%g1 + 0x08;
-
-#include <asm/dcu.h>
-
-	.globl	cheetah_patch_copyops
-cheetah_patch_copyops:
-	ULTRA3_DO_PATCH(memcpy, U3memcpy)
-	ULTRA3_DO_PATCH(__copy_from_user, U3copy_from_user)
-	ULTRA3_DO_PATCH(__copy_to_user, U3copy_to_user)
-	ULTRA3_DO_PATCH(__copy_in_user, U3copy_in_user)
-#if 0 /* Causes data corruption, nop out the optimization
-       * for now -DaveM
-       */
-	ldxa			[%g0] ASI_DCU_CONTROL_REG, %g3
-	sethi			%uhi(DCU_PE), %o3
-	sllx			%o3, 32, %o3
-	andcc			%g3, %o3, %g0
-	be,pn			%xcc, pcache_disabled
-	 nop
-#endif
-	ULTRA3_PCACHE_DO_NOP(U3memcpy)
-	ULTRA3_PCACHE_DO_NOP(U3copy_from_user)
-	ULTRA3_PCACHE_DO_NOP(U3copy_to_user)
-	ULTRA3_PCACHE_DO_NOP(cheetah_copy_user_page)
-#if 0
-pcache_disabled:
-#endif
-	retl
-	 nop
-#undef BRANCH_ALWAYS
-#undef NOP
-#undef ULTRA3_DO_PATCH
-#endif /* __KERNEL__ */
-
-	.align			32
-#ifdef __KERNEL__
-	andcc			%o0, 7, %g2			! IEU1	Group
-#endif
-VIS_enter:
-	be,pt			%xcc, dest_is_8byte_aligned	! CTI
-#ifdef __KERNEL__
-	 nop							! IEU0	Group
-#else
-	 andcc			%o0, 0x38, %g5			! IEU1	Group
-#endif
-do_dest_8byte_align:
-	mov			8, %g1				! IEU0
-	sub			%g1, %g2, %g2			! IEU0	Group
-	andcc			%o0, 1, %g0			! IEU1
-	be,pt			%icc, 2f			! CTI
-	 sub			%o2, %g2, %o2			! IEU0	Group
-1:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %o5, 
-				add %o2, %g2)			! Load	Group
-	add			%o1, 1, %o1			! IEU0
-	add			%o0, 1, %o0			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 1, %g2			! IEU1	Group
-	be,pn			%xcc, 3f			! CTI
-	 EX2(STB		%o5, [%o0 - 1] ASINORMAL,
-				add %g2, 1, %g2,
-				add %o2, %g2)			! Store
-2:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %o5, 
-				add %o2, %g2)			! Load	Group
-	add			%o0, 2, %o0			! IEU0
-	EX2(LDUB		[%o1 + 1] ASINORMAL, %g3,
-				sub %o0, 2, %o0,
-				add %o2, %g2)			! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 2, %g2			! IEU1	Group
-	EX2(STB			%o5, [%o0 - 2] ASINORMAL,
-				add %g2, 2, %g2,
-				add %o2, %g2)			! Store
-	add			%o1, 2, %o1			! IEU0
-	bne,pt			%xcc, 2b			! CTI	Group
-	 EX2(STB		%g3, [%o0 - 1] ASINORMAL,
-				add %g2, 1, %g2,
-				add %o2, %g2)			! Store
-#ifdef __KERNEL__
-3:
-dest_is_8byte_aligned:
-	VISEntry
-	andcc			%o0, 0x38, %g5			! IEU1	Group
-#else
-3:	andcc			%o0, 0x38, %g5			! IEU1	Group
-dest_is_8byte_aligned:
-#endif
-	be,pt			%icc, dest_is_64byte_aligned	! CTI
-	 mov			64, %g1				! IEU0
-	fmovd			%f0, %f2			! FPU
-	sub			%g1, %g5, %g5			! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	alignaddr		%o1, %g0, %g1			! GRU	Group
-	EXO2(LDDF		[%g1] ASINORMAL, %f4)		! Load	Group
-	sub			%o2, %g5, %o2			! IEU0
-1:	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f6,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 0x8, %g1			! IEU0	Group
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f4, %f6, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %g5, 8, %g5,
-				add %o2, %g5)			! Store
-	add			%o1, 8, %o1			! IEU0	Group
-	be,pn			%xcc, dest_is_64byte_aligned	! CTI
-	 add			%o0, 8, %o0			! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f4,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 8, %g1			! IEU0
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f6, %f4, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %g5, 8, %g5,
-				add %o2, %g5)			! Store
-	add			%o1, 8, %o1			! IEU0
-	ASI_SETSRC_NOBLK					! LSU	Group
-	bne,pt			%xcc, 1b			! CTI	Group
-	 add			%o0, 8, %o0			! IEU0
-dest_is_64byte_aligned:
-	membar		  #LoadStore | #StoreStore | #StoreLoad	! LSU	Group
-#ifndef __KERNEL__
-	wr			%g0, ASI_BLK_P, %asi		! LSU	Group
-#endif
-	subcc			%o2, 0x40, %g7			! IEU1	Group
-	mov			%o1, %g1			! IEU0
-	andncc			%g7, (0x40 - 1), %g7		! IEU1	Group
-	srl			%g1, 3, %g2			! IEU0
-	sub			%o2, %g7, %g3			! IEU0	Group
-	andn			%o1, (0x40 - 1), %o1		! IEU1
-	and			%g2, 7, %g2			! IEU0	Group
-	andncc			%g3, 0x7, %g3			! IEU1
-	fmovd			%f0, %f2			! FPU
-	sub			%g3, 0x10, %g3			! IEU0	Group
-	sub			%o2, %g7, %o2			! IEU1
-#ifdef __KERNEL__
-	or			asi_src, ASI_BLK_OR, asi_src	! IEU0	Group
-	or			asi_dest, ASI_BLK_OR, asi_dest	! IEU1
-#endif
-	alignaddr		%g1, %g0, %g0			! GRU	Group
-	add			%g1, %g7, %g1			! IEU0	Group
-	subcc			%o2, %g3, %o2			! IEU1
-	ASI_SETSRC_BLK						! LSU	Group
-	EXVIS1(LDBLK		[%o1 + 0x00] ASIBLK, %f0)	! LSU	Group
-	add			%g1, %g3, %g1			! IEU0
-	EXVIS1(LDBLK		[%o1 + 0x40] ASIBLK, %f16)	! LSU	Group
-	sub			%g7, 0x80, %g7			! IEU0
-	EXVIS(LDBLK		[%o1 + 0x80] ASIBLK, %f32)	! LSU	Group
-#ifdef __KERNEL__
-vispc:	sll			%g2, 9, %g2			! IEU0	Group
-	sethi			%hi(vis00), %g5			! IEU1
-	or			%g5, %lo(vis00), %g5		! IEU0	Group
-	jmpl			%g5 + %g2, %g0			! CTI	Group brk forced
-	 addcc			%o1, 0xc0, %o1			! IEU1	Group
-#else
-								! Clk1	Group 8-(
-								! Clk2	Group 8-(
-								! Clk3	Group 8-(
-								! Clk4	Group 8-(
-vispc:	rd			%pc, %g5			! PDU	Group 8-(
-	addcc			%g5, %lo(vis00 - vispc), %g5	! IEU1	Group
-	sll			%g2, 9, %g2			! IEU0
-	jmpl			%g5 + %g2, %g0			! CTI	Group brk forced
-	 addcc			%o1, 0xc0, %o1			! IEU1	Group
-#endif
-	.align			512		/* OK, here comes the fun part... */
-vis00:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) LOOP_CHUNK1(o1, o0, g7, vis01)
-      FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) LOOP_CHUNK2(o1, o0, g7, vis02)
-      FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  LOOP_CHUNK3(o1, o0, g7, vis03)
-      b,pt			%xcc, vis00+4; faligndata %f0, %f2, %f48
-vis01:FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  STORE_JUMP(o0, f48, finish_f0) membar #Sync
-vis02:FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_JUMP(o0, f48, finish_f16) membar #Sync
-vis03:FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_JUMP(o0, f48, finish_f32) membar #Sync
-      VISLOOP_PAD
-vis10:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) LOOP_CHUNK1(o1, o0, g7, vis11)
-      FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) LOOP_CHUNK2(o1, o0, g7, vis12)
-      FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  LOOP_CHUNK3(o1, o0, g7, vis13)
-      b,pt			%xcc, vis10+4; faligndata %f2, %f4, %f48
-vis11:FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  STORE_JUMP(o0, f48, finish_f2) membar #Sync
-vis12:FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_JUMP(o0, f48, finish_f18) membar #Sync
-vis13:FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_JUMP(o0, f48, finish_f34) membar #Sync
-      VISLOOP_PAD
-vis20:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) LOOP_CHUNK1(o1, o0, g7, vis21)
-      FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) LOOP_CHUNK2(o1, o0, g7, vis22)
-      FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  LOOP_CHUNK3(o1, o0, g7, vis23)
-      b,pt			%xcc, vis20+4; faligndata %f4, %f6, %f48
-vis21:FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  STORE_JUMP(o0, f48, finish_f4) membar #Sync
-vis22:FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_JUMP(o0, f48, finish_f20) membar #Sync
-vis23:FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_JUMP(o0, f48, finish_f36) membar #Sync
-      VISLOOP_PAD
-vis30:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) LOOP_CHUNK1(o1, o0, g7, vis31)
-      FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) LOOP_CHUNK2(o1, o0, g7, vis32)
-      FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  LOOP_CHUNK3(o1, o0, g7, vis33)
-      b,pt			%xcc, vis30+4; faligndata %f6, %f8, %f48
-vis31:FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  STORE_JUMP(o0, f48, finish_f6) membar #Sync
-vis32:FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_JUMP(o0, f48, finish_f22) membar #Sync
-vis33:FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_JUMP(o0, f48, finish_f38) membar #Sync
-      VISLOOP_PAD
-vis40:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) LOOP_CHUNK1(o1, o0, g7, vis41)
-      FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) LOOP_CHUNK2(o1, o0, g7, vis42)
-      FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  LOOP_CHUNK3(o1, o0, g7, vis43)
-      b,pt			%xcc, vis40+4; faligndata %f8, %f10, %f48
-vis41:FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  STORE_JUMP(o0, f48, finish_f8) membar #Sync
-vis42:FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)  STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_JUMP(o0, f48, finish_f24) membar #Sync
-vis43:FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_JUMP(o0, f48, finish_f40) membar #Sync
-      VISLOOP_PAD
-vis50:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) LOOP_CHUNK1(o1, o0, g7, vis51)
-      FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) LOOP_CHUNK2(o1, o0, g7, vis52)
-      FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) LOOP_CHUNK3(o1, o0, g7, vis53)
-      b,pt			%xcc, vis50+4; faligndata %f10, %f12, %f48
-vis51:FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_JUMP(o0, f48, finish_f10) membar #Sync
-vis52:FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_JUMP(o0, f48, finish_f26) membar #Sync
-vis53:FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_JUMP(o0, f48, finish_f42) membar #Sync
-      VISLOOP_PAD
-vis60:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) LOOP_CHUNK1(o1, o0, g7, vis61)
-      FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) LOOP_CHUNK2(o1, o0, g7, vis62)
-      FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) LOOP_CHUNK3(o1, o0, g7, vis63)
-      b,pt			%xcc, vis60+4; faligndata %f12, %f14, %f48
-vis61:FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_JUMP(o0, f48, finish_f12) membar #Sync
-vis62:FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_JUMP(o0, f48, finish_f28) membar #Sync
-vis63:FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_JUMP(o0, f48, finish_f44) membar #Sync
-      VISLOOP_PAD
-vis70:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) LOOP_CHUNK1(o1, o0, g7, vis71)
-      FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) LOOP_CHUNK2(o1, o0, g7, vis72)
-      FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) LOOP_CHUNK3(o1, o0, g7, vis73)
-      b,pt			%xcc, vis70+4; faligndata %f14, %f16, %f48
-vis71:FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_JUMP(o0, f48, finish_f14) membar #Sync
-vis72:FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_JUMP(o0, f48, finish_f30) membar #Sync
-vis73:FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_SYNC(o0, f48) membar #Sync
-      FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, finish_f46) membar #Sync
-      VISLOOP_PAD
-finish_f0:	FINISH_VISCHUNK(o0, f0,  f2,  g3)
-finish_f2:	FINISH_VISCHUNK(o0, f2,  f4,  g3)
-finish_f4:	FINISH_VISCHUNK(o0, f4,  f6,  g3)
-finish_f6:	FINISH_VISCHUNK(o0, f6,  f8,  g3)
-finish_f8:	FINISH_VISCHUNK(o0, f8,  f10, g3)
-finish_f10:	FINISH_VISCHUNK(o0, f10, f12, g3)
-finish_f12:	FINISH_VISCHUNK(o0, f12, f14, g3)
-finish_f14:	UNEVEN_VISCHUNK(o0, f14, f0,  g3)
-finish_f16:	FINISH_VISCHUNK(o0, f16, f18, g3)
-finish_f18:	FINISH_VISCHUNK(o0, f18, f20, g3)
-finish_f20:	FINISH_VISCHUNK(o0, f20, f22, g3)
-finish_f22:	FINISH_VISCHUNK(o0, f22, f24, g3)
-finish_f24:	FINISH_VISCHUNK(o0, f24, f26, g3)
-finish_f26:	FINISH_VISCHUNK(o0, f26, f28, g3)
-finish_f28:	FINISH_VISCHUNK(o0, f28, f30, g3)
-finish_f30:	UNEVEN_VISCHUNK(o0, f30, f0,  g3)
-finish_f32:	FINISH_VISCHUNK(o0, f32, f34, g3)
-finish_f34:	FINISH_VISCHUNK(o0, f34, f36, g3)
-finish_f36:	FINISH_VISCHUNK(o0, f36, f38, g3)
-finish_f38:	FINISH_VISCHUNK(o0, f38, f40, g3)
-finish_f40:	FINISH_VISCHUNK(o0, f40, f42, g3)
-finish_f42:	FINISH_VISCHUNK(o0, f42, f44, g3)
-finish_f44:	FINISH_VISCHUNK(o0, f44, f46, g3)
-finish_f46:	UNEVEN_VISCHUNK_LAST(o0, f46, f0,  g3)
-vis_out_slk:
-#ifdef __KERNEL__
-	srl		asi_src, 3, %g5				! IEU0	Group
-	xor		asi_src, ASI_BLK_XOR1, asi_src		! IEU1
-	xor		asi_src, %g5, asi_src			! IEU0	Group
-#endif
-vis_slk:ASI_SETSRC_NOBLK					! LSU	Group
-	EXVIS3(LDDF	[%o1] ASINORMAL, %f2)			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	subcc		%g3, 8, %g3				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata	%f0, %f2, %f8				! GRU	Group
-	EXVIS4(STDF	%f8, [%o0] ASINORMAL)			! Store
-	bl,pn		%xcc, vis_out_slp			! CTI
-	 add		%o0, 8, %o0				! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXVIS3(LDDF	[%o1] ASINORMAL, %f0)			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	subcc		%g3, 8, %g3				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata	%f2, %f0, %f8				! GRU	Group
-	EXVIS4(STDF	%f8, [%o0] ASINORMAL)			! Store
-	bge,pt		%xcc, vis_slk				! CTI
-	 add		%o0, 8, %o0				! IEU0	Group
-vis_out_slp:
-#ifdef __KERNEL__
-	brz,pt		%o2, vis_ret				! CTI	Group
-	 mov		%g1, %o1				! IEU0
-	ba,pt		%xcc, vis_slp+4				! CTI	Group
-	 ASI_SETSRC_NOBLK					! LSU	Group
-#endif
-vis_out:brz,pt		%o2, vis_ret				! CTI	Group
-	 mov		%g1, %o1				! IEU0
-#ifdef __KERNEL__
-	srl		asi_src, 3, %g5				! IEU0	Group
-	xor		asi_src, ASI_BLK_XOR1, asi_src		! IEU1
-	xor		asi_src, %g5, asi_src			! IEU0	Group
-#endif
-vis_slp:ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1
-	bne,pt		%xcc, vis_slp				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store	Group
-vis_ret:membar		#StoreLoad | #StoreStore		! LSU	Group
-	FPU_CLEAN_RETL
-
-
-__memcpy_short:
-	andcc		%o2, 1, %g0				! IEU1	Group
-	be,pt		%icc, 2f				! CTI
-1:	 ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD	Group
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1	Group
-	be,pn		%xcc, short_ret				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-2:	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD	Group
-	add		%o0, 2, %o0				! IEU0
-	EX2(LDUB	[%o1 + 1] ASINORMAL, %o5,
-			sub %o0, 2, %o0,
-			add %o2, %g0)				! LOAD	Group
-	add		%o1, 2, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 2, %o2				! IEU1	Group
-	EX(STB		%g5, [%o0 - 2] ASINORMAL,
-			add %o2, 2)				! Store
-	bne,pt		%xcc, 2b				! CTI
-	 EX(STB		%o5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-short_ret:
-	NORMAL_RETL
-
-#ifndef __KERNEL__
-memcpy_private:
-memcpy:
-#ifndef REGS_64BIT
-	srl		%o2, 0, %o2				! IEU1	Group
-#endif	
-	brz,pn		%o2, short_ret				! CTI	Group
-	 mov		%o0, %g6				! IEU0
-#endif
-__memcpy_entry:
-	cmp		%o2, 15					! IEU1	Group
-	bleu,pn		%xcc, __memcpy_short			! CTI
-	 cmp		%o2, (64 * 6)				! IEU1	Group
-	bgeu,pn		%xcc, VIS_enter				! CTI
-	 andcc		%o0, 7, %g2				! IEU1	Group
-	sub		%o0, %o1, %g5				! IEU0
-	andcc		%g5, 3, %o5				! IEU1	Group
-	bne,pn		%xcc, memcpy_noVIS_misaligned		! CTI
-	 andcc		%o1, 3, %g0				! IEU1	Group
-#ifdef REGS_64BIT
-	be,a,pt		%xcc, 3f				! CTI
-	 andcc		%o1, 4, %g0				! IEU1	Group
-	andcc		%o1, 1, %g0				! IEU1	Group
-#else /* !REGS_64BIT */
-	be,pt		%xcc, 5f				! CTI
-	 andcc		%o1, 1, %g0				! IEU1	Group
-#endif /* !REGS_64BIT */
-	be,pn		%xcc, 4f				! CTI
-	 andcc		%o1, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUB	[%o1] ASINORMAL, %g2)			! Load	Group
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	sub		%o2, 1, %o2				! IEU0	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	bne,pn		%xcc, 5f				! CTI	Group
-	 EX(STB		%g2, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store
-4:	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUH	[%o1] ASINORMAL, %g2)			! Load	Group
-	add		%o1, 2, %o1				! IEU0
-	add		%o0, 2, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	sub		%o2, 2, %o2				! IEU0
-	EX(STH		%g2, [%o0 - 2] ASINORMAL,
-			add %o2, 2)				! Store	Group + bubble
-#ifdef REGS_64BIT
-5:	andcc		%o1, 4, %g0				! IEU1
-3:	be,a,pn		%xcc, 2f				! CTI
-	 andcc		%o2, -128, %g7				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EXO2(LDUW	[%o1] ASINORMAL, %g5)			! Load	Group
-	add		%o1, 4, %o1				! IEU0
-	add		%o0, 4, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	sub		%o2, 4, %o2				! IEU0	Group
-	EX(STW		%g5, [%o0 - 4] ASINORMAL,
-			add %o2, 4)				! Store
-	andcc		%o2, -128, %g7				! IEU1	Group
-2:	be,pn		%xcc, 3f				! CTI
-	 andcc		%o0, 4, %g0				! IEU1	Group
-	be,pn		%xcc, 82f + 4				! CTI	Group
-#else /* !REGS_64BIT */
-5:	andcc		%o2, -128, %g7				! IEU1
-	be,a,pn		%xcc, 41f				! CTI
-	 andcc		%o2, 0x70, %g7				! IEU1	Group
-#endif /* !REGS_64BIT */
-5:	MOVE_BIGCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x20, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
-	MOVE_BIGCHUNK(o1, o0, 0x60, g1, g3, g5, o5)
-	EXT(5b,35f,VIScopyfixup1)
-35:	subcc		%g7, 128, %g7				! IEU1	Group
-	add		%o1, 128, %o1				! IEU0
-	bne,pt		%xcc, 5b				! CTI
-	 add		%o0, 128, %o0				! IEU0	Group
-3:	andcc		%o2, 0x70, %g7				! IEU1	Group
-41:	be,pn		%xcc, 80f				! CTI
-	 andcc		%o2, 8, %g0				! IEU1	Group
-#ifdef __KERNEL__
-79:	sethi		%hi(80f), %o5				! IEU0
-	sll		%g7, 1, %g5				! IEU0	Group
-	add		%o1, %g7, %o1				! IEU1
-	srl		%g7, 1, %g2				! IEU0  Group
-	sub		%o5, %g5, %o5				! IEU1
-	sub		%o5, %g2, %o5				! IEU0  Group
-	jmpl		%o5 + %lo(80f), %g0			! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#else
-								! Clk1 8-(
-								! Clk2 8-(
-								! Clk3 8-(
-								! Clk4 8-(
-79:	rd		%pc, %o5				! PDU	Group
-	sll		%g7, 1, %g5				! IEU0	Group
-	add		%o1, %g7, %o1				! IEU1
-	sub		%o5, %g5, %o5				! IEU0  Group
-	jmpl		%o5 + %lo(80f - 79b), %g0		! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#endif
-36:	MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g5, o5)
-	MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g5, o5)
-	EXT(36b,80f,VIScopyfixup2)
-80:	be,pt		%xcc, 81f				! CTI
-	 andcc		%o2, 4, %g0				! IEU1
-#ifdef REGS_64BIT
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDX		[%o1] ASINORMAL, %g2,
-			and %o2, 0xf)				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STW		%g2, [%o0 - 0x4] ASINORMAL,
-			and %o2, 0xf)				! Store	Group
-	add		%o1, 8, %o1				! IEU1
-	srlx		%g2, 32, %g2				! IEU0	Group
-	EX2(STW		%g2, [%o0 - 0x8] ASINORMAL,
-			and %o2, 0xf, %o2,
-			sub %o2, 4)				! Store
-#else /* !REGS_64BIT */
-	lduw		[%o1], %g2				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	lduw		[%o1 + 0x4], %g3			! Load	Group
-	add		%o1, 8, %o1				! IEU0
-	stw		%g2, [%o0 - 0x8]			! Store	Group
-	stw		%g3, [%o0 - 0x4]			! Store	Group
-#endif /* !REGS_64BIT */
-81:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUW		[%o1] ASINORMAL, %g2,
-			and %o2, 0x7)				! Load	Group
-	add		%o1, 4, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STW		%g2, [%o0] ASINORMAL,
-			and %o2, 0x7)				! Store	Group
-	add		%o0, 4, %o0				! IEU0
-1:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 1, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUH		[%o1] ASINORMAL, %g2,
-			and %o2, 0x3)				! Load	Group
-	add		%o1, 2, %o1				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STH		%g2, [%o0] ASINORMAL,
-			and %o2, 0x3)				! Store	Group
-	add		%o0, 2, %o0				! IEU0
-1:	be,pt		%xcc, normal_retl			! CTI
-	 nop							! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB		[%o1] ASINORMAL, %g2,
-			add %g0, 1)				! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STB		%g2, [%o0] ASINORMAL,
-			add %g0, 1)				! Store	Group + bubble
-normal_retl:
-	NORMAL_RETL
-
-#ifdef REGS_64BIT
-82:	MOVE_BIGALIGNCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
-	MOVE_BIGALIGNCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
-	EXT(82b,37f,VIScopyfixup3)
-37:	subcc		%g7, 128, %g7				! IEU1	Group
-	add		%o1, 128, %o1				! IEU0
-	bne,pt		%xcc, 82b				! CTI
-	 add		%o0, 128, %o0				! IEU0	Group
-	andcc		%o2, 0x70, %g7				! IEU1
-	be,pn		%xcc, 84f				! CTI
-	 andcc		%o2, 8, %g0				! IEU1	Group
-#ifdef __KERNEL__
-83:	srl		%g7, 1, %g5				! IEU0
-	sethi		%hi(84f), %o5				! IEU0	Group
-	add		%g7, %g5, %g5				! IEU1
-	add		%o1, %g7, %o1				! IEU0	Group
-	sub		%o5, %g5, %o5				! IEU1
-	jmpl		%o5 + %lo(84f), %g0			! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#else
-								! Clk1 8-(
-								! Clk2 8-(
-								! Clk3 8-(
-								! Clk4 8-(
-83:	rd		%pc, %o5				! PDU	Group
-	add		%o1, %g7, %o1				! IEU0	Group
-	sub		%o5, %g7, %o5				! IEU1
-	jmpl		%o5 + %lo(84f - 83b), %g0		! CTI	Group brk forced
-	 add		%o0, %g7, %o0				! IEU0	Group
-#endif
-38:	MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3)
-	MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3)
-	EXT(38b,84f,VIScopyfixup4)
-84:	be,pt		%xcc, 85f				! CTI	Group
-	 andcc		%o2, 4, %g0				! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDX		[%o1] ASINORMAL, %g2,
-			and %o2, 0xf)				! Load	Group
-	add		%o0, 8, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 8, %o1				! IEU0	Group
-	EX(STX		%g2, [%o0 - 0x8] ASINORMAL,
-			and %o2, 0xf)				! Store
-85:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 2, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUW		[%o1] ASINORMAL, %g2,
-			and %o2, 0x7)				! Load	Group
-	add		%o0, 4, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 4, %o1				! IEU0	Group
-	EX(STW		%g2, [%o0 - 0x4] ASINORMAL,
-			and %o2, 0x7)				! Store
-1:	be,pt		%xcc, 1f				! CTI
-	 andcc		%o2, 1, %g0				! IEU1	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUH		[%o1] ASINORMAL, %g2,
-			and %o2, 0x3)				! Load	Group
-	add		%o0, 2, %o0				! IEU0
-	ASI_SETDST_NOBLK					! LSU	Group
-	add		%o1, 2, %o1				! IEU0	Group
-	EX(STH		%g2, [%o0 - 0x2] ASINORMAL,
-			and %o2, 0x3)				! Store
-1:	be,pt		%xcc, 1f				! CTI
-	 nop							! IEU0	Group
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB		[%o1] ASINORMAL, %g2,
-			add %g0, 1)				! Load	Group
-	ASI_SETDST_NOBLK					! LSU	Group
-	EX(STB		%g2, [%o0] ASINORMAL,
-			add %g0, 1)				! Store	Group + bubble
-1:	NORMAL_RETL
-#endif	/* REGS_64BIT */
-
-memcpy_noVIS_misaligned:
-	brz,pt			%g2, 2f				! CTI	Group
-	 mov			8, %g1				! IEU0
-	sub			%g1, %g2, %g2			! IEU0	Group
-	sub			%o2, %g2, %o2			! IEU0	Group
-1:	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDUB			[%o1] ASINORMAL, %g5,
-				add %o2, %g2)			! Load	Group
-	add			%o1, 1, %o1			! IEU0
-	add			%o0, 1, %o0			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc			%g2, 1, %g2			! IEU1	Group
-	bne,pt			%xcc, 1b			! CTI
-	 EX2(STB		%g5, [%o0 - 1] ASINORMAL,
-				add %o2, %g2, %o2,
-				add %o2, 1)			! Store
-2:
-#ifdef __KERNEL__
-	VISEntry
-#endif
-	andn			%o2, 7, %g5 			! IEU0	Group
-	and			%o2, 7, %o2			! IEU1
-	fmovd			%f0, %f2			! FPU
-	ASI_SETSRC_NOBLK					! LSU	Group
-	alignaddr		%o1, %g0, %g1			! GRU	Group
-	EXO2(LDDF		[%g1] ASINORMAL, %f4)		! Load	Group
-1:	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f6,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 0x8, %g1			! IEU0	Group
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f4, %f6, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %o2, %g5, %o2,
-				add %o2, 8)			! Store
-	add			%o1, 8, %o1			! IEU0	Group
-	be,pn			%xcc, end_cruft			! CTI
-	 add			%o0, 8, %o0			! IEU1
-	ASI_SETSRC_NOBLK					! LSU	Group
-	EX(LDDF			[%g1 + 0x8] ASINORMAL, %f4,
-				add %o2, %g5)			! Load	Group
-	add			%g1, 8, %g1			! IEU0
-	subcc			%g5, 8, %g5			! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	faligndata		%f6, %f4, %f0			! GRU	Group
-	EX2(STDF		%f0, [%o0] ASINORMAL,
-				add %o2, %g5, %o2,
-				add %o2, 8)			! Store
-	add			%o1, 8, %o1			! IEU0
-	ASI_SETSRC_NOBLK					! LSU	Group
-	bne,pn			%xcc, 1b			! CTI	Group
-	 add			%o0, 8, %o0			! IEU0
-end_cruft:
-	brz,pn			%o2, fpu_retl			! CTI	Group
-#ifndef __KERNEL__
-	 nop							! IEU0
-#else
-	 ASI_SETSRC_NOBLK					! LSU	Group
-#endif
-	EXO2(LDUB	[%o1] ASINORMAL, %g5)			! LOAD
-	add		%o1, 1, %o1				! IEU0
-	add		%o0, 1, %o0				! IEU1
-	ASI_SETDST_NOBLK					! LSU	Group
-	subcc		%o2, 1, %o2				! IEU1
-	bne,pt		%xcc, vis_slp				! CTI
-	 EX(STB		%g5, [%o0 - 1] ASINORMAL,
-			add %o2, 1)				! Store	Group
-fpu_retl:
-	FPU_RETL
-
-#ifdef __KERNEL__
-	.globl		__memcpy_end
-__memcpy_end:
-
-		.section	.fixup
-		.align		4
-VIScopyfixup_reto2:
-		mov		%o2, %o1
-VIScopyfixup_ret:
-		/* If this is copy_from_user(), zero out the rest of the
-		 * kernel buffer.
-		 */
-		ldub		[%g6 + AOFF_task_thread + AOFF_thread_current_ds], %o4
-		andcc		asi_src, 0x1, %g0
-		be,pt		%icc, 1f
-		 VISExit
-		andcc		asi_dest, 0x1, %g0
-		bne,pn		%icc, 1f
-		 nop
-		save		%sp, -160, %sp
-		mov		%i0, %o0
-		call		__bzero
-		 mov		%i1, %o1
-		restore
-1:		mov		%o1, %o0
-		retl
-		 wr		%o4, %g0, %asi
-VIScopyfixup1:	subcc		%g2, 18, %g2
-		add		%o0, 32, %o0
-		bgeu,a,pt	%icc, VIScopyfixup1
-		 sub		%g7, 32, %g7
-		sub		%o0, 32, %o0
-		rd		%pc, %g5
-		add		%g2, (18 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 2f
-.byte		0, 0, 0, 0, 0, 0, 0, 4, 4, 8, 12, 12, 16, 20, 20, 24, 28, 28
-		.align		4
-VIScopyfixup2:	mov		(7 * 16), %g7
-1:		subcc		%g2, 10, %g2
-		bgeu,a,pt	%icc, 1b
-		 sub		%g7, 16, %g7
-		sub		%o0, %g7, %o0
-		rd		%pc, %g5
-		add		%g2, (10 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 4f
-.byte		0, 0, 0, 0, 0, 4, 4, 8, 12, 12
-		.align		4
-VIScopyfixup3:	subcc		%g2, 10, %g2
-		add		%o0, 32, %o0
-		bgeu,a,pt	%icc, VIScopyfixup3
-		 sub		%g7, 32, %g7
-		sub		%o0, 32, %o0
-		rd		%pc, %g5
-		add		%g2, (10 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 2f
-.byte		0, 0, 0, 0, 0, 0, 0, 8, 16, 24
-		.align		4
-2:		and		%o2, 0x7f, %o2
-		sub		%g7, %g2, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%g7, %o2, %o1
-VIScopyfixup4:	mov		(7 * 16), %g7
-3:		subcc		%g2, 6, %g2
-		bgeu,a,pt	%icc, 3b
-		 sub		%g7, 16, %g7
-		sub		%o0, %g7, %o0
-		rd		%pc, %g5
-		add		%g2, (6 + 16), %g2
-		ldub		[%g5 + %g2], %g2
-		ba,a,pt		%xcc, 4f
-.byte		0, 0, 0, 0, 0, 8
-		.align		4
-4:		and		%o2, 0xf, %o2
-		sub		%g7, %g2, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%g7, %o2, %o1
-VIScopyfixup_vis2:
-		sub		%o2, 0x40, %o2
-VIScopyfixup_vis0:
-		add		%o2, 0x80, %o2
-VIScopyfixup_vis1:
-		add		%g7, %g3, %g7
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%o2, %g7, %o1
-VIScopyfixup_vis4:
-		add		%g3, 8, %g3
-VIScopyfixup_vis3:
-		add		%g3, 8, %g3
-		ba,pt		%xcc, VIScopyfixup_ret
-		 add		%o2, %g3, %o1
-#endif
-
-#ifdef __KERNEL__
-		.text
-		.align		32
-
-		.globl		__memmove
-		.type		__memmove,@function
-
-		.globl		memmove
-		.type		memmove,@function
-
-memmove:
-__memmove:	cmp		%o0, %o1
-		blu,pt		%xcc, memcpy_private
-		 sub		%o0, %o1, %g5
-		add		%o1, %o2, %g3
-		cmp		%g3, %o0
-		bleu,pt		%xcc, memcpy_private
-		 add		%o1, %o2, %g5
-		add		%o0, %o2, %o5
-
-		sub		%g5, 1, %o1
-		sub		%o5, 1, %o0
-1:		ldub		[%o1], %g5
-		subcc		%o2, 1, %o2
-		sub		%o1, 1, %o1
-		stb		%g5, [%o0]
-		bne,pt		%icc, 1b
-		 sub		%o0, 1, %o0
-
-		retl
-		 clr		%o0
-#endif
diff -Nru a/arch/sparc64/lib/copy_in_user.S b/arch/sparc64/lib/copy_in_user.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/copy_in_user.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,114 @@
+/* copy_in_user.S: Copy from userspace to userspace.
+ *
+ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+ */
+
+#include <asm/asi.h>
+
+#define XCC xcc
+
+#define EX(x,y)			\
+98:	x,y;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	retl;			\
+	 mov 1, %o0;		\
+	.section __ex_table;	\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+	.register	%g2,#scratch
+	.register	%g3,#scratch
+
+	.text
+	.align	32
+
+	/* Don't try to get too fancy here, just nice and
+	 * simple.  This is predominantly used for well aligned
+	 * small copies in the compat layer.  It is also used
+	 * to copy register windows around during thread cloning.
+	 */
+
+	.globl	___copy_in_user
+___copy_in_user:	/* %o0=dst, %o1=src, %o2=len */
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+	rd		%asi, %g1
+	cmp		%g1, ASI_AIUS
+	bne,pn		%icc, memcpy_user_stub
+	 nop
+
+	cmp		%o2, 0
+	be,pn		%XCC, 85f
+	 or		%o0, %o1, %o3
+	cmp		%o2, 16
+	bleu,a,pn	%XCC, 80f
+	 or		%o3, %o2, %o3
+
+	/* 16 < len <= 64 */
+	andcc		%o3, 0x7, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+	andn		%o2, 0x7, %o4
+	and		%o2, 0x7, %o2
+1:	subcc		%o4, 0x8, %o4
+	EX(ldxa [%o1] %asi, %o5)
+	EX(stxa %o5, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 1b
+	 add		%o1, 0x8, %o1
+	andcc		%o2, 0x4, %g0
+	be,pt		%XCC, 1f
+	 nop
+	sub		%o2, 0x4, %o2
+	EX(lduwa [%o1] %asi, %o5)
+	EX(stwa %o5, [%o1 + %o3] ASI_AIUS)
+	add		%o1, 0x4, %o1
+1:	cmp		%o2, 0
+	be,pt		%XCC, 85f
+	 nop
+	ba,pt		%xcc, 90f
+	 nop
+
+80:	/* 0 < len <= 16 */
+	andcc		%o3, 0x3, %g0
+	bne,pn		%XCC, 90f
+	 sub		%o0, %o1, %o3
+
+82:
+	subcc		%o2, 4, %o2
+	EX(lduwa [%o1] %asi, %g1)
+	EX(stwa %g1, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 82b
+	 add		%o1, 4, %o1
+
+85:	retl
+	 clr		%o0
+
+	.align	32
+90:
+	subcc		%o2, 1, %o2
+	EX(lduba [%o1] %asi, %g1)
+	EX(stba %g1, [%o1 + %o3] ASI_AIUS)
+	bgu,pt		%XCC, 90b
+	 add		%o1, 1, %o1
+	retl
+	 clr		%o0
+
+	/* Act like copy_{to,in}_user(), ie. return zero instead
+	 * of original destination pointer.  This is invoked when
+	 * copy_{to,in}_user() finds that %asi is kernel space.
+	 */
+	.globl		memcpy_user_stub
+memcpy_user_stub:
+	save		%sp, -192, %sp
+	mov		%i0, %o0
+	mov		%i1, %o1
+	call		memcpy
+	 mov		%i2, %o2
+	ret
+	 restore	%g0, %g0, %o0
diff -Nru a/arch/sparc64/lib/memmove.S b/arch/sparc64/lib/memmove.S
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/memmove.S	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,31 @@
+/* memmove.S: Simple memmove implementation.
+ *
+ * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+
+	.text
+	.align	32
+	.globl	memmove
+memmove:
+	mov		%o0, %g1
+	cmp		%o0, %o1
+	blu,pt		%xcc, memcpy
+	 sub		%o0, %o1, %g5
+	add		%o1, %o2, %g3
+	cmp		%g3, %o0
+	bleu,pt		%xcc, memcpy
+	 add		%o1, %o2, %g5
+	add		%o0, %o2, %o5
+
+	sub		%g5, 1, %o1
+	sub		%o5, 1, %o0
+1:	ldub		[%o1], %g5
+	subcc		%o2, 1, %o2
+	sub		%o1, 1, %o1
+	stb		%g5, [%o0]
+	bne,pt		%icc, 1b
+	 sub		%o0, 1, %o0
+
+	retl
+	 mov		%g1, %o0
diff -Nru a/arch/sparc64/lib/user_fixup.c b/arch/sparc64/lib/user_fixup.c
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/arch/sparc64/lib/user_fixup.c	2004-08-26 14:19:04 -07:00
@@ -0,0 +1,68 @@
+/* user_fixup.c: Fix up user copy faults.
+ *
+ * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+ */
+
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <asm/uaccess.h>
+
+/* Calculating the exact fault address when using
+ * block loads and stores can be very complicated.
+ * Instead of trying to be clever and handling all
+ * of the cases, just fix things up simply here.
+ */
+
+unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
+{
+	char *dst = to;
+	const char __user *src = from;
+
+	while (size--) {
+		if (__get_user(*dst, src))
+			break;
+		dst++;
+		src++;
+	}
+
+	if (size)
+		memset(dst, 0, size);
+
+	return size;
+}
+
+unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
+{
+	char __user *dst = to;
+	const char *src = from;
+
+	while (size--) {
+		if (__put_user(*src, dst))
+			break;
+		dst++;
+		src++;
+	}
+
+	return size;
+}
+
+unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
+{
+	char __user *dst = to;
+	char __user *src = from;
+
+	while (size--) {
+		char tmp;
+
+		if (__get_user(tmp, src))
+			break;
+		if (__put_user(tmp, dst))
+			break;
+		dst++;
+		src++;
+	}
+
+	return size;
+}
diff -Nru a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
--- a/arch/sparc64/mm/init.c	2004-04-28 13:35:32 -07:00
+++ b/arch/sparc64/mm/init.c	2004-09-02 14:47:07 -07:00
@@ -63,6 +63,7 @@
 unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
 
 /* Initial ramdisk setup */
+extern unsigned long sparc_ramdisk_image64;
 extern unsigned int sparc_ramdisk_image;
 extern unsigned int sparc_ramdisk_size;
 
@@ -1251,10 +1252,12 @@
 
 #ifdef CONFIG_BLK_DEV_INITRD
 	/* Now have to check initial ramdisk, so that bootmap does not overwrite it */
-	if (sparc_ramdisk_image) {
-		if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
-			sparc_ramdisk_image -= KERNBASE;
-		initrd_start = sparc_ramdisk_image + phys_base;
+	if (sparc_ramdisk_image || sparc_ramdisk_image64) {
+		unsigned long ramdisk_image = sparc_ramdisk_image ?
+			sparc_ramdisk_image : sparc_ramdisk_image64;
+		if (ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
+			ramdisk_image -= KERNBASE;
+		initrd_start = ramdisk_image + phys_base;
 		initrd_end = initrd_start + sparc_ramdisk_size;
 		if (initrd_end > end_of_phys_memory) {
 			printk(KERN_CRIT "initrd extends beyond end of memory "
@@ -1300,6 +1303,10 @@
 			    initrd_start, size);
 #endif
 		/* Resert the initrd image area. */
+#ifdef CONFIG_DEBUG_BOOTMEM
+		prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
+			initrd_start, initrd_end);
+#endif
 		reserve_bootmem(initrd_start, size);
 		*pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
 
@@ -1353,7 +1360,7 @@
 	if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
 		bigkernel = 1;
 #ifdef CONFIG_BLK_DEV_INITRD
-	if (sparc_ramdisk_image)
+	if (sparc_ramdisk_image || sparc_ramdisk_image64)
 		real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size));
 #endif
 
diff -Nru a/crypto/Config.in b/crypto/Config.in
--- a/crypto/Config.in	2004-08-10 18:27:14 -07:00
+++ b/crypto/Config.in	2004-08-24 11:28:59 -07:00
@@ -58,6 +58,7 @@
   fi
   tristate       '  SHA256 digest algorithm' CONFIG_CRYPTO_SHA256
   tristate       '  SHA384 and SHA512 digest algorithms' CONFIG_CRYPTO_SHA512
+  tristate       '  Whirlpool digest algorithm' CONFIG_CRYPTO_WHIRLPOOL
   if [ "$CONFIG_INET_ESP" = "y" -o \
        "$CONFIG_INET_ESP" = "m" -o \
        "$CONFIG_INET6_ESP" = "y" -o \
diff -Nru a/crypto/Makefile b/crypto/Makefile
--- a/crypto/Makefile	2004-08-10 18:27:14 -07:00
+++ b/crypto/Makefile	2004-08-24 11:28:59 -07:00
@@ -19,6 +19,7 @@
 obj-$(CONFIG_CRYPTO_SHA1) += sha1.o
 obj-$(CONFIG_CRYPTO_SHA256) += sha256.o
 obj-$(CONFIG_CRYPTO_SHA512) += sha512.o
+obj-$(CONFIG_CRYPTO_WHIRLPOOL) += whirlpool.o
 obj-$(CONFIG_CRYPTO_DES) += des.o
 obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish.o
 obj-$(CONFIG_CRYPTO_TWOFISH) += twofish.o
diff -Nru a/crypto/tcrypt.c b/crypto/tcrypt.c
--- a/crypto/tcrypt.c	2004-08-10 18:27:14 -07:00
+++ b/crypto/tcrypt.c	2004-08-24 11:28:59 -07:00
@@ -63,7 +63,7 @@
 static char *check[] = {
 	"des", "md5", "des3_ede", "rot13", "sha1", "sha256", "blowfish",
 	"twofish", "serpent", "sha384", "sha512", "md4", "aes", "cast6", 
-	"arc4", "michael_mic", "deflate", "tea", "xtea", NULL
+	"arc4", "michael_mic", "deflate", "tea", "xtea", "whirlpool", NULL
 };
 
 static void
@@ -581,6 +581,7 @@
 
 		test_hash("sha384", sha384_tv_template, SHA384_TEST_VECTORS);
 		test_hash("sha512", sha512_tv_template, SHA512_TEST_VECTORS);
+		test_hash("whirlpool", whirlpool_tv_template, WHIRLPOOL_TEST_VECTORS);
 		test_deflate();		
 #ifdef CONFIG_CRYPTO_HMAC
 		test_hmac("md5", hmac_md5_tv_template, HMAC_MD5_TEST_VECTORS);
@@ -685,6 +686,9 @@
 	case 21:
 		test_cipher ("khazad", MODE_ECB, ENCRYPT, khazad_enc_tv_template, KHAZAD_ENC_TEST_VECTORS);
 		test_cipher ("khazad", MODE_ECB, DECRYPT, khazad_dec_tv_template, KHAZAD_DEC_TEST_VECTORS);
+		break;
+	case 22:
+		test_hash("whirlpool", whirlpool_tv_template, WHIRLPOOL_TEST_VECTORS);
 		break;
 
 #ifdef CONFIG_CRYPTO_HMAC
diff -Nru a/crypto/tcrypt.h b/crypto/tcrypt.h
--- a/crypto/tcrypt.h	2004-08-10 18:33:06 -07:00
+++ b/crypto/tcrypt.h	2004-08-24 11:29:00 -07:00
@@ -551,6 +551,109 @@
 #endif	/* CONFIG_CRYPTO_HMAC */
 
 /*
+ * WHIRLPOOL test vectors from Whirlpool package 
+ * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE
+ * submission
+ */
+#define WHIRLPOOL_TEST_VECTORS	8
+
+struct hash_testvec whirlpool_tv_template[] = {
+	{ 
+		.plaintext = "",
+		.psize	= 0,
+		.digest	= { 0x19, 0xFA, 0x61, 0xD7, 0x55, 0x22, 0xA4, 0x66,
+			    0x9B, 0x44, 0xE3, 0x9C, 0x1D, 0x2E, 0x17, 0x26,
+			    0xC5, 0x30, 0x23, 0x21, 0x30, 0xD4, 0x07, 0xF8,
+			    0x9A, 0xFE, 0xE0, 0x96, 0x49, 0x97, 0xF7, 0xA7,
+			    0x3E, 0x83, 0xBE, 0x69, 0x8B, 0x28, 0x8F, 0xEB,
+			    0xCF, 0x88, 0xE3, 0xE0, 0x3C, 0x4F, 0x07, 0x57,
+			    0xEA, 0x89, 0x64, 0xE5, 0x9B, 0x63, 0xD9, 0x37,
+			    0x08, 0xB1, 0x38, 0xCC, 0x42, 0xA6, 0x6E, 0xB3 },
+
+
+	}, {
+		.plaintext = "a",
+		.psize	= 1,
+		.digest	= { 0x8A, 0xCA, 0x26, 0x02, 0x79, 0x2A, 0xEC, 0x6F,
+			    0x11, 0xA6, 0x72, 0x06, 0x53, 0x1F, 0xB7, 0xD7,
+			    0xF0, 0xDF, 0xF5, 0x94, 0x13, 0x14, 0x5E, 0x69,
+			    0x73, 0xC4, 0x50, 0x01, 0xD0, 0x08, 0x7B, 0x42,
+			    0xD1, 0x1B, 0xC6, 0x45, 0x41, 0x3A, 0xEF, 0xF6,
+			    0x3A, 0x42, 0x39, 0x1A, 0x39, 0x14, 0x5A, 0x59,
+			    0x1A, 0x92, 0x20, 0x0D, 0x56, 0x01, 0x95, 0xE5,
+			    0x3B, 0x47, 0x85, 0x84, 0xFD, 0xAE, 0x23, 0x1A },
+	}, {
+		.plaintext = "abc",
+		.psize	= 3,
+		.digest	= { 0x4E, 0x24, 0x48, 0xA4, 0xC6, 0xF4, 0x86, 0xBB,
+			    0x16, 0xB6, 0x56, 0x2C, 0x73, 0xB4, 0x02, 0x0B,
+			    0xF3, 0x04, 0x3E, 0x3A, 0x73, 0x1B, 0xCE, 0x72,
+			    0x1A, 0xE1, 0xB3, 0x03, 0xD9, 0x7E, 0x6D, 0x4C,
+			    0x71, 0x81, 0xEE, 0xBD, 0xB6, 0xC5, 0x7E, 0x27,
+			    0x7D, 0x0E, 0x34, 0x95, 0x71, 0x14, 0xCB, 0xD6,
+			    0xC7, 0x97, 0xFC, 0x9D, 0x95, 0xD8, 0xB5, 0x82,
+			    0xD2, 0x25, 0x29, 0x20, 0x76, 0xD4, 0xEE, 0xF5 },
+	}, {
+		.plaintext = "message digest",
+		.psize	= 14,
+		.digest	= { 0x37, 0x8C, 0x84, 0xA4, 0x12, 0x6E, 0x2D, 0xC6, 
+			    0xE5, 0x6D, 0xCC, 0x74, 0x58, 0x37, 0x7A, 0xAC, 
+			    0x83, 0x8D, 0x00, 0x03, 0x22, 0x30, 0xF5, 0x3C, 
+			    0xE1, 0xF5, 0x70, 0x0C, 0x0F, 0xFB, 0x4D, 0x3B, 
+			    0x84, 0x21, 0x55, 0x76, 0x59, 0xEF, 0x55, 0xC1, 
+			    0x06, 0xB4, 0xB5, 0x2A, 0xC5, 0xA4, 0xAA, 0xA6, 
+			    0x92, 0xED, 0x92, 0x00, 0x52, 0x83, 0x8F, 0x33, 
+			    0x62, 0xE8, 0x6D, 0xBD, 0x37, 0xA8, 0x90, 0x3E },
+	}, {
+		.plaintext = "abcdefghijklmnopqrstuvwxyz",
+		.psize	= 26,
+		.digest	= { 0xF1, 0xD7, 0x54, 0x66, 0x26, 0x36, 0xFF, 0xE9,
+			    0x2C, 0x82, 0xEB, 0xB9, 0x21, 0x2A, 0x48, 0x4A,
+			    0x8D, 0x38, 0x63, 0x1E, 0xAD, 0x42, 0x38, 0xF5,
+			    0x44, 0x2E, 0xE1, 0x3B, 0x80, 0x54, 0xE4, 0x1B,
+			    0x08, 0xBF, 0x2A, 0x92, 0x51, 0xC3, 0x0B, 0x6A,
+			    0x0B, 0x8A, 0xAE, 0x86, 0x17, 0x7A, 0xB4, 0xA6,
+			    0xF6, 0x8F, 0x67, 0x3E, 0x72, 0x07, 0x86, 0x5D,
+			    0x5D, 0x98, 0x19, 0xA3, 0xDB, 0xA4, 0xEB, 0x3B },
+	}, {
+		.plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+			     "abcdefghijklmnopqrstuvwxyz0123456789",
+		.psize	= 62,
+		.digest	= { 0xDC, 0x37, 0xE0, 0x08, 0xCF, 0x9E, 0xE6, 0x9B,
+			    0xF1, 0x1F, 0x00, 0xED, 0x9A, 0xBA, 0x26, 0x90,
+			    0x1D, 0xD7, 0xC2, 0x8C, 0xDE, 0xC0, 0x66, 0xCC,
+			    0x6A, 0xF4, 0x2E, 0x40, 0xF8, 0x2F, 0x3A, 0x1E,
+			    0x08, 0xEB, 0xA2, 0x66, 0x29, 0x12, 0x9D, 0x8F,
+			    0xB7, 0xCB, 0x57, 0x21, 0x1B, 0x92, 0x81, 0xA6,
+			    0x55, 0x17, 0xCC, 0x87, 0x9D, 0x7B, 0x96, 0x21,
+			    0x42, 0xC6, 0x5F, 0x5A, 0x7A, 0xF0, 0x14, 0x67 },
+	}, {
+		.plaintext = "1234567890123456789012345678901234567890"
+			     "1234567890123456789012345678901234567890",
+		.psize	= 80,
+		.digest	= { 0x46, 0x6E, 0xF1, 0x8B, 0xAB, 0xB0, 0x15, 0x4D,
+			    0x25, 0xB9, 0xD3, 0x8A, 0x64, 0x14, 0xF5, 0xC0,
+			    0x87, 0x84, 0x37, 0x2B, 0xCC, 0xB2, 0x04, 0xD6,
+			    0x54, 0x9C, 0x4A, 0xFA, 0xDB, 0x60, 0x14, 0x29,
+			    0x4D, 0x5B, 0xD8, 0xDF, 0x2A, 0x6C, 0x44, 0xE5,
+			    0x38, 0xCD, 0x04, 0x7B, 0x26, 0x81, 0xA5, 0x1A,
+			    0x2C, 0x60, 0x48, 0x1E, 0x88, 0xC5, 0xA2, 0x0B,
+			    0x2C, 0x2A, 0x80, 0xCF, 0x3A, 0x9A, 0x08, 0x3B },
+	}, {
+		.plaintext = "abcdbcdecdefdefgefghfghighijhijk",
+		.psize	= 32,
+		.digest	= { 0x2A, 0x98, 0x7E, 0xA4, 0x0F, 0x91, 0x70, 0x61, 
+			    0xF5, 0xD6, 0xF0, 0xA0, 0xE4, 0x64, 0x4F, 0x48,
+			    0x8A, 0x7A, 0x5A, 0x52, 0xDE, 0xEE, 0x65, 0x62,
+			    0x07, 0xC5, 0x62, 0xF9, 0x88, 0xE9, 0x5C, 0x69,
+			    0x16, 0xBD, 0xC8, 0x03, 0x1B, 0xC5, 0xBE, 0x1B,
+			    0x7B, 0x94, 0x76, 0x39, 0xFE, 0x05, 0x0B, 0x56,
+			    0x93, 0x9B, 0xAA, 0xA0, 0xAD, 0xFF, 0x9A, 0xE6,
+			    0x74, 0x5B, 0x7B, 0x18, 0x1C, 0x3B, 0xE3, 0xFD },
+	},
+};
+
+/*
  * DES test vectors.
  */
 #define DES_ENC_TEST_VECTORS		10
diff -Nru a/crypto/whirlpool.c b/crypto/whirlpool.c
--- /dev/null	Wed Dec 31 16:00:00 196900
+++ b/crypto/whirlpool.c	2004-08-24 11:28:55 -07:00
@@ -0,0 +1,1131 @@
+/*
+ * Cryptographic API.
+ *
+ * Whirlpool hashing Algorithm
+ *
+ * The Whirlpool algorithm was developed by Paulo S. L. M. Barreto and
+ * Vincent Rijmen.  It has been selected as one of cryptographic
+ * primitives by the NESSIE project http://www.cryptonessie.org/
+ *
+ * The original authors have disclaimed all copyright interest in this
+ * code and thus put it in the public domain. The subsequent authors
+ * have put this under the GNU General Public License.
+ *
+ * By Aaron Grothe ajgrothe@yahoo.com, August 23, 2004
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <asm/scatterlist.h>
+#include <linux/crypto.h>
+
+#define WHIRLPOOL_DIGEST_SIZE 64
+#define WHIRLPOOL_BLOCK_SIZE  64
+#define WHIRLPOOL_LENGTHBYTES 32
+
+#define WHIRLPOOL_ROUNDS 10
+
+struct whirlpool_ctx {
+	u8  bitLength[WHIRLPOOL_LENGTHBYTES];
+	u8  buffer[WHIRLPOOL_BLOCK_SIZE];
+	int bufferBits;
+	int bufferPos;
+	u64 hash[WHIRLPOOL_DIGEST_SIZE/8];
+};
+
+/*
+ * Though Whirlpool is endianness-neutral, the encryption tables are listed
+ * in BIG-ENDIAN format, which is adopted throughout this implementation
+ * (but little-endian notation would be equally suitable if consistently
+ * employed).
+ */
+
+static const u64 C0[256] = {
+	0x18186018c07830d8ULL, 0x23238c2305af4626ULL, 0xc6c63fc67ef991b8ULL,
+	0xe8e887e8136fcdfbULL, 0x878726874ca113cbULL, 0xb8b8dab8a9626d11ULL,
+	0x0101040108050209ULL, 0x4f4f214f426e9e0dULL, 0x3636d836adee6c9bULL,
+	0xa6a6a2a6590451ffULL, 0xd2d26fd2debdb90cULL, 0xf5f5f3f5fb06f70eULL,
+	0x7979f979ef80f296ULL, 0x6f6fa16f5fcede30ULL, 0x91917e91fcef3f6dULL,
+	0x52525552aa07a4f8ULL, 0x60609d6027fdc047ULL, 0xbcbccabc89766535ULL,
+	0x9b9b569baccd2b37ULL, 0x8e8e028e048c018aULL, 0xa3a3b6a371155bd2ULL,
+	0x0c0c300c603c186cULL, 0x7b7bf17bff8af684ULL, 0x3535d435b5e16a80ULL,
+	0x1d1d741de8693af5ULL, 0xe0e0a7e05347ddb3ULL, 0xd7d77bd7f6acb321ULL,
+	0xc2c22fc25eed999cULL, 0x2e2eb82e6d965c43ULL, 0x4b4b314b627a9629ULL,
+	0xfefedffea321e15dULL, 0x575741578216aed5ULL, 0x15155415a8412abdULL,
+	0x7777c1779fb6eee8ULL, 0x3737dc37a5eb6e92ULL, 0xe5e5b3e57b56d79eULL,
+	0x9f9f469f8cd92313ULL, 0xf0f0e7f0d317fd23ULL, 0x4a4a354a6a7f9420ULL,
+	0xdada4fda9e95a944ULL, 0x58587d58fa25b0a2ULL, 0xc9c903c906ca8fcfULL,
+	0x2929a429558d527cULL, 0x0a0a280a5022145aULL, 0xb1b1feb1e14f7f50ULL,
+	0xa0a0baa0691a5dc9ULL, 0x6b6bb16b7fdad614ULL, 0x85852e855cab17d9ULL,
+	0xbdbdcebd8173673cULL, 0x5d5d695dd234ba8fULL, 0x1010401080502090ULL,
+	0xf4f4f7f4f303f507ULL, 0xcbcb0bcb16c08bddULL, 0x3e3ef83eedc67cd3ULL,
+	0x0505140528110a2dULL, 0x676781671fe6ce78ULL, 0xe4e4b7e47353d597ULL,
+	0x27279c2725bb4e02ULL, 0x4141194132588273ULL, 0x8b8b168b2c9d0ba7ULL,
+	0xa7a7a6a7510153f6ULL, 0x7d7de97dcf94fab2ULL, 0x95956e95dcfb3749ULL,
+	0xd8d847d88e9fad56ULL, 0xfbfbcbfb8b30eb70ULL, 0xeeee9fee2371c1cdULL,
+	0x7c7ced7cc791f8bbULL, 0x6666856617e3cc71ULL, 0xdddd53dda68ea77bULL,
+	0x17175c17b84b2eafULL, 0x4747014702468e45ULL, 0x9e9e429e84dc211aULL,
+	0xcaca0fca1ec589d4ULL, 0x2d2db42d75995a58ULL, 0xbfbfc6bf9179632eULL,
+	0x07071c07381b0e3fULL, 0xadad8ead012347acULL, 0x5a5a755aea2fb4b0ULL,
+	0x838336836cb51befULL, 0x3333cc3385ff66b6ULL, 0x636391633ff2c65cULL,
+	0x02020802100a0412ULL, 0xaaaa92aa39384993ULL, 0x7171d971afa8e2deULL,
+	0xc8c807c80ecf8dc6ULL, 0x19196419c87d32d1ULL, 0x494939497270923bULL,
+	0xd9d943d9869aaf5fULL, 0xf2f2eff2c31df931ULL, 0xe3e3abe34b48dba8ULL,
+	0x5b5b715be22ab6b9ULL, 0x88881a8834920dbcULL, 0x9a9a529aa4c8293eULL,
+	0x262698262dbe4c0bULL, 0x3232c8328dfa64bfULL, 0xb0b0fab0e94a7d59ULL,
+	0xe9e983e91b6acff2ULL, 0x0f0f3c0f78331e77ULL, 0xd5d573d5e6a6b733ULL,
+	0x80803a8074ba1df4ULL, 0xbebec2be997c6127ULL, 0xcdcd13cd26de87ebULL,
+	0x3434d034bde46889ULL, 0x48483d487a759032ULL, 0xffffdbffab24e354ULL,
+	0x7a7af57af78ff48dULL, 0x90907a90f4ea3d64ULL, 0x5f5f615fc23ebe9dULL,
+	0x202080201da0403dULL, 0x6868bd6867d5d00fULL, 0x1a1a681ad07234caULL,
+	0xaeae82ae192c41b7ULL, 0xb4b4eab4c95e757dULL, 0x54544d549a19a8ceULL,
+	0x93937693ece53b7fULL, 0x222288220daa442fULL, 0x64648d6407e9c863ULL,
+	0xf1f1e3f1db12ff2aULL, 0x7373d173bfa2e6ccULL, 0x12124812905a2482ULL,
+	0x40401d403a5d807aULL, 0x0808200840281048ULL, 0xc3c32bc356e89b95ULL,
+	0xecec97ec337bc5dfULL, 0xdbdb4bdb9690ab4dULL, 0xa1a1bea1611f5fc0ULL,
+	0x8d8d0e8d1c830791ULL, 0x3d3df43df5c97ac8ULL, 0x97976697ccf1335bULL,
+	0x0000000000000000ULL, 0xcfcf1bcf36d483f9ULL, 0x2b2bac2b4587566eULL,
+	0x7676c57697b3ece1ULL, 0x8282328264b019e6ULL, 0xd6d67fd6fea9b128ULL,
+	0x1b1b6c1bd87736c3ULL, 0xb5b5eeb5c15b7774ULL, 0xafaf86af112943beULL,
+	0x6a6ab56a77dfd41dULL, 0x50505d50ba0da0eaULL, 0x45450945124c8a57ULL,
+	0xf3f3ebf3cb18fb38ULL, 0x3030c0309df060adULL, 0xefef9bef2b74c3c4ULL,
+	0x3f3ffc3fe5c37edaULL, 0x55554955921caac7ULL, 0xa2a2b2a2791059dbULL,
+	0xeaea8fea0365c9e9ULL, 0x656589650fecca6aULL, 0xbabad2bab9686903ULL,
+	0x2f2fbc2f65935e4aULL, 0xc0c027c04ee79d8eULL, 0xdede5fdebe81a160ULL,
+	0x1c1c701ce06c38fcULL, 0xfdfdd3fdbb2ee746ULL, 0x4d4d294d52649a1fULL,
+	0x92927292e4e03976ULL, 0x7575c9758fbceafaULL, 0x06061806301e0c36ULL,
+	0x8a8a128a249809aeULL, 0xb2b2f2b2f940794bULL, 0xe6e6bfe66359d185ULL,
+	0x0e0e380e70361c7eULL, 0x1f1f7c1ff8633ee7ULL, 0x6262956237f7c455ULL,
+	0xd4d477d4eea3b53aULL, 0xa8a89aa829324d81ULL, 0x96966296c4f43152ULL,
+	0xf9f9c3f99b3aef62ULL, 0xc5c533c566f697a3ULL, 0x2525942535b14a10ULL,
+	0x59597959f220b2abULL, 0x84842a8454ae15d0ULL, 0x7272d572b7a7e4c5ULL,
+	0x3939e439d5dd72ecULL, 0x4c4c2d4c5a619816ULL, 0x5e5e655eca3bbc94ULL,
+	0x7878fd78e785f09fULL, 0x3838e038ddd870e5ULL, 0x8c8c0a8c14860598ULL,
+	0xd1d163d1c6b2bf17ULL, 0xa5a5aea5410b57e4ULL, 0xe2e2afe2434dd9a1ULL,
+	0x616199612ff8c24eULL, 0xb3b3f6b3f1457b42ULL, 0x2121842115a54234ULL,
+	0x9c9c4a9c94d62508ULL, 0x1e1e781ef0663ceeULL, 0x4343114322528661ULL,
+	0xc7c73bc776fc93b1ULL, 0xfcfcd7fcb32be54fULL, 0x0404100420140824ULL,
+	0x51515951b208a2e3ULL, 0x99995e99bcc72f25ULL, 0x6d6da96d4fc4da22ULL,
+	0x0d0d340d68391a65ULL, 0xfafacffa8335e979ULL, 0xdfdf5bdfb684a369ULL,
+	0x7e7ee57ed79bfca9ULL, 0x242490243db44819ULL, 0x3b3bec3bc5d776feULL,
+	0xabab96ab313d4b9aULL, 0xcece1fce3ed181f0ULL, 0x1111441188552299ULL,
+	0x8f8f068f0c890383ULL, 0x4e4e254e4a6b9c04ULL, 0xb7b7e6b7d1517366ULL,
+	0xebeb8beb0b60cbe0ULL, 0x3c3cf03cfdcc78c1ULL, 0x81813e817cbf1ffdULL,
+	0x94946a94d4fe3540ULL, 0xf7f7fbf7eb0cf31cULL, 0xb9b9deb9a1676f18ULL,
+	0x13134c13985f268bULL, 0x2c2cb02c7d9c5851ULL, 0xd3d36bd3d6b8bb05ULL,
+	0xe7e7bbe76b5cd38cULL, 0x6e6ea56e57cbdc39ULL, 0xc4c437c46ef395aaULL,
+	0x03030c03180f061bULL, 0x565645568a13acdcULL, 0x44440d441a49885eULL,
+	0x7f7fe17fdf9efea0ULL, 0xa9a99ea921374f88ULL, 0x2a2aa82a4d825467ULL,
+	0xbbbbd6bbb16d6b0aULL, 0xc1c123c146e29f87ULL, 0x53535153a202a6f1ULL,
+	0xdcdc57dcae8ba572ULL, 0x0b0b2c0b58271653ULL, 0x9d9d4e9d9cd32701ULL,
+	0x6c6cad6c47c1d82bULL, 0x3131c43195f562a4ULL, 0x7474cd7487b9e8f3ULL,
+	0xf6f6fff6e309f115ULL, 0x464605460a438c4cULL, 0xacac8aac092645a5ULL,
+	0x89891e893c970fb5ULL, 0x14145014a04428b4ULL, 0xe1e1a3e15b42dfbaULL,
+	0x16165816b04e2ca6ULL, 0x3a3ae83acdd274f7ULL, 0x6969b9696fd0d206ULL,
+	0x09092409482d1241ULL, 0x7070dd70a7ade0d7ULL, 0xb6b6e2b6d954716fULL,
+	0xd0d067d0ceb7bd1eULL, 0xeded93ed3b7ec7d6ULL, 0xcccc17cc2edb85e2ULL,
+	0x424215422a578468ULL, 0x98985a98b4c22d2cULL, 0xa4a4aaa4490e55edULL,
+	0x2828a0285d885075ULL, 0x5c5c6d5cda31b886ULL, 0xf8f8c7f8933fed6bULL,
+	0x8686228644a411c2ULL,
+};
+
+static const u64 C1[256] = {
+	0xd818186018c07830ULL, 0x2623238c2305af46ULL, 0xb8c6c63fc67ef991ULL,
+	0xfbe8e887e8136fcdULL, 0xcb878726874ca113ULL, 0x11b8b8dab8a9626dULL,
+	0x0901010401080502ULL, 0x0d4f4f214f426e9eULL, 0x9b3636d836adee6cULL,
+	0xffa6a6a2a6590451ULL, 0x0cd2d26fd2debdb9ULL, 0x0ef5f5f3f5fb06f7ULL,
+	0x967979f979ef80f2ULL, 0x306f6fa16f5fcedeULL, 0x6d91917e91fcef3fULL,
+	0xf852525552aa07a4ULL, 0x4760609d6027fdc0ULL, 0x35bcbccabc897665ULL,
+	0x379b9b569baccd2bULL, 0x8a8e8e028e048c01ULL, 0xd2a3a3b6a371155bULL,
+	0x6c0c0c300c603c18ULL, 0x847b7bf17bff8af6ULL, 0x803535d435b5e16aULL,
+	0xf51d1d741de8693aULL, 0xb3e0e0a7e05347ddULL, 0x21d7d77bd7f6acb3ULL,
+	0x9cc2c22fc25eed99ULL, 0x432e2eb82e6d965cULL, 0x294b4b314b627a96ULL,
+	0x5dfefedffea321e1ULL, 0xd5575741578216aeULL, 0xbd15155415a8412aULL,
+	0xe87777c1779fb6eeULL, 0x923737dc37a5eb6eULL, 0x9ee5e5b3e57b56d7ULL,
+	0x139f9f469f8cd923ULL, 0x23f0f0e7f0d317fdULL, 0x204a4a354a6a7f94ULL,
+	0x44dada4fda9e95a9ULL, 0xa258587d58fa25b0ULL, 0xcfc9c903c906ca8fULL,
+	0x7c2929a429558d52ULL, 0x5a0a0a280a502214ULL, 0x50b1b1feb1e14f7fULL,
+	0xc9a0a0baa0691a5dULL, 0x146b6bb16b7fdad6ULL, 0xd985852e855cab17ULL,
+	0x3cbdbdcebd817367ULL, 0x8f5d5d695dd234baULL, 0x9010104010805020ULL,
+	0x07f4f4f7f4f303f5ULL, 0xddcbcb0bcb16c08bULL, 0xd33e3ef83eedc67cULL,
+	0x2d0505140528110aULL, 0x78676781671fe6ceULL, 0x97e4e4b7e47353d5ULL,
+	0x0227279c2725bb4eULL, 0x7341411941325882ULL, 0xa78b8b168b2c9d0bULL,
+	0xf6a7a7a6a7510153ULL, 0xb27d7de97dcf94faULL, 0x4995956e95dcfb37ULL,
+	0x56d8d847d88e9fadULL, 0x70fbfbcbfb8b30ebULL, 0xcdeeee9fee2371c1ULL,
+	0xbb7c7ced7cc791f8ULL, 0x716666856617e3ccULL, 0x7bdddd53dda68ea7ULL,
+	0xaf17175c17b84b2eULL, 0x454747014702468eULL, 0x1a9e9e429e84dc21ULL,
+	0xd4caca0fca1ec589ULL, 0x582d2db42d75995aULL, 0x2ebfbfc6bf917963ULL,
+	0x3f07071c07381b0eULL, 0xacadad8ead012347ULL, 0xb05a5a755aea2fb4ULL,
+	0xef838336836cb51bULL, 0xb63333cc3385ff66ULL, 0x5c636391633ff2c6ULL,
+	0x1202020802100a04ULL, 0x93aaaa92aa393849ULL, 0xde7171d971afa8e2ULL,
+	0xc6c8c807c80ecf8dULL, 0xd119196419c87d32ULL, 0x3b49493949727092ULL,
+	0x5fd9d943d9869aafULL, 0x31f2f2eff2c31df9ULL, 0xa8e3e3abe34b48dbULL,
+	0xb95b5b715be22ab6ULL, 0xbc88881a8834920dULL, 0x3e9a9a529aa4c829ULL,
+	0x0b262698262dbe4cULL, 0xbf3232c8328dfa64ULL, 0x59b0b0fab0e94a7dULL,
+	0xf2e9e983e91b6acfULL, 0x770f0f3c0f78331eULL, 0x33d5d573d5e6a6b7ULL,
+	0xf480803a8074ba1dULL, 0x27bebec2be997c61ULL, 0xebcdcd13cd26de87ULL,
+	0x893434d034bde468ULL, 0x3248483d487a7590ULL, 0x54ffffdbffab24e3ULL,
+	0x8d7a7af57af78ff4ULL, 0x6490907a90f4ea3dULL, 0x9d5f5f615fc23ebeULL,
+	0x3d202080201da040ULL, 0x0f6868bd6867d5d0ULL, 0xca1a1a681ad07234ULL,
+	0xb7aeae82ae192c41ULL, 0x7db4b4eab4c95e75ULL, 0xce54544d549a19a8ULL,
+	0x7f93937693ece53bULL, 0x2f222288220daa44ULL, 0x6364648d6407e9c8ULL,
+	0x2af1f1e3f1db12ffULL, 0xcc7373d173bfa2e6ULL, 0x8212124812905a24ULL,
+	0x7a40401d403a5d80ULL, 0x4808082008402810ULL, 0x95c3c32bc356e89bULL,
+	0xdfecec97ec337bc5ULL, 0x4ddbdb4bdb9690abULL, 0xc0a1a1bea1611f5fULL,
+	0x918d8d0e8d1c8307ULL, 0xc83d3df43df5c97aULL, 0x5b97976697ccf133ULL,
+	0x0000000000000000ULL, 0xf9cfcf1bcf36d483ULL, 0x6e2b2bac2b458756ULL,
+	0xe17676c57697b3ecULL, 0xe68282328264b019ULL, 0x28d6d67fd6fea9b1ULL,
+	0xc31b1b6c1bd87736ULL, 0x74b5b5eeb5c15b77ULL, 0xbeafaf86af112943ULL,
+	0x1d6a6ab56a77dfd4ULL, 0xea50505d50ba0da0ULL, 0x5745450945124c8aULL,
+	0x38f3f3ebf3cb18fbULL, 0xad3030c0309df060ULL, 0xc4efef9bef2b74c3ULL,
+	0xda3f3ffc3fe5c37eULL, 0xc755554955921caaULL, 0xdba2a2b2a2791059ULL,
+	0xe9eaea8fea0365c9ULL, 0x6a656589650feccaULL, 0x03babad2bab96869ULL,
+	0x4a2f2fbc2f65935eULL, 0x8ec0c027c04ee79dULL, 0x60dede5fdebe81a1ULL,
+	0xfc1c1c701ce06c38ULL, 0x46fdfdd3fdbb2ee7ULL, 0x1f4d4d294d52649aULL,
+	0x7692927292e4e039ULL, 0xfa7575c9758fbceaULL, 0x3606061806301e0cULL,
+	0xae8a8a128a249809ULL, 0x4bb2b2f2b2f94079ULL, 0x85e6e6bfe66359d1ULL,
+	0x7e0e0e380e70361cULL, 0xe71f1f7c1ff8633eULL, 0x556262956237f7c4ULL,
+	0x3ad4d477d4eea3b5ULL, 0x81a8a89aa829324dULL, 0x5296966296c4f431ULL,
+	0x62f9f9c3f99b3aefULL, 0xa3c5c533c566f697ULL, 0x102525942535b14aULL,
+	0xab59597959f220b2ULL, 0xd084842a8454ae15ULL, 0xc57272d572b7a7e4ULL,
+	0xec3939e439d5dd72ULL, 0x164c4c2d4c5a6198ULL, 0x945e5e655eca3bbcULL,
+	0x9f7878fd78e785f0ULL, 0xe53838e038ddd870ULL, 0x988c8c0a8c148605ULL,
+	0x17d1d163d1c6b2bfULL, 0xe4a5a5aea5410b57ULL, 0xa1e2e2afe2434dd9ULL,
+	0x4e616199612ff8c2ULL, 0x42b3b3f6b3f1457bULL, 0x342121842115a542ULL,
+	0x089c9c4a9c94d625ULL, 0xee1e1e781ef0663cULL, 0x6143431143225286ULL,
+	0xb1c7c73bc776fc93ULL, 0x4ffcfcd7fcb32be5ULL, 0x2404041004201408ULL,
+	0xe351515951b208a2ULL, 0x2599995e99bcc72fULL, 0x226d6da96d4fc4daULL,
+	0x650d0d340d68391aULL, 0x79fafacffa8335e9ULL, 0x69dfdf5bdfb684a3ULL,
+	0xa97e7ee57ed79bfcULL, 0x19242490243db448ULL, 0xfe3b3bec3bc5d776ULL,
+	0x9aabab96ab313d4bULL, 0xf0cece1fce3ed181ULL, 0x9911114411885522ULL,
+	0x838f8f068f0c8903ULL, 0x044e4e254e4a6b9cULL, 0x66b7b7e6b7d15173ULL,
+	0xe0ebeb8beb0b60cbULL, 0xc13c3cf03cfdcc78ULL, 0xfd81813e817cbf1fULL,
+	0x4094946a94d4fe35ULL, 0x1cf7f7fbf7eb0cf3ULL, 0x18b9b9deb9a1676fULL,
+	0x8b13134c13985f26ULL, 0x512c2cb02c7d9c58ULL, 0x05d3d36bd3d6b8bbULL,
+	0x8ce7e7bbe76b5cd3ULL, 0x396e6ea56e57cbdcULL, 0xaac4c437c46ef395ULL,
+	0x1b03030c03180f06ULL, 0xdc565645568a13acULL, 0x5e44440d441a4988ULL,
+	0xa07f7fe17fdf9efeULL, 0x88a9a99ea921374fULL, 0x672a2aa82a4d8254ULL,
+	0x0abbbbd6bbb16d6bULL, 0x87c1c123c146e29fULL, 0xf153535153a202a6ULL,
+	0x72dcdc57dcae8ba5ULL, 0x530b0b2c0b582716ULL, 0x019d9d4e9d9cd327ULL,
+	0x2b6c6cad6c47c1d8ULL, 0xa43131c43195f562ULL, 0xf37474cd7487b9e8ULL,
+	0x15f6f6fff6e309f1ULL, 0x4c464605460a438cULL, 0xa5acac8aac092645ULL,
+	0xb589891e893c970fULL, 0xb414145014a04428ULL, 0xbae1e1a3e15b42dfULL,
+	0xa616165816b04e2cULL, 0xf73a3ae83acdd274ULL, 0x066969b9696fd0d2ULL,
+	0x4109092409482d12ULL, 0xd77070dd70a7ade0ULL, 0x6fb6b6e2b6d95471ULL,
+	0x1ed0d067d0ceb7bdULL, 0xd6eded93ed3b7ec7ULL, 0xe2cccc17cc2edb85ULL,
+	0x68424215422a5784ULL, 0x2c98985a98b4c22dULL, 0xeda4a4aaa4490e55ULL,
+	0x752828a0285d8850ULL, 0x865c5c6d5cda31b8ULL, 0x6bf8f8c7f8933fedULL,
+	0xc28686228644a411ULL,
+};
+
+static const u64 C2[256] = {
+	0x30d818186018c078ULL, 0x462623238c2305afULL, 0x91b8c6c63fc67ef9ULL,
+	0xcdfbe8e887e8136fULL, 0x13cb878726874ca1ULL, 0x6d11b8b8dab8a962ULL,
+	0x0209010104010805ULL, 0x9e0d4f4f214f426eULL, 0x6c9b3636d836adeeULL,
+	0x51ffa6a6a2a65904ULL, 0xb90cd2d26fd2debdULL, 0xf70ef5f5f3f5fb06ULL,
+	0xf2967979f979ef80ULL, 0xde306f6fa16f5fceULL, 0x3f6d91917e91fcefULL,
+	0xa4f852525552aa07ULL, 0xc04760609d6027fdULL, 0x6535bcbccabc8976ULL,
+	0x2b379b9b569baccdULL, 0x018a8e8e028e048cULL, 0x5bd2a3a3b6a37115ULL,
+	0x186c0c0c300c603cULL, 0xf6847b7bf17bff8aULL, 0x6a803535d435b5e1ULL,
+	0x3af51d1d741de869ULL, 0xddb3e0e0a7e05347ULL, 0xb321d7d77bd7f6acULL,
+	0x999cc2c22fc25eedULL, 0x5c432e2eb82e6d96ULL, 0x96294b4b314b627aULL,
+	0xe15dfefedffea321ULL, 0xaed5575741578216ULL, 0x2abd15155415a841ULL,
+	0xeee87777c1779fb6ULL, 0x6e923737dc37a5ebULL, 0xd79ee5e5b3e57b56ULL,
+	0x23139f9f469f8cd9ULL, 0xfd23f0f0e7f0d317ULL, 0x94204a4a354a6a7fULL,
+	0xa944dada4fda9e95ULL, 0xb0a258587d58fa25ULL, 0x8fcfc9c903c906caULL,
+	0x527c2929a429558dULL, 0x145a0a0a280a5022ULL, 0x7f50b1b1feb1e14fULL,
+	0x5dc9a0a0baa0691aULL, 0xd6146b6bb16b7fdaULL, 0x17d985852e855cabULL,
+	0x673cbdbdcebd8173ULL, 0xba8f5d5d695dd234ULL, 0x2090101040108050ULL,
+	0xf507f4f4f7f4f303ULL, 0x8bddcbcb0bcb16c0ULL, 0x7cd33e3ef83eedc6ULL,
+	0x0a2d050514052811ULL, 0xce78676781671fe6ULL, 0xd597e4e4b7e47353ULL,
+	0x4e0227279c2725bbULL, 0x8273414119413258ULL, 0x0ba78b8b168b2c9dULL,
+	0x53f6a7a7a6a75101ULL, 0xfab27d7de97dcf94ULL, 0x374995956e95dcfbULL,
+	0xad56d8d847d88e9fULL, 0xeb70fbfbcbfb8b30ULL, 0xc1cdeeee9fee2371ULL,
+	0xf8bb7c7ced7cc791ULL, 0xcc716666856617e3ULL, 0xa77bdddd53dda68eULL,
+	0x2eaf17175c17b84bULL, 0x8e45474701470246ULL, 0x211a9e9e429e84dcULL,
+	0x89d4caca0fca1ec5ULL, 0x5a582d2db42d7599ULL, 0x632ebfbfc6bf9179ULL,
+	0x0e3f07071c07381bULL, 0x47acadad8ead0123ULL, 0xb4b05a5a755aea2fULL,
+	0x1bef838336836cb5ULL, 0x66b63333cc3385ffULL, 0xc65c636391633ff2ULL,
+	0x041202020802100aULL, 0x4993aaaa92aa3938ULL, 0xe2de7171d971afa8ULL,
+	0x8dc6c8c807c80ecfULL, 0x32d119196419c87dULL, 0x923b494939497270ULL,
+	0xaf5fd9d943d9869aULL, 0xf931f2f2eff2c31dULL, 0xdba8e3e3abe34b48ULL,
+	0xb6b95b5b715be22aULL, 0x0dbc88881a883492ULL, 0x293e9a9a529aa4c8ULL,
+	0x4c0b262698262dbeULL, 0x64bf3232c8328dfaULL, 0x7d59b0b0fab0e94aULL,
+	0xcff2e9e983e91b6aULL, 0x1e770f0f3c0f7833ULL, 0xb733d5d573d5e6a6ULL,
+	0x1df480803a8074baULL, 0x6127bebec2be997cULL, 0x87ebcdcd13cd26deULL,
+	0x68893434d034bde4ULL, 0x903248483d487a75ULL, 0xe354ffffdbffab24ULL,
+	0xf48d7a7af57af78fULL, 0x3d6490907a90f4eaULL, 0xbe9d5f5f615fc23eULL,
+	0x403d202080201da0ULL, 0xd00f6868bd6867d5ULL, 0x34ca1a1a681ad072ULL,
+	0x41b7aeae82ae192cULL, 0x757db4b4eab4c95eULL, 0xa8ce54544d549a19ULL,
+	0x3b7f93937693ece5ULL, 0x442f222288220daaULL, 0xc86364648d6407e9ULL,
+	0xff2af1f1e3f1db12ULL, 0xe6cc7373d173bfa2ULL, 0x248212124812905aULL,
+	0x807a40401d403a5dULL, 0x1048080820084028ULL, 0x9b95c3c32bc356e8ULL,
+	0xc5dfecec97ec337bULL, 0xab4ddbdb4bdb9690ULL, 0x5fc0a1a1bea1611fULL,
+	0x07918d8d0e8d1c83ULL, 0x7ac83d3df43df5c9ULL, 0x335b97976697ccf1ULL,
+	0x0000000000000000ULL, 0x83f9cfcf1bcf36d4ULL, 0x566e2b2bac2b4587ULL,
+	0xece17676c57697b3ULL, 0x19e68282328264b0ULL, 0xb128d6d67fd6fea9ULL,
+	0x36c31b1b6c1bd877ULL, 0x7774b5b5eeb5c15bULL, 0x43beafaf86af1129ULL,
+	0xd41d6a6ab56a77dfULL, 0xa0ea50505d50ba0dULL, 0x8a5745450945124cULL,
+	0xfb38f3f3ebf3cb18ULL, 0x60ad3030c0309df0ULL, 0xc3c4efef9bef2b74ULL,
+	0x7eda3f3ffc3fe5c3ULL, 0xaac755554955921cULL, 0x59dba2a2b2a27910ULL,
+	0xc9e9eaea8fea0365ULL, 0xca6a656589650fecULL, 0x6903babad2bab968ULL,
+	0x5e4a2f2fbc2f6593ULL, 0x9d8ec0c027c04ee7ULL, 0xa160dede5fdebe81ULL,
+	0x38fc1c1c701ce06cULL, 0xe746fdfdd3fdbb2eULL, 0x9a1f4d4d294d5264ULL,
+	0x397692927292e4e0ULL, 0xeafa7575c9758fbcULL, 0x0c3606061806301eULL,
+	0x09ae8a8a128a2498ULL, 0x794bb2b2f2b2f940ULL, 0xd185e6e6bfe66359ULL,
+	0x1c7e0e0e380e7036ULL, 0x3ee71f1f7c1ff863ULL, 0xc4556262956237f7ULL,
+	0xb53ad4d477d4eea3ULL, 0x4d81a8a89aa82932ULL, 0x315296966296c4f4ULL,
+	0xef62f9f9c3f99b3aULL, 0x97a3c5c533c566f6ULL, 0x4a102525942535b1ULL,
+	0xb2ab59597959f220ULL, 0x15d084842a8454aeULL, 0xe4c57272d572b7a7ULL,
+	0x72ec3939e439d5ddULL, 0x98164c4c2d4c5a61ULL, 0xbc945e5e655eca3bULL,
+	0xf09f7878fd78e785ULL, 0x70e53838e038ddd8ULL, 0x05988c8c0a8c1486ULL,
+	0xbf17d1d163d1c6b2ULL, 0x57e4a5a5aea5410bULL, 0xd9a1e2e2afe2434dULL,
+	0xc24e616199612ff8ULL, 0x7b42b3b3f6b3f145ULL, 0x42342121842115a5ULL,
+	0x25089c9c4a9c94d6ULL, 0x3cee1e1e781ef066ULL, 0x8661434311432252ULL,
+	0x93b1c7c73bc776fcULL, 0xe54ffcfcd7fcb32bULL, 0x0824040410042014ULL,
+	0xa2e351515951b208ULL, 0x2f2599995e99bcc7ULL, 0xda226d6da96d4fc4ULL,
+	0x1a650d0d340d6839ULL, 0xe979fafacffa8335ULL, 0xa369dfdf5bdfb684ULL,
+	0xfca97e7ee57ed79bULL, 0x4819242490243db4ULL, 0x76fe3b3bec3bc5d7ULL,
+	0x4b9aabab96ab313dULL, 0x81f0cece1fce3ed1ULL, 0x2299111144118855ULL,
+	0x03838f8f068f0c89ULL, 0x9c044e4e254e4a6bULL, 0x7366b7b7e6b7d151ULL,
+	0xcbe0ebeb8beb0b60ULL, 0x78c13c3cf03cfdccULL, 0x1ffd81813e817cbfULL,
+	0x354094946a94d4feULL, 0xf31cf7f7fbf7eb0cULL, 0x6f18b9b9deb9a167ULL,
+	0x268b13134c13985fULL, 0x58512c2cb02c7d9cULL, 0xbb05d3d36bd3d6b8ULL,
+	0xd38ce7e7bbe76b5cULL, 0xdc396e6ea56e57cbULL, 0x95aac4c437c46ef3ULL,
+	0x061b03030c03180fULL, 0xacdc565645568a13ULL, 0x885e44440d441a49ULL,
+	0xfea07f7fe17fdf9eULL, 0x4f88a9a99ea92137ULL, 0x54672a2aa82a4d82ULL,
+	0x6b0abbbbd6bbb16dULL, 0x9f87c1c123c146e2ULL, 0xa6f153535153a202ULL,
+	0xa572dcdc57dcae8bULL, 0x16530b0b2c0b5827ULL, 0x27019d9d4e9d9cd3ULL,
+	0xd82b6c6cad6c47c1ULL, 0x62a43131c43195f5ULL, 0xe8f37474cd7487b9ULL,
+	0xf115f6f6fff6e309ULL, 0x8c4c464605460a43ULL, 0x45a5acac8aac0926ULL,
+	0x0fb589891e893c97ULL, 0x28b414145014a044ULL, 0xdfbae1e1a3e15b42ULL,
+	0x2ca616165816b04eULL, 0x74f73a3ae83acdd2ULL, 0xd2066969b9696fd0ULL,
+	0x124109092409482dULL, 0xe0d77070dd70a7adULL, 0x716fb6b6e2b6d954ULL,
+	0xbd1ed0d067d0ceb7ULL, 0xc7d6eded93ed3b7eULL, 0x85e2cccc17cc2edbULL,
+	0x8468424215422a57ULL, 0x2d2c98985a98b4c2ULL, 0x55eda4a4aaa4490eULL,
+	0x50752828a0285d88ULL, 0xb8865c5c6d5cda31ULL, 0xed6bf8f8c7f8933fULL,
+	0x11c28686228644a4ULL,
+};
+
+static const u64 C3[256] = {
+	0x7830d818186018c0ULL, 0xaf462623238c2305ULL, 0xf991b8c6c63fc67eULL,
+	0x6fcdfbe8e887e813ULL, 0xa113cb878726874cULL, 0x626d11b8b8dab8a9ULL,
+	0x0502090101040108ULL, 0x6e9e0d4f4f214f42ULL, 0xee6c9b3636d836adULL,
+	0x0451ffa6a6a2a659ULL, 0xbdb90cd2d26fd2deULL, 0x06f70ef5f5f3f5fbULL,
+	0x80f2967979f979efULL, 0xcede306f6fa16f5fULL, 0xef3f6d91917e91fcULL,
+	0x07a4f852525552aaULL, 0xfdc04760609d6027ULL, 0x766535bcbccabc89ULL,
+	0xcd2b379b9b569bacULL, 0x8c018a8e8e028e04ULL, 0x155bd2a3a3b6a371ULL,
+	0x3c186c0c0c300c60ULL, 0x8af6847b7bf17bffULL, 0xe16a803535d435b5ULL,
+	0x693af51d1d741de8ULL, 0x47ddb3e0e0a7e053ULL, 0xacb321d7d77bd7f6ULL,
+	0xed999cc2c22fc25eULL, 0x965c432e2eb82e6dULL, 0x7a96294b4b314b62ULL,
+	0x21e15dfefedffea3ULL, 0x16aed55757415782ULL, 0x412abd15155415a8ULL,
+	0xb6eee87777c1779fULL, 0xeb6e923737dc37a5ULL, 0x56d79ee5e5b3e57bULL,
+	0xd923139f9f469f8cULL, 0x17fd23f0f0e7f0d3ULL, 0x7f94204a4a354a6aULL,
+	0x95a944dada4fda9eULL, 0x25b0a258587d58faULL, 0xca8fcfc9c903c906ULL,
+	0x8d527c2929a42955ULL, 0x22145a0a0a280a50ULL, 0x4f7f50b1b1feb1e1ULL,
+	0x1a5dc9a0a0baa069ULL, 0xdad6146b6bb16b7fULL, 0xab17d985852e855cULL,
+	0x73673cbdbdcebd81ULL, 0x34ba8f5d5d695dd2ULL, 0x5020901010401080ULL,
+	0x03f507f4f4f7f4f3ULL, 0xc08bddcbcb0bcb16ULL, 0xc67cd33e3ef83eedULL,
+	0x110a2d0505140528ULL, 0xe6ce78676781671fULL, 0x53d597e4e4b7e473ULL,
+	0xbb4e0227279c2725ULL, 0x5882734141194132ULL, 0x9d0ba78b8b168b2cULL,
+	0x0153f6a7a7a6a751ULL, 0x94fab27d7de97dcfULL, 0xfb374995956e95dcULL,
+	0x9fad56d8d847d88eULL, 0x30eb70fbfbcbfb8bULL, 0x71c1cdeeee9fee23ULL,
+	0x91f8bb7c7ced7cc7ULL, 0xe3cc716666856617ULL, 0x8ea77bdddd53dda6ULL,
+	0x4b2eaf17175c17b8ULL, 0x468e454747014702ULL, 0xdc211a9e9e429e84ULL,
+	0xc589d4caca0fca1eULL, 0x995a582d2db42d75ULL, 0x79632ebfbfc6bf91ULL,
+	0x1b0e3f07071c0738ULL, 0x2347acadad8ead01ULL, 0x2fb4b05a5a755aeaULL,
+	0xb51bef838336836cULL, 0xff66b63333cc3385ULL, 0xf2c65c636391633fULL,
+	0x0a04120202080210ULL, 0x384993aaaa92aa39ULL, 0xa8e2de7171d971afULL,
+	0xcf8dc6c8c807c80eULL, 0x7d32d119196419c8ULL, 0x70923b4949394972ULL,
+	0x9aaf5fd9d943d986ULL, 0x1df931f2f2eff2c3ULL, 0x48dba8e3e3abe34bULL,
+	0x2ab6b95b5b715be2ULL, 0x920dbc88881a8834ULL, 0xc8293e9a9a529aa4ULL,
+	0xbe4c0b262698262dULL, 0xfa64bf3232c8328dULL, 0x4a7d59b0b0fab0e9ULL,
+	0x6acff2e9e983e91bULL, 0x331e770f0f3c0f78ULL, 0xa6b733d5d573d5e6ULL,
+	0xba1df480803a8074ULL, 0x7c6127bebec2be99ULL, 0xde87ebcdcd13cd26ULL,
+	0xe468893434d034bdULL, 0x75903248483d487aULL, 0x24e354ffffdbffabULL,
+	0x8ff48d7a7af57af7ULL, 0xea3d6490907a90f4ULL, 0x3ebe9d5f5f615fc2ULL,
+	0xa0403d202080201dULL, 0xd5d00f6868bd6867ULL, 0x7234ca1a1a681ad0ULL,
+	0x2c41b7aeae82ae19ULL, 0x5e757db4b4eab4c9ULL, 0x19a8ce54544d549aULL,
+	0xe53b7f93937693ecULL, 0xaa442f222288220dULL, 0xe9c86364648d6407ULL,
+	0x12ff2af1f1e3f1dbULL, 0xa2e6cc7373d173bfULL, 0x5a24821212481290ULL,
+	0x5d807a40401d403aULL, 0x2810480808200840ULL, 0xe89b95c3c32bc356ULL,
+	0x7bc5dfecec97ec33ULL, 0x90ab4ddbdb4bdb96ULL, 0x1f5fc0a1a1bea161ULL,
+	0x8307918d8d0e8d1cULL, 0xc97ac83d3df43df5ULL, 0xf1335b97976697ccULL,
+	0x0000000000000000ULL, 0xd483f9cfcf1bcf36ULL, 0x87566e2b2bac2b45ULL,
+	0xb3ece17676c57697ULL, 0xb019e68282328264ULL, 0xa9b128d6d67fd6feULL,
+	0x7736c31b1b6c1bd8ULL, 0x5b7774b5b5eeb5c1ULL, 0x2943beafaf86af11ULL,
+	0xdfd41d6a6ab56a77ULL, 0x0da0ea50505d50baULL, 0x4c8a574545094512ULL,
+	0x18fb38f3f3ebf3cbULL, 0xf060ad3030c0309dULL, 0x74c3c4efef9bef2bULL,
+	0xc37eda3f3ffc3fe5ULL, 0x1caac75555495592ULL, 0x1059dba2a2b2a279ULL,
+	0x65c9e9eaea8fea03ULL, 0xecca6a656589650fULL, 0x686903babad2bab9ULL,
+	0x935e4a2f2fbc2f65ULL, 0xe79d8ec0c027c04eULL, 0x81a160dede5fdebeULL,
+	0x6c38fc1c1c701ce0ULL, 0x2ee746fdfdd3fdbbULL, 0x649a1f4d4d294d52ULL,
+	0xe0397692927292e4ULL, 0xbceafa7575c9758fULL, 0x1e0c360606180630ULL,
+	0x9809ae8a8a128a24ULL, 0x40794bb2b2f2b2f9ULL, 0x59d185e6e6bfe663ULL,
+	0x361c7e0e0e380e70ULL, 0x633ee71f1f7c1ff8ULL, 0xf7c4556262956237ULL,
+	0xa3b53ad4d477d4eeULL, 0x324d81a8a89aa829ULL, 0xf4315296966296c4ULL,
+	0x3aef62f9f9c3f99bULL, 0xf697a3c5c533c566ULL, 0xb14a102525942535ULL,
+	0x20b2ab59597959f2ULL, 0xae15d084842a8454ULL, 0xa7e4c57272d572b7ULL,
+	0xdd72ec3939e439d5ULL, 0x6198164c4c2d4c5aULL, 0x3bbc945e5e655ecaULL,
+	0x85f09f7878fd78e7ULL, 0xd870e53838e038ddULL, 0x8605988c8c0a8c14ULL,
+	0xb2bf17d1d163d1c6ULL, 0x0b57e4a5a5aea541ULL, 0x4dd9a1e2e2afe243ULL,
+	0xf8c24e616199612fULL, 0x457b42b3b3f6b3f1ULL, 0xa542342121842115ULL,
+	0xd625089c9c4a9c94ULL, 0x663cee1e1e781ef0ULL, 0x5286614343114322ULL,
+	0xfc93b1c7c73bc776ULL, 0x2be54ffcfcd7fcb3ULL, 0x1408240404100420ULL,
+	0x08a2e351515951b2ULL, 0xc72f2599995e99bcULL, 0xc4da226d6da96d4fULL,
+	0x391a650d0d340d68ULL, 0x35e979fafacffa83ULL, 0x84a369dfdf5bdfb6ULL,
+	0x9bfca97e7ee57ed7ULL, 0xb44819242490243dULL, 0xd776fe3b3bec3bc5ULL,
+	0x3d4b9aabab96ab31ULL, 0xd181f0cece1fce3eULL, 0x5522991111441188ULL,
+	0x8903838f8f068f0cULL, 0x6b9c044e4e254e4aULL, 0x517366b7b7e6b7d1ULL,
+	0x60cbe0ebeb8beb0bULL, 0xcc78c13c3cf03cfdULL, 0xbf1ffd81813e817cULL,
+	0xfe354094946a94d4ULL, 0x0cf31cf7f7fbf7ebULL, 0x676f18b9b9deb9a1ULL,
+	0x5f268b13134c1398ULL, 0x9c58512c2cb02c7dULL, 0xb8bb05d3d36bd3d6ULL,
+	0x5cd38ce7e7bbe76bULL, 0xcbdc396e6ea56e57ULL, 0xf395aac4c437c46eULL,
+	0x0f061b03030c0318ULL, 0x13acdc565645568aULL, 0x49885e44440d441aULL,
+	0x9efea07f7fe17fdfULL, 0x374f88a9a99ea921ULL, 0x8254672a2aa82a4dULL,
+	0x6d6b0abbbbd6bbb1ULL, 0xe29f87c1c123c146ULL, 0x02a6f153535153a2ULL,
+	0x8ba572dcdc57dcaeULL, 0x2716530b0b2c0b58ULL, 0xd327019d9d4e9d9cULL,
+	0xc1d82b6c6cad6c47ULL, 0xf562a43131c43195ULL, 0xb9e8f37474cd7487ULL,
+	0x09f115f6f6fff6e3ULL, 0x438c4c464605460aULL, 0x2645a5acac8aac09ULL,
+	0x970fb589891e893cULL, 0x4428b414145014a0ULL, 0x42dfbae1e1a3e15bULL,
+	0x4e2ca616165816b0ULL, 0xd274f73a3ae83acdULL, 0xd0d2066969b9696fULL,
+	0x2d12410909240948ULL, 0xade0d77070dd70a7ULL, 0x54716fb6b6e2b6d9ULL,
+	0xb7bd1ed0d067d0ceULL, 0x7ec7d6eded93ed3bULL, 0xdb85e2cccc17cc2eULL,
+	0x578468424215422aULL, 0xc22d2c98985a98b4ULL, 0x0e55eda4a4aaa449ULL,
+	0x8850752828a0285dULL, 0x31b8865c5c6d5cdaULL, 0x3fed6bf8f8c7f893ULL,
+	0xa411c28686228644ULL,
+};
+
+static const u64 C4[256] = {
+	0xc07830d818186018ULL, 0x05af462623238c23ULL, 0x7ef991b8c6c63fc6ULL,
+	0x136fcdfbe8e887e8ULL, 0x4ca113cb87872687ULL, 0xa9626d11b8b8dab8ULL,
+	0x0805020901010401ULL, 0x426e9e0d4f4f214fULL, 0xadee6c9b3636d836ULL,
+	0x590451ffa6a6a2a6ULL, 0xdebdb90cd2d26fd2ULL, 0xfb06f70ef5f5f3f5ULL,
+	0xef80f2967979f979ULL, 0x5fcede306f6fa16fULL, 0xfcef3f6d91917e91ULL,
+	0xaa07a4f852525552ULL, 0x27fdc04760609d60ULL, 0x89766535bcbccabcULL,
+	0xaccd2b379b9b569bULL, 0x048c018a8e8e028eULL, 0x71155bd2a3a3b6a3ULL,
+	0x603c186c0c0c300cULL, 0xff8af6847b7bf17bULL, 0xb5e16a803535d435ULL,
+	0xe8693af51d1d741dULL, 0x5347ddb3e0e0a7e0ULL, 0xf6acb321d7d77bd7ULL,
+	0x5eed999cc2c22fc2ULL, 0x6d965c432e2eb82eULL, 0x627a96294b4b314bULL,
+	0xa321e15dfefedffeULL, 0x8216aed557574157ULL, 0xa8412abd15155415ULL,
+	0x9fb6eee87777c177ULL, 0xa5eb6e923737dc37ULL, 0x7b56d79ee5e5b3e5ULL,
+	0x8cd923139f9f469fULL, 0xd317fd23f0f0e7f0ULL, 0x6a7f94204a4a354aULL,
+	0x9e95a944dada4fdaULL, 0xfa25b0a258587d58ULL, 0x06ca8fcfc9c903c9ULL,
+	0x558d527c2929a429ULL, 0x5022145a0a0a280aULL, 0xe14f7f50b1b1feb1ULL,
+	0x691a5dc9a0a0baa0ULL, 0x7fdad6146b6bb16bULL, 0x5cab17d985852e85ULL,
+	0x8173673cbdbdcebdULL, 0xd234ba8f5d5d695dULL, 0x8050209010104010ULL,
+	0xf303f507f4f4f7f4ULL, 0x16c08bddcbcb0bcbULL, 0xedc67cd33e3ef83eULL,
+	0x28110a2d05051405ULL, 0x1fe6ce7867678167ULL, 0x7353d597e4e4b7e4ULL,
+	0x25bb4e0227279c27ULL, 0x3258827341411941ULL, 0x2c9d0ba78b8b168bULL,
+	0x510153f6a7a7a6a7ULL, 0xcf94fab27d7de97dULL, 0xdcfb374995956e95ULL,
+	0x8e9fad56d8d847d8ULL, 0x8b30eb70fbfbcbfbULL, 0x2371c1cdeeee9feeULL,
+	0xc791f8bb7c7ced7cULL, 0x17e3cc7166668566ULL, 0xa68ea77bdddd53ddULL,
+	0xb84b2eaf17175c17ULL, 0x02468e4547470147ULL, 0x84dc211a9e9e429eULL,
+	0x1ec589d4caca0fcaULL, 0x75995a582d2db42dULL, 0x9179632ebfbfc6bfULL,
+	0x381b0e3f07071c07ULL, 0x012347acadad8eadULL, 0xea2fb4b05a5a755aULL,
+	0x6cb51bef83833683ULL, 0x85ff66b63333cc33ULL, 0x3ff2c65c63639163ULL,
+	0x100a041202020802ULL, 0x39384993aaaa92aaULL, 0xafa8e2de7171d971ULL,
+	0x0ecf8dc6c8c807c8ULL, 0xc87d32d119196419ULL, 0x7270923b49493949ULL,
+	0x869aaf5fd9d943d9ULL, 0xc31df931f2f2eff2ULL, 0x4b48dba8e3e3abe3ULL,
+	0xe22ab6b95b5b715bULL, 0x34920dbc88881a88ULL, 0xa4c8293e9a9a529aULL,
+	0x2dbe4c0b26269826ULL, 0x8dfa64bf3232c832ULL, 0xe94a7d59b0b0fab0ULL,
+	0x1b6acff2e9e983e9ULL, 0x78331e770f0f3c0fULL, 0xe6a6b733d5d573d5ULL,
+	0x74ba1df480803a80ULL, 0x997c6127bebec2beULL, 0x26de87ebcdcd13cdULL,
+	0xbde468893434d034ULL, 0x7a75903248483d48ULL, 0xab24e354ffffdbffULL,
+	0xf78ff48d7a7af57aULL, 0xf4ea3d6490907a90ULL, 0xc23ebe9d5f5f615fULL,
+	0x1da0403d20208020ULL, 0x67d5d00f6868bd68ULL, 0xd07234ca1a1a681aULL,
+	0x192c41b7aeae82aeULL, 0xc95e757db4b4eab4ULL, 0x9a19a8ce54544d54ULL,
+	0xece53b7f93937693ULL, 0x0daa442f22228822ULL, 0x07e9c86364648d64ULL,
+	0xdb12ff2af1f1e3f1ULL, 0xbfa2e6cc7373d173ULL, 0x905a248212124812ULL,
+	0x3a5d807a40401d40ULL, 0x4028104808082008ULL, 0x56e89b95c3c32bc3ULL,
+	0x337bc5dfecec97ecULL, 0x9690ab4ddbdb4bdbULL, 0x611f5fc0a1a1bea1ULL,
+	0x1c8307918d8d0e8dULL, 0xf5c97ac83d3df43dULL, 0xccf1335b97976697ULL,
+	0x0000000000000000ULL, 0x36d483f9cfcf1bcfULL, 0x4587566e2b2bac2bULL,
+	0x97b3ece17676c576ULL, 0x64b019e682823282ULL, 0xfea9b128d6d67fd6ULL,
+	0xd87736c31b1b6c1bULL, 0xc15b7774b5b5eeb5ULL, 0x112943beafaf86afULL,
+	0x77dfd41d6a6ab56aULL, 0xba0da0ea50505d50ULL, 0x124c8a5745450945ULL,
+	0xcb18fb38f3f3ebf3ULL, 0x9df060ad3030c030ULL, 0x2b74c3c4efef9befULL,
+	0xe5c37eda3f3ffc3fULL, 0x921caac755554955ULL, 0x791059dba2a2b2a2ULL,
+	0x0365c9e9eaea8feaULL, 0x0fecca6a65658965ULL, 0xb9686903babad2baULL,
+	0x65935e4a2f2fbc2fULL, 0x4ee79d8ec0c027c0ULL, 0xbe81a160dede5fdeULL,
+	0xe06c38fc1c1c701cULL, 0xbb2ee746fdfdd3fdULL, 0x52649a1f4d4d294dULL,
+	0xe4e0397692927292ULL, 0x8fbceafa7575c975ULL, 0x301e0c3606061806ULL,
+	0x249809ae8a8a128aULL, 0xf940794bb2b2f2b2ULL, 0x6359d185e6e6bfe6ULL,
+	0x70361c7e0e0e380eULL, 0xf8633ee71f1f7c1fULL, 0x37f7c45562629562ULL,
+	0xeea3b53ad4d477d4ULL, 0x29324d81a8a89aa8ULL, 0xc4f4315296966296ULL,
+	0x9b3aef62f9f9c3f9ULL, 0x66f697a3c5c533c5ULL, 0x35b14a1025259425ULL,
+	0xf220b2ab59597959ULL, 0x54ae15d084842a84ULL, 0xb7a7e4c57272d572ULL,
+	0xd5dd72ec3939e439ULL, 0x5a6198164c4c2d4cULL, 0xca3bbc945e5e655eULL,
+	0xe785f09f7878fd78ULL, 0xddd870e53838e038ULL, 0x148605988c8c0a8cULL,
+	0xc6b2bf17d1d163d1ULL, 0x410b57e4a5a5aea5ULL, 0x434dd9a1e2e2afe2ULL,
+	0x2ff8c24e61619961ULL, 0xf1457b42b3b3f6b3ULL, 0x15a5423421218421ULL,
+	0x94d625089c9c4a9cULL, 0xf0663cee1e1e781eULL, 0x2252866143431143ULL,
+	0x76fc93b1c7c73bc7ULL, 0xb32be54ffcfcd7fcULL, 0x2014082404041004ULL,
+	0xb208a2e351515951ULL, 0xbcc72f2599995e99ULL, 0x4fc4da226d6da96dULL,
+	0x68391a650d0d340dULL, 0x8335e979fafacffaULL, 0xb684a369dfdf5bdfULL,
+	0xd79bfca97e7ee57eULL, 0x3db4481924249024ULL, 0xc5d776fe3b3bec3bULL,
+	0x313d4b9aabab96abULL, 0x3ed181f0cece1fceULL, 0x8855229911114411ULL,
+	0x0c8903838f8f068fULL, 0x4a6b9c044e4e254eULL, 0xd1517366b7b7e6b7ULL,
+	0x0b60cbe0ebeb8bebULL, 0xfdcc78c13c3cf03cULL, 0x7cbf1ffd81813e81ULL,
+	0xd4fe354094946a94ULL, 0xeb0cf31cf7f7fbf7ULL, 0xa1676f18b9b9deb9ULL,
+	0x985f268b13134c13ULL, 0x7d9c58512c2cb02cULL, 0xd6b8bb05d3d36bd3ULL,
+	0x6b5cd38ce7e7bbe7ULL, 0x57cbdc396e6ea56eULL, 0x6ef395aac4c437c4ULL,
+	0x180f061b03030c03ULL, 0x8a13acdc56564556ULL, 0x1a49885e44440d44ULL,
+	0xdf9efea07f7fe17fULL, 0x21374f88a9a99ea9ULL, 0x4d8254672a2aa82aULL,
+	0xb16d6b0abbbbd6bbULL, 0x46e29f87c1c123c1ULL, 0xa202a6f153535153ULL,
+	0xae8ba572dcdc57dcULL, 0x582716530b0b2c0bULL, 0x9cd327019d9d4e9dULL,
+	0x47c1d82b6c6cad6cULL, 0x95f562a43131c431ULL, 0x87b9e8f37474cd74ULL,
+	0xe309f115f6f6fff6ULL, 0x0a438c4c46460546ULL, 0x092645a5acac8aacULL,
+	0x3c970fb589891e89ULL, 0xa04428b414145014ULL, 0x5b42dfbae1e1a3e1ULL,
+	0xb04e2ca616165816ULL, 0xcdd274f73a3ae83aULL, 0x6fd0d2066969b969ULL,
+	0x482d124109092409ULL, 0xa7ade0d77070dd70ULL, 0xd954716fb6b6e2b6ULL,
+	0xceb7bd1ed0d067d0ULL, 0x3b7ec7d6eded93edULL, 0x2edb85e2cccc17ccULL,
+	0x2a57846842421542ULL, 0xb4c22d2c98985a98ULL, 0x490e55eda4a4aaa4ULL,
+	0x5d8850752828a028ULL, 0xda31b8865c5c6d5cULL, 0x933fed6bf8f8c7f8ULL,
+	0x44a411c286862286ULL,
+};
+
+static const u64 C5[256] = {
+	0x18c07830d8181860ULL, 0x2305af462623238cULL, 0xc67ef991b8c6c63fULL,
+	0xe8136fcdfbe8e887ULL, 0x874ca113cb878726ULL, 0xb8a9626d11b8b8daULL,
+	0x0108050209010104ULL, 0x4f426e9e0d4f4f21ULL, 0x36adee6c9b3636d8ULL,
+	0xa6590451ffa6a6a2ULL, 0xd2debdb90cd2d26fULL, 0xf5fb06f70ef5f5f3ULL,
+	0x79ef80f2967979f9ULL, 0x6f5fcede306f6fa1ULL, 0x91fcef3f6d91917eULL,
+	0x52aa07a4f8525255ULL, 0x6027fdc04760609dULL, 0xbc89766535bcbccaULL,
+	0x9baccd2b379b9b56ULL, 0x8e048c018a8e8e02ULL, 0xa371155bd2a3a3b6ULL,
+	0x0c603c186c0c0c30ULL, 0x7bff8af6847b7bf1ULL, 0x35b5e16a803535d4ULL,
+	0x1de8693af51d1d74ULL, 0xe05347ddb3e0e0a7ULL, 0xd7f6acb321d7d77bULL,
+	0xc25eed999cc2c22fULL, 0x2e6d965c432e2eb8ULL, 0x4b627a96294b4b31ULL,
+	0xfea321e15dfefedfULL, 0x578216aed5575741ULL, 0x15a8412abd151554ULL,
+	0x779fb6eee87777c1ULL, 0x37a5eb6e923737dcULL, 0xe57b56d79ee5e5b3ULL,
+	0x9f8cd923139f9f46ULL, 0xf0d317fd23f0f0e7ULL, 0x4a6a7f94204a4a35ULL,
+	0xda9e95a944dada4fULL, 0x58fa25b0a258587dULL, 0xc906ca8fcfc9c903ULL,
+	0x29558d527c2929a4ULL, 0x0a5022145a0a0a28ULL, 0xb1e14f7f50b1b1feULL,
+	0xa0691a5dc9a0a0baULL, 0x6b7fdad6146b6bb1ULL, 0x855cab17d985852eULL,
+	0xbd8173673cbdbdceULL, 0x5dd234ba8f5d5d69ULL, 0x1080502090101040ULL,
+	0xf4f303f507f4f4f7ULL, 0xcb16c08bddcbcb0bULL, 0x3eedc67cd33e3ef8ULL,
+	0x0528110a2d050514ULL, 0x671fe6ce78676781ULL, 0xe47353d597e4e4b7ULL,
+	0x2725bb4e0227279cULL, 0x4132588273414119ULL, 0x8b2c9d0ba78b8b16ULL,
+	0xa7510153f6a7a7a6ULL, 0x7dcf94fab27d7de9ULL, 0x95dcfb374995956eULL,
+	0xd88e9fad56d8d847ULL, 0xfb8b30eb70fbfbcbULL, 0xee2371c1cdeeee9fULL,
+	0x7cc791f8bb7c7cedULL, 0x6617e3cc71666685ULL, 0xdda68ea77bdddd53ULL,
+	0x17b84b2eaf17175cULL, 0x4702468e45474701ULL, 0x9e84dc211a9e9e42ULL,
+	0xca1ec589d4caca0fULL, 0x2d75995a582d2db4ULL, 0xbf9179632ebfbfc6ULL,
+	0x07381b0e3f07071cULL, 0xad012347acadad8eULL, 0x5aea2fb4b05a5a75ULL,
+	0x836cb51bef838336ULL, 0x3385ff66b63333ccULL, 0x633ff2c65c636391ULL,
+	0x02100a0412020208ULL, 0xaa39384993aaaa92ULL, 0x71afa8e2de7171d9ULL,
+	0xc80ecf8dc6c8c807ULL, 0x19c87d32d1191964ULL, 0x497270923b494939ULL,
+	0xd9869aaf5fd9d943ULL, 0xf2c31df931f2f2efULL, 0xe34b48dba8e3e3abULL,
+	0x5be22ab6b95b5b71ULL, 0x8834920dbc88881aULL, 0x9aa4c8293e9a9a52ULL,
+	0x262dbe4c0b262698ULL, 0x328dfa64bf3232c8ULL, 0xb0e94a7d59b0b0faULL,
+	0xe91b6acff2e9e983ULL, 0x0f78331e770f0f3cULL, 0xd5e6a6b733d5d573ULL,
+	0x8074ba1df480803aULL, 0xbe997c6127bebec2ULL, 0xcd26de87ebcdcd13ULL,
+	0x34bde468893434d0ULL, 0x487a75903248483dULL, 0xffab24e354ffffdbULL,
+	0x7af78ff48d7a7af5ULL, 0x90f4ea3d6490907aULL, 0x5fc23ebe9d5f5f61ULL,
+	0x201da0403d202080ULL, 0x6867d5d00f6868bdULL, 0x1ad07234ca1a1a68ULL,
+	0xae192c41b7aeae82ULL, 0xb4c95e757db4b4eaULL, 0x549a19a8ce54544dULL,
+	0x93ece53b7f939376ULL, 0x220daa442f222288ULL, 0x6407e9c86364648dULL,
+	0xf1db12ff2af1f1e3ULL, 0x73bfa2e6cc7373d1ULL, 0x12905a2482121248ULL,
+	0x403a5d807a40401dULL, 0x0840281048080820ULL, 0xc356e89b95c3c32bULL,
+	0xec337bc5dfecec97ULL, 0xdb9690ab4ddbdb4bULL, 0xa1611f5fc0a1a1beULL,
+	0x8d1c8307918d8d0eULL, 0x3df5c97ac83d3df4ULL, 0x97ccf1335b979766ULL,
+	0x0000000000000000ULL, 0xcf36d483f9cfcf1bULL, 0x2b4587566e2b2bacULL,
+	0x7697b3ece17676c5ULL, 0x8264b019e6828232ULL, 0xd6fea9b128d6d67fULL,
+	0x1bd87736c31b1b6cULL, 0xb5c15b7774b5b5eeULL, 0xaf112943beafaf86ULL,
+	0x6a77dfd41d6a6ab5ULL, 0x50ba0da0ea50505dULL, 0x45124c8a57454509ULL,
+	0xf3cb18fb38f3f3ebULL, 0x309df060ad3030c0ULL, 0xef2b74c3c4efef9bULL,
+	0x3fe5c37eda3f3ffcULL, 0x55921caac7555549ULL, 0xa2791059dba2a2b2ULL,
+	0xea0365c9e9eaea8fULL, 0x650fecca6a656589ULL, 0xbab9686903babad2ULL,
+	0x2f65935e4a2f2fbcULL, 0xc04ee79d8ec0c027ULL, 0xdebe81a160dede5fULL,
+	0x1ce06c38fc1c1c70ULL, 0xfdbb2ee746fdfdd3ULL, 0x4d52649a1f4d4d29ULL,
+	0x92e4e03976929272ULL, 0x758fbceafa7575c9ULL, 0x06301e0c36060618ULL,
+	0x8a249809ae8a8a12ULL, 0xb2f940794bb2b2f2ULL, 0xe66359d185e6e6bfULL,
+	0x0e70361c7e0e0e38ULL, 0x1ff8633ee71f1f7cULL, 0x6237f7c455626295ULL,
+	0xd4eea3b53ad4d477ULL, 0xa829324d81a8a89aULL, 0x96c4f43152969662ULL,
+	0xf99b3aef62f9f9c3ULL, 0xc566f697a3c5c533ULL, 0x2535b14a10252594ULL,
+	0x59f220b2ab595979ULL, 0x8454ae15d084842aULL, 0x72b7a7e4c57272d5ULL,
+	0x39d5dd72ec3939e4ULL, 0x4c5a6198164c4c2dULL, 0x5eca3bbc945e5e65ULL,
+	0x78e785f09f7878fdULL, 0x38ddd870e53838e0ULL, 0x8c148605988c8c0aULL,
+	0xd1c6b2bf17d1d163ULL, 0xa5410b57e4a5a5aeULL, 0xe2434dd9a1e2e2afULL,
+	0x612ff8c24e616199ULL, 0xb3f1457b42b3b3f6ULL, 0x2115a54234212184ULL,
+	0x9c94d625089c9c4aULL, 0x1ef0663cee1e1e78ULL, 0x4322528661434311ULL,
+	0xc776fc93b1c7c73bULL, 0xfcb32be54ffcfcd7ULL, 0x0420140824040410ULL,
+	0x51b208a2e3515159ULL, 0x99bcc72f2599995eULL, 0x6d4fc4da226d6da9ULL,
+	0x0d68391a650d0d34ULL, 0xfa8335e979fafacfULL, 0xdfb684a369dfdf5bULL,
+	0x7ed79bfca97e7ee5ULL, 0x243db44819242490ULL, 0x3bc5d776fe3b3becULL,
+	0xab313d4b9aabab96ULL, 0xce3ed181f0cece1fULL, 0x1188552299111144ULL,
+	0x8f0c8903838f8f06ULL, 0x4e4a6b9c044e4e25ULL, 0xb7d1517366b7b7e6ULL,
+	0xeb0b60cbe0ebeb8bULL, 0x3cfdcc78c13c3cf0ULL, 0x817cbf1ffd81813eULL,
+	0x94d4fe354094946aULL, 0xf7eb0cf31cf7f7fbULL, 0xb9a1676f18b9b9deULL,
+	0x13985f268b13134cULL, 0x2c7d9c58512c2cb0ULL, 0xd3d6b8bb05d3d36bULL,
+	0xe76b5cd38ce7e7bbULL, 0x6e57cbdc396e6ea5ULL, 0xc46ef395aac4c437ULL,
+	0x03180f061b03030cULL, 0x568a13acdc565645ULL, 0x441a49885e44440dULL,
+	0x7fdf9efea07f7fe1ULL, 0xa921374f88a9a99eULL, 0x2a4d8254672a2aa8ULL,
+	0xbbb16d6b0abbbbd6ULL, 0xc146e29f87c1c123ULL, 0x53a202a6f1535351ULL,
+	0xdcae8ba572dcdc57ULL, 0x0b582716530b0b2cULL, 0x9d9cd327019d9d4eULL,
+	0x6c47c1d82b6c6cadULL, 0x3195f562a43131c4ULL, 0x7487b9e8f37474cdULL,
+	0xf6e309f115f6f6ffULL, 0x460a438c4c464605ULL, 0xac092645a5acac8aULL,
+	0x893c970fb589891eULL, 0x14a04428b4141450ULL, 0xe15b42dfbae1e1a3ULL,
+	0x16b04e2ca6161658ULL, 0x3acdd274f73a3ae8ULL, 0x696fd0d2066969b9ULL,
+	0x09482d1241090924ULL, 0x70a7ade0d77070ddULL, 0xb6d954716fb6b6e2ULL,
+	0xd0ceb7bd1ed0d067ULL, 0xed3b7ec7d6eded93ULL, 0xcc2edb85e2cccc17ULL,
+	0x422a578468424215ULL, 0x98b4c22d2c98985aULL, 0xa4490e55eda4a4aaULL,
+	0x285d8850752828a0ULL, 0x5cda31b8865c5c6dULL, 0xf8933fed6bf8f8c7ULL,
+	0x8644a411c2868622ULL,
+};
+
+static const u64 C6[256] = {
+	0x6018c07830d81818ULL, 0x8c2305af46262323ULL, 0x3fc67ef991b8c6c6ULL,
+	0x87e8136fcdfbe8e8ULL, 0x26874ca113cb8787ULL, 0xdab8a9626d11b8b8ULL,
+	0x0401080502090101ULL, 0x214f426e9e0d4f4fULL, 0xd836adee6c9b3636ULL,
+	0xa2a6590451ffa6a6ULL, 0x6fd2debdb90cd2d2ULL, 0xf3f5fb06f70ef5f5ULL,
+	0xf979ef80f2967979ULL, 0xa16f5fcede306f6fULL, 0x7e91fcef3f6d9191ULL,
+	0x5552aa07a4f85252ULL, 0x9d6027fdc0476060ULL, 0xcabc89766535bcbcULL,
+	0x569baccd2b379b9bULL, 0x028e048c018a8e8eULL, 0xb6a371155bd2a3a3ULL,
+	0x300c603c186c0c0cULL, 0xf17bff8af6847b7bULL, 0xd435b5e16a803535ULL,
+	0x741de8693af51d1dULL, 0xa7e05347ddb3e0e0ULL, 0x7bd7f6acb321d7d7ULL,
+	0x2fc25eed999cc2c2ULL, 0xb82e6d965c432e2eULL, 0x314b627a96294b4bULL,
+	0xdffea321e15dfefeULL, 0x41578216aed55757ULL, 0x5415a8412abd1515ULL,
+	0xc1779fb6eee87777ULL, 0xdc37a5eb6e923737ULL, 0xb3e57b56d79ee5e5ULL,
+	0x469f8cd923139f9fULL, 0xe7f0d317fd23f0f0ULL, 0x354a6a7f94204a4aULL,
+	0x4fda9e95a944dadaULL, 0x7d58fa25b0a25858ULL, 0x03c906ca8fcfc9c9ULL,
+	0xa429558d527c2929ULL, 0x280a5022145a0a0aULL, 0xfeb1e14f7f50b1b1ULL,
+	0xbaa0691a5dc9a0a0ULL, 0xb16b7fdad6146b6bULL, 0x2e855cab17d98585ULL,
+	0xcebd8173673cbdbdULL, 0x695dd234ba8f5d5dULL, 0x4010805020901010ULL,
+	0xf7f4f303f507f4f4ULL, 0x0bcb16c08bddcbcbULL, 0xf83eedc67cd33e3eULL,
+	0x140528110a2d0505ULL, 0x81671fe6ce786767ULL, 0xb7e47353d597e4e4ULL,
+	0x9c2725bb4e022727ULL, 0x1941325882734141ULL, 0x168b2c9d0ba78b8bULL,
+	0xa6a7510153f6a7a7ULL, 0xe97dcf94fab27d7dULL, 0x6e95dcfb37499595ULL,
+	0x47d88e9fad56d8d8ULL, 0xcbfb8b30eb70fbfbULL, 0x9fee2371c1cdeeeeULL,
+	0xed7cc791f8bb7c7cULL, 0x856617e3cc716666ULL, 0x53dda68ea77bddddULL,
+	0x5c17b84b2eaf1717ULL, 0x014702468e454747ULL, 0x429e84dc211a9e9eULL,
+	0x0fca1ec589d4cacaULL, 0xb42d75995a582d2dULL, 0xc6bf9179632ebfbfULL,
+	0x1c07381b0e3f0707ULL, 0x8ead012347acadadULL, 0x755aea2fb4b05a5aULL,
+	0x36836cb51bef8383ULL, 0xcc3385ff66b63333ULL, 0x91633ff2c65c6363ULL,
+	0x0802100a04120202ULL, 0x92aa39384993aaaaULL, 0xd971afa8e2de7171ULL,
+	0x07c80ecf8dc6c8c8ULL, 0x6419c87d32d11919ULL, 0x39497270923b4949ULL,
+	0x43d9869aaf5fd9d9ULL, 0xeff2c31df931f2f2ULL, 0xabe34b48dba8e3e3ULL,
+	0x715be22ab6b95b5bULL, 0x1a8834920dbc8888ULL, 0x529aa4c8293e9a9aULL,
+	0x98262dbe4c0b2626ULL, 0xc8328dfa64bf3232ULL, 0xfab0e94a7d59b0b0ULL,
+	0x83e91b6acff2e9e9ULL, 0x3c0f78331e770f0fULL, 0x73d5e6a6b733d5d5ULL,
+	0x3a8074ba1df48080ULL, 0xc2be997c6127bebeULL, 0x13cd26de87ebcdcdULL,
+	0xd034bde468893434ULL, 0x3d487a7590324848ULL, 0xdbffab24e354ffffULL,
+	0xf57af78ff48d7a7aULL, 0x7a90f4ea3d649090ULL, 0x615fc23ebe9d5f5fULL,
+	0x80201da0403d2020ULL, 0xbd6867d5d00f6868ULL, 0x681ad07234ca1a1aULL,
+	0x82ae192c41b7aeaeULL, 0xeab4c95e757db4b4ULL, 0x4d549a19a8ce5454ULL,
+	0x7693ece53b7f9393ULL, 0x88220daa442f2222ULL, 0x8d6407e9c8636464ULL,
+	0xe3f1db12ff2af1f1ULL, 0xd173bfa2e6cc7373ULL, 0x4812905a24821212ULL,
+	0x1d403a5d807a4040ULL, 0x2008402810480808ULL, 0x2bc356e89b95c3c3ULL,
+	0x97ec337bc5dfececULL, 0x4bdb9690ab4ddbdbULL, 0xbea1611f5fc0a1a1ULL,
+	0x0e8d1c8307918d8dULL, 0xf43df5c97ac83d3dULL, 0x6697ccf1335b9797ULL,
+	0x0000000000000000ULL, 0x1bcf36d483f9cfcfULL, 0xac2b4587566e2b2bULL,
+	0xc57697b3ece17676ULL, 0x328264b019e68282ULL, 0x7fd6fea9b128d6d6ULL,
+	0x6c1bd87736c31b1bULL, 0xeeb5c15b7774b5b5ULL, 0x86af112943beafafULL,
+	0xb56a77dfd41d6a6aULL, 0x5d50ba0da0ea5050ULL, 0x0945124c8a574545ULL,
+	0xebf3cb18fb38f3f3ULL, 0xc0309df060ad3030ULL, 0x9bef2b74c3c4efefULL,
+	0xfc3fe5c37eda3f3fULL, 0x4955921caac75555ULL, 0xb2a2791059dba2a2ULL,
+	0x8fea0365c9e9eaeaULL, 0x89650fecca6a6565ULL, 0xd2bab9686903babaULL,
+	0xbc2f65935e4a2f2fULL, 0x27c04ee79d8ec0c0ULL, 0x5fdebe81a160dedeULL,
+	0x701ce06c38fc1c1cULL, 0xd3fdbb2ee746fdfdULL, 0x294d52649a1f4d4dULL,
+	0x7292e4e039769292ULL, 0xc9758fbceafa7575ULL, 0x1806301e0c360606ULL,
+	0x128a249809ae8a8aULL, 0xf2b2f940794bb2b2ULL, 0xbfe66359d185e6e6ULL,
+	0x380e70361c7e0e0eULL, 0x7c1ff8633ee71f1fULL, 0x956237f7c4556262ULL,
+	0x77d4eea3b53ad4d4ULL, 0x9aa829324d81a8a8ULL, 0x6296c4f431529696ULL,
+	0xc3f99b3aef62f9f9ULL, 0x33c566f697a3c5c5ULL, 0x942535b14a102525ULL,
+	0x7959f220b2ab5959ULL, 0x2a8454ae15d08484ULL, 0xd572b7a7e4c57272ULL,
+	0xe439d5dd72ec3939ULL, 0x2d4c5a6198164c4cULL, 0x655eca3bbc945e5eULL,
+	0xfd78e785f09f7878ULL, 0xe038ddd870e53838ULL, 0x0a8c148605988c8cULL,
+	0x63d1c6b2bf17d1d1ULL, 0xaea5410b57e4a5a5ULL, 0xafe2434dd9a1e2e2ULL,
+	0x99612ff8c24e6161ULL, 0xf6b3f1457b42b3b3ULL, 0x842115a542342121ULL,
+	0x4a9c94d625089c9cULL, 0x781ef0663cee1e1eULL, 0x1143225286614343ULL,
+	0x3bc776fc93b1c7c7ULL, 0xd7fcb32be54ffcfcULL, 0x1004201408240404ULL,
+	0x5951b208a2e35151ULL, 0x5e99bcc72f259999ULL, 0xa96d4fc4da226d6dULL,
+	0x340d68391a650d0dULL, 0xcffa8335e979fafaULL, 0x5bdfb684a369dfdfULL,
+	0xe57ed79bfca97e7eULL, 0x90243db448192424ULL, 0xec3bc5d776fe3b3bULL,
+	0x96ab313d4b9aababULL, 0x1fce3ed181f0ceceULL, 0x4411885522991111ULL,
+	0x068f0c8903838f8fULL, 0x254e4a6b9c044e4eULL, 0xe6b7d1517366b7b7ULL,
+	0x8beb0b60cbe0ebebULL, 0xf03cfdcc78c13c3cULL, 0x3e817cbf1ffd8181ULL,
+	0x6a94d4fe35409494ULL, 0xfbf7eb0cf31cf7f7ULL, 0xdeb9a1676f18b9b9ULL,
+	0x4c13985f268b1313ULL, 0xb02c7d9c58512c2cULL, 0x6bd3d6b8bb05d3d3ULL,
+	0xbbe76b5cd38ce7e7ULL, 0xa56e57cbdc396e6eULL, 0x37c46ef395aac4c4ULL,
+	0x0c03180f061b0303ULL, 0x45568a13acdc5656ULL, 0x0d441a49885e4444ULL,
+	0xe17fdf9efea07f7fULL, 0x9ea921374f88a9a9ULL, 0xa82a4d8254672a2aULL,
+	0xd6bbb16d6b0abbbbULL, 0x23c146e29f87c1c1ULL, 0x5153a202a6f15353ULL,
+	0x57dcae8ba572dcdcULL, 0x2c0b582716530b0bULL, 0x4e9d9cd327019d9dULL,
+	0xad6c47c1d82b6c6cULL, 0xc43195f562a43131ULL, 0xcd7487b9e8f37474ULL,
+	0xfff6e309f115f6f6ULL, 0x05460a438c4c4646ULL, 0x8aac092645a5acacULL,
+	0x1e893c970fb58989ULL, 0x5014a04428b41414ULL, 0xa3e15b42dfbae1e1ULL,
+	0x5816b04e2ca61616ULL, 0xe83acdd274f73a3aULL, 0xb9696fd0d2066969ULL,
+	0x2409482d12410909ULL, 0xdd70a7ade0d77070ULL, 0xe2b6d954716fb6b6ULL,
+	0x67d0ceb7bd1ed0d0ULL, 0x93ed3b7ec7d6ededULL, 0x17cc2edb85e2ccccULL,
+	0x15422a5784684242ULL, 0x5a98b4c22d2c9898ULL, 0xaaa4490e55eda4a4ULL,
+	0xa0285d8850752828ULL, 0x6d5cda31b8865c5cULL, 0xc7f8933fed6bf8f8ULL,
+	0x228644a411c28686ULL,
+};
+
+static const u64 C7[256] = {
+	0x186018c07830d818ULL, 0x238c2305af462623ULL, 0xc63fc67ef991b8c6ULL,
+	0xe887e8136fcdfbe8ULL, 0x8726874ca113cb87ULL, 0xb8dab8a9626d11b8ULL,
+	0x0104010805020901ULL, 0x4f214f426e9e0d4fULL, 0x36d836adee6c9b36ULL,
+	0xa6a2a6590451ffa6ULL, 0xd26fd2debdb90cd2ULL, 0xf5f3f5fb06f70ef5ULL,
+	0x79f979ef80f29679ULL, 0x6fa16f5fcede306fULL, 0x917e91fcef3f6d91ULL,
+	0x525552aa07a4f852ULL, 0x609d6027fdc04760ULL, 0xbccabc89766535bcULL,
+	0x9b569baccd2b379bULL, 0x8e028e048c018a8eULL, 0xa3b6a371155bd2a3ULL,
+	0x0c300c603c186c0cULL, 0x7bf17bff8af6847bULL, 0x35d435b5e16a8035ULL,
+	0x1d741de8693af51dULL, 0xe0a7e05347ddb3e0ULL, 0xd77bd7f6acb321d7ULL,
+	0xc22fc25eed999cc2ULL, 0x2eb82e6d965c432eULL, 0x4b314b627a96294bULL,
+	0xfedffea321e15dfeULL, 0x5741578216aed557ULL, 0x155415a8412abd15ULL,
+	0x77c1779fb6eee877ULL, 0x37dc37a5eb6e9237ULL, 0xe5b3e57b56d79ee5ULL,
+	0x9f469f8cd923139fULL, 0xf0e7f0d317fd23f0ULL, 0x4a354a6a7f94204aULL,
+	0xda4fda9e95a944daULL, 0x587d58fa25b0a258ULL, 0xc903c906ca8fcfc9ULL,
+	0x29a429558d527c29ULL, 0x0a280a5022145a0aULL, 0xb1feb1e14f7f50b1ULL,
+	0xa0baa0691a5dc9a0ULL, 0x6bb16b7fdad6146bULL, 0x852e855cab17d985ULL,
+	0xbdcebd8173673cbdULL, 0x5d695dd234ba8f5dULL, 0x1040108050209010ULL,
+	0xf4f7f4f303f507f4ULL, 0xcb0bcb16c08bddcbULL, 0x3ef83eedc67cd33eULL,
+	0x05140528110a2d05ULL, 0x6781671fe6ce7867ULL, 0xe4b7e47353d597e4ULL,
+	0x279c2725bb4e0227ULL, 0x4119413258827341ULL, 0x8b168b2c9d0ba78bULL,
+	0xa7a6a7510153f6a7ULL, 0x7de97dcf94fab27dULL, 0x956e95dcfb374995ULL,
+	0xd847d88e9fad56d8ULL, 0xfbcbfb8b30eb70fbULL, 0xee9fee2371c1cdeeULL,
+	0x7ced7cc791f8bb7cULL, 0x66856617e3cc7166ULL, 0xdd53dda68ea77bddULL,
+	0x175c17b84b2eaf17ULL, 0x47014702468e4547ULL, 0x9e429e84dc211a9eULL,
+	0xca0fca1ec589d4caULL, 0x2db42d75995a582dULL, 0xbfc6bf9179632ebfULL,
+	0x071c07381b0e3f07ULL, 0xad8ead012347acadULL, 0x5a755aea2fb4b05aULL,
+	0x8336836cb51bef83ULL, 0x33cc3385ff66b633ULL, 0x6391633ff2c65c63ULL,
+	0x020802100a041202ULL, 0xaa92aa39384993aaULL, 0x71d971afa8e2de71ULL,
+	0xc807c80ecf8dc6c8ULL, 0x196419c87d32d119ULL, 0x4939497270923b49ULL,
+	0xd943d9869aaf5fd9ULL, 0xf2eff2c31df931f2ULL, 0xe3abe34b48dba8e3ULL,
+	0x5b715be22ab6b95bULL, 0x881a8834920dbc88ULL, 0x9a529aa4c8293e9aULL,
+	0x2698262dbe4c0b26ULL, 0x32c8328dfa64bf32ULL, 0xb0fab0e94a7d59b0ULL,
+	0xe983e91b6acff2e9ULL, 0x0f3c0f78331e770fULL, 0xd573d5e6a6b733d5ULL,
+	0x803a8074ba1df480ULL, 0xbec2be997c6127beULL, 0xcd13cd26de87ebcdULL,
+	0x34d034bde4688934ULL, 0x483d487a75903248ULL, 0xffdbffab24e354ffULL,
+	0x7af57af78ff48d7aULL, 0x907a90f4ea3d6490ULL, 0x5f615fc23ebe9d5fULL,
+	0x2080201da0403d20ULL, 0x68bd6867d5d00f68ULL, 0x1a681ad07234ca1aULL,
+	0xae82ae192c41b7aeULL, 0xb4eab4c95e757db4ULL, 0x544d549a19a8ce54ULL,
+	0x937693ece53b7f93ULL, 0x2288220daa442f22ULL, 0x648d6407e9c86364ULL,
+	0xf1e3f1db12ff2af1ULL, 0x73d173bfa2e6cc73ULL, 0x124812905a248212ULL,
+	0x401d403a5d807a40ULL, 0x0820084028104808ULL, 0xc32bc356e89b95c3ULL,
+	0xec97ec337bc5dfecULL, 0xdb4bdb9690ab4ddbULL, 0xa1bea1611f5fc0a1ULL,
+	0x8d0e8d1c8307918dULL, 0x3df43df5c97ac83dULL, 0x976697ccf1335b97ULL,
+	0x0000000000000000ULL, 0xcf1bcf36d483f9cfULL, 0x2bac2b4587566e2bULL,
+	0x76c57697b3ece176ULL, 0x82328264b019e682ULL, 0xd67fd6fea9b128d6ULL,
+	0x1b6c1bd87736c31bULL, 0xb5eeb5c15b7774b5ULL, 0xaf86af112943beafULL,
+	0x6ab56a77dfd41d6aULL, 0x505d50ba0da0ea50ULL, 0x450945124c8a5745ULL,
+	0xf3ebf3cb18fb38f3ULL, 0x30c0309df060ad30ULL, 0xef9bef2b74c3c4efULL,
+	0x3ffc3fe5c37eda3fULL, 0x554955921caac755ULL, 0xa2b2a2791059dba2ULL,
+	0xea8fea0365c9e9eaULL, 0x6589650fecca6a65ULL, 0xbad2bab9686903baULL,
+	0x2fbc2f65935e4a2fULL, 0xc027c04ee79d8ec0ULL, 0xde5fdebe81a160deULL,
+	0x1c701ce06c38fc1cULL, 0xfdd3fdbb2ee746fdULL, 0x4d294d52649a1f4dULL,
+	0x927292e4e0397692ULL, 0x75c9758fbceafa75ULL, 0x061806301e0c3606ULL,
+	0x8a128a249809ae8aULL, 0xb2f2b2f940794bb2ULL, 0xe6bfe66359d185e6ULL,
+	0x0e380e70361c7e0eULL, 0x1f7c1ff8633ee71fULL, 0x62956237f7c45562ULL,
+	0xd477d4eea3b53ad4ULL, 0xa89aa829324d81a8ULL, 0x966296c4f4315296ULL,
+	0xf9c3f99b3aef62f9ULL, 0xc533c566f697a3c5ULL, 0x25942535b14a1025ULL,
+	0x597959f220b2ab59ULL, 0x842a8454ae15d084ULL, 0x72d572b7a7e4c572ULL,
+	0x39e439d5dd72ec39ULL, 0x4c2d4c5a6198164cULL, 0x5e655eca3bbc945eULL,
+	0x78fd78e785f09f78ULL, 0x38e038ddd870e538ULL, 0x8c0a8c148605988cULL,
+	0xd163d1c6b2bf17d1ULL, 0xa5aea5410b57e4a5ULL, 0xe2afe2434dd9a1e2ULL,
+	0x6199612ff8c24e61ULL, 0xb3f6b3f1457b42b3ULL, 0x21842115a5423421ULL,
+	0x9c4a9c94d625089cULL, 0x1e781ef0663cee1eULL, 0x4311432252866143ULL,
+	0xc73bc776fc93b1c7ULL, 0xfcd7fcb32be54ffcULL, 0x0410042014082404ULL,
+	0x515951b208a2e351ULL, 0x995e99bcc72f2599ULL, 0x6da96d4fc4da226dULL,
+	0x0d340d68391a650dULL, 0xfacffa8335e979faULL, 0xdf5bdfb684a369dfULL,
+	0x7ee57ed79bfca97eULL, 0x2490243db4481924ULL, 0x3bec3bc5d776fe3bULL,
+	0xab96ab313d4b9aabULL, 0xce1fce3ed181f0ceULL, 0x1144118855229911ULL,
+	0x8f068f0c8903838fULL, 0x4e254e4a6b9c044eULL, 0xb7e6b7d1517366b7ULL,
+	0xeb8beb0b60cbe0ebULL, 0x3cf03cfdcc78c13cULL, 0x813e817cbf1ffd81ULL,
+	0x946a94d4fe354094ULL, 0xf7fbf7eb0cf31cf7ULL, 0xb9deb9a1676f18b9ULL,
+	0x134c13985f268b13ULL, 0x2cb02c7d9c58512cULL, 0xd36bd3d6b8bb05d3ULL,
+	0xe7bbe76b5cd38ce7ULL, 0x6ea56e57cbdc396eULL, 0xc437c46ef395aac4ULL,
+	0x030c03180f061b03ULL, 0x5645568a13acdc56ULL, 0x440d441a49885e44ULL,
+	0x7fe17fdf9efea07fULL, 0xa99ea921374f88a9ULL, 0x2aa82a4d8254672aULL,
+	0xbbd6bbb16d6b0abbULL, 0xc123c146e29f87c1ULL, 0x535153a202a6f153ULL,
+	0xdc57dcae8ba572dcULL, 0x0b2c0b582716530bULL, 0x9d4e9d9cd327019dULL,
+	0x6cad6c47c1d82b6cULL, 0x31c43195f562a431ULL, 0x74cd7487b9e8f374ULL,
+	0xf6fff6e309f115f6ULL, 0x4605460a438c4c46ULL, 0xac8aac092645a5acULL,
+	0x891e893c970fb589ULL, 0x145014a04428b414ULL, 0xe1a3e15b42dfbae1ULL,
+	0x165816b04e2ca616ULL, 0x3ae83acdd274f73aULL, 0x69b9696fd0d20669ULL,
+	0x092409482d124109ULL, 0x70dd70a7ade0d770ULL, 0xb6e2b6d954716fb6ULL,
+	0xd067d0ceb7bd1ed0ULL, 0xed93ed3b7ec7d6edULL, 0xcc17cc2edb85e2ccULL,
+	0x4215422a57846842ULL, 0x985a98b4c22d2c98ULL, 0xa4aaa4490e55eda4ULL,
+	0x28a0285d88507528ULL, 0x5c6d5cda31b8865cULL, 0xf8c7f8933fed6bf8ULL,
+	0x86228644a411c286ULL,
+};
+
+static const u64 rc[WHIRLPOOL_ROUNDS + 1] = {
+	0x0000000000000000ULL, 0x1823c6e887b8014fULL, 0x36a6d2f5796f9152ULL,
+	0x60bc9b8ea30c7b35ULL, 0x1de0d7c22e4bfe57ULL, 0x157737e59ff04adaULL,
+	0x58c9290ab1a06b85ULL, 0xbd5d10f4cb3e0567ULL, 0xe427418ba77d95d8ULL,
+	0xfbee7c66dd17479eULL, 0xca2dbf07ad5a8333ULL,
+};
+
+/**
+ * The core Whirlpool transform.
+ */
+
+static void whirlpool_process_buffer(struct whirlpool_ctx *wctx) {
+	int i, r;
+	u64 K[8];        /* the round key */
+	u64 block[8];    /* mu(buffer) */
+	u64 state[8];    /* the cipher state */
+	u64 L[8];
+	u8 *buffer = wctx->buffer;
+
+	for (i = 0; i < 8; i++, buffer += 8) {
+		block[i] =
+		(((u64)buffer[0]        ) << 56) ^
+		(((u64)buffer[1] & 0xffL) << 48) ^
+		(((u64)buffer[2] & 0xffL) << 40) ^
+		(((u64)buffer[3] & 0xffL) << 32) ^
+		(((u64)buffer[4] & 0xffL) << 24) ^
+		(((u64)buffer[5] & 0xffL) << 16) ^
+		(((u64)buffer[6] & 0xffL) <<  8) ^
+		(((u64)buffer[7] & 0xffL)      );
+	}
+
+	state[0] = block[0] ^ (K[0] = wctx->hash[0]);
+	state[1] = block[1] ^ (K[1] = wctx->hash[1]);
+	state[2] = block[2] ^ (K[2] = wctx->hash[2]);
+	state[3] = block[3] ^ (K[3] = wctx->hash[3]);
+	state[4] = block[4] ^ (K[4] = wctx->hash[4]);
+	state[5] = block[5] ^ (K[5] = wctx->hash[5]);
+	state[6] = block[6] ^ (K[6] = wctx->hash[6]);
+	state[7] = block[7] ^ (K[7] = wctx->hash[7]);
+
+	for (r = 1; r <= WHIRLPOOL_ROUNDS; r++) {
+
+		L[0] = C0[(int)(K[0] >> 56)       ] ^
+			   C1[(int)(K[7] >> 48) & 0xff] ^
+			   C2[(int)(K[6] >> 40) & 0xff] ^
+			   C3[(int)(K[5] >> 32) & 0xff] ^
+			   C4[(int)(K[4] >> 24) & 0xff] ^
+			   C5[(int)(K[3] >> 16) & 0xff] ^
+			   C6[(int)(K[2] >>  8) & 0xff] ^
+			   C7[(int)(K[1]      ) & 0xff] ^
+			   rc[r];
+
+		L[1] = C0[(int)(K[1] >> 56)       ] ^
+			   C1[(int)(K[0] >> 48) & 0xff] ^
+			   C2[(int)(K[7] >> 40) & 0xff] ^
+			   C3[(int)(K[6] >> 32) & 0xff] ^
+			   C4[(int)(K[5] >> 24) & 0xff] ^
+			   C5[(int)(K[4] >> 16) & 0xff] ^
+			   C6[(int)(K[3] >>  8) & 0xff] ^
+			   C7[(int)(K[2]      ) & 0xff];
+
+		L[2] = C0[(int)(K[2] >> 56)       ] ^
+			   C1[(int)(K[1] >> 48) & 0xff] ^
+			   C2[(int)(K[0] >> 40) & 0xff] ^
+			   C3[(int)(K[7] >> 32) & 0xff] ^
+			   C4[(int)(K[6] >> 24) & 0xff] ^
+			   C5[(int)(K[5] >> 16) & 0xff] ^
+			   C6[(int)(K[4] >>  8) & 0xff] ^
+			   C7[(int)(K[3]      ) & 0xff];
+
+		L[3] = C0[(int)(K[3] >> 56)       ] ^
+			   C1[(int)(K[2] >> 48) & 0xff] ^
+			   C2[(int)(K[1] >> 40) & 0xff] ^
+			   C3[(int)(K[0] >> 32) & 0xff] ^
+			   C4[(int)(K[7] >> 24) & 0xff] ^
+			   C5[(int)(K[6] >> 16) & 0xff] ^
+			   C6[(int)(K[5] >>  8) & 0xff] ^
+			   C7[(int)(K[4]      ) & 0xff];
+
+		L[4] = C0[(int)(K[4] >> 56)       ] ^
+			   C1[(int)(K[3] >> 48) & 0xff] ^
+			   C2[(int)(K[2] >> 40) & 0xff] ^
+			   C3[(int)(K[1] >> 32) & 0xff] ^
+			   C4[(int)(K[0] >> 24) & 0xff] ^
+			   C5[(int)(K[7] >> 16) & 0xff] ^
+			   C6[(int)(K[6] >>  8) & 0xff] ^
+			   C7[(int)(K[5]      ) & 0xff];
+
+		L[5] = C0[(int)(K[5] >> 56)       ] ^
+			   C1[(int)(K[4] >> 48) & 0xff] ^
+			   C2[(int)(K[3] >> 40) & 0xff] ^
+			   C3[(int)(K[2] >> 32) & 0xff] ^
+			   C4[(int)(K[1] >> 24) & 0xff] ^
+			   C5[(int)(K[0] >> 16) & 0xff] ^
+			   C6[(int)(K[7] >>  8) & 0xff] ^
+			   C7[(int)(K[6]      ) & 0xff];
+
+		L[6] = C0[(int)(K[6] >> 56)       ] ^
+			   C1[(int)(K[5] >> 48) & 0xff] ^
+			   C2[(int)(K[4] >> 40) & 0xff] ^
+			   C3[(int)(K[3] >> 32) & 0xff] ^
+			   C4[(int)(K[2] >> 24) & 0xff] ^
+			   C5[(int)(K[1] >> 16) & 0xff] ^
+			   C6[(int)(K[0] >>  8) & 0xff] ^
+			   C7[(int)(K[7]      ) & 0xff];
+
+		L[7] = C0[(int)(K[7] >> 56)       ] ^
+			   C1[(int)(K[6] >> 48) & 0xff] ^
+			   C2[(int)(K[5] >> 40) & 0xff] ^
+			   C3[(int)(K[4] >> 32) & 0xff] ^
+			   C4[(int)(K[3] >> 24) & 0xff] ^
+			   C5[(int)(K[2] >> 16) & 0xff] ^
+			   C6[(int)(K[1] >>  8) & 0xff] ^
+			   C7[(int)(K[0]      ) & 0xff];
+
+		K[0] = L[0];
+		K[1] = L[1];
+		K[2] = L[2];
+		K[3] = L[3];
+		K[4] = L[4];
+		K[5] = L[5];
+		K[6] = L[6];
+		K[7] = L[7];
+
+		L[0] = C0[(int)(state[0] >> 56)       ] ^
+			   C1[(int)(state[7] >> 48) & 0xff] ^
+			   C2[(int)(state[6] >> 40) & 0xff] ^
+			   C3[(int)(state[5] >> 32) & 0xff] ^
+			   C4[(int)(state[4] >> 24) & 0xff] ^
+			   C5[(int)(state[3] >> 16) & 0xff] ^
+			   C6[(int)(state[2] >>  8) & 0xff] ^
+			   C7[(int)(state[1]      ) & 0xff] ^
+			   K[0];
+
+		L[1] = C0[(int)(state[1] >> 56)       ] ^
+			   C1[(int)(state[0] >> 48) & 0xff] ^
+			   C2[(int)(state[7] >> 40) & 0xff] ^
+			   C3[(int)(state[6] >> 32) & 0xff] ^
+			   C4[(int)(state[5] >> 24) & 0xff] ^
+			   C5[(int)(state[4] >> 16) & 0xff] ^
+			   C6[(int)(state[3] >>  8) & 0xff] ^
+			   C7[(int)(state[2]      ) & 0xff] ^
+			   K[1];
+
+		L[2] = C0[(int)(state[2] >> 56)       ] ^
+			   C1[(int)(state[1] >> 48) & 0xff] ^
+			   C2[(int)(state[0] >> 40) & 0xff] ^
+			   C3[(int)(state[7] >> 32) & 0xff] ^
+			   C4[(int)(state[6] >> 24) & 0xff] ^
+			   C5[(int)(state[5] >> 16) & 0xff] ^
+			   C6[(int)(state[4] >>  8) & 0xff] ^
+			   C7[(int)(state[3]      ) & 0xff] ^
+			   K[2];
+
+		L[3] = C0[(int)(state[3] >> 56)       ] ^
+			   C1[(int)(state[2] >> 48) & 0xff] ^
+			   C2[(int)(state[1] >> 40) & 0xff] ^
+			   C3[(int)(state[0] >> 32) & 0xff] ^
+			   C4[(int)(state[7] >> 24) & 0xff] ^
+			   C5[(int)(state[6] >> 16) & 0xff] ^
+			   C6[(int)(state[5] >>  8) & 0xff] ^
+			   C7[(int)(state[4]      ) & 0xff] ^
+			   K[3];
+
+		L[4] = C0[(int)(state[4] >> 56)       ] ^
+			   C1[(int)(state[3] >> 48) & 0xff] ^
+			   C2[(int)(state[2] >> 40) & 0xff] ^
+			   C3[(int)(state[1] >> 32) & 0xff] ^
+			   C4[(int)(state[0] >> 24) & 0xff] ^
+			   C5[(int)(state[7] >> 16) & 0xff] ^
+			   C6[(int)(state[6] >>  8) & 0xff] ^
+			   C7[(int)(state[5]      ) & 0xff] ^
+			   K[4];
+
+		L[5] = C0[(int)(state[5] >> 56)       ] ^
+			   C1[(int)(state[4] >> 48) & 0xff] ^
+			   C2[(int)(state[3] >> 40) & 0xff] ^
+			   C3[(int)(state[2] >> 32) & 0xff] ^
+			   C4[(int)(state[1] >> 24) & 0xff] ^
+			   C5[(int)(state[0] >> 16) & 0xff] ^
+			   C6[(int)(state[7] >>  8) & 0xff] ^
+			   C7[(int)(state[6]      ) & 0xff] ^
+			   K[5];
+
+		L[6] = C0[(int)(state[6] >> 56)       ] ^
+			   C1[(int)(state[5] >> 48) & 0xff] ^
+			   C2[(int)(state[4] >> 40) & 0xff] ^
+			   C3[(int)(state[3] >> 32) & 0xff] ^
+			   C4[(int)(state[2] >> 24) & 0xff] ^
+			   C5[(int)(state[1] >> 16) & 0xff] ^
+			   C6[(int)(state[0] >>  8) & 0xff] ^
+			   C7[(int)(state[7]      ) & 0xff] ^
+			   K[6];
+
+		L[7] = C0[(int)(state[7] >> 56)       ] ^
+			   C1[(int)(state[6] >> 48) & 0xff] ^
+			   C2[(int)(state[5] >> 40) & 0xff] ^
+			   C3[(int)(state[4] >> 32) & 0xff] ^
+			   C4[(int)(state[3] >> 24) & 0xff] ^
+			   C5[(int)(state[2] >> 16) & 0xff] ^
+			   C6[(int)(state[1] >>  8) & 0xff] ^
+			   C7[(int)(state[0]      ) & 0xff] ^
+			   K[7];
+
+		state[0] = L[0];
+		state[1] = L[1];
+		state[2] = L[2];
+		state[3] = L[3];
+		state[4] = L[4];
+		state[5] = L[5];
+		state[6] = L[6];
+		state[7] = L[7];
+	}
+	/*
+	* apply the Miyaguchi-Preneel compression function:
+	*/
+	wctx->hash[0] ^= state[0] ^ block[0];
+	wctx->hash[1] ^= state[1] ^ block[1];
+	wctx->hash[2] ^= state[2] ^ block[2];
+	wctx->hash[3] ^= state[3] ^ block[3];
+	wctx->hash[4] ^= state[4] ^ block[4];
+	wctx->hash[5] ^= state[5] ^ block[5];
+	wctx->hash[6] ^= state[6] ^ block[6];
+	wctx->hash[7] ^= state[7] ^ block[7];
+
+}
+
+static void whirlpool_init (void *ctx) {
+	int i;
+	struct whirlpool_ctx *wctx = ctx;
+
+	memset(wctx->bitLength, 0, 32);
+	wctx->bufferBits = wctx->bufferPos = 0;
+	wctx->buffer[0] = 0;
+	for (i = 0; i < 8; i++) {
+		wctx->hash[i] = 0L;
+	}
+}
+
+static void whirlpool_update(void *ctx, const u8 *source, unsigned int len)
+{
+
+	struct whirlpool_ctx *wctx = ctx;
+	int sourcePos    = 0;
+	unsigned int bits_len = len * 8; // convert to number of bits
+	int sourceGap    = (8 - ((int)bits_len & 7)) & 7;
+	int bufferRem    = wctx->bufferBits & 7;
+	int i;
+	u32 b, carry;
+	u8 *buffer       = wctx->buffer;
+	u8 *bitLength    = wctx->bitLength;
+	int bufferBits   = wctx->bufferBits;
+	int bufferPos    = wctx->bufferPos;
+
+	u64 value = bits_len;
+	for (i = 31, carry = 0; i >= 0 && (carry != 0 || value != 0ULL); i--) {
+		carry += bitLength[i] + ((u32)value & 0xff);
+		bitLength[i] = (u8)carry;
+		carry >>= 8;
+		value >>= 8;
+	}
+	while (bits_len > 8) {
+		b = ((source[sourcePos] << sourceGap) & 0xff) |
+		((source[sourcePos + 1] & 0xff) >> (8 - sourceGap));
+		buffer[bufferPos++] |= (u8)(b >> bufferRem);
+		bufferBits += 8 - bufferRem;
+		if (bufferBits == WHIRLPOOL_DIGEST_SIZE * 8) {
+			whirlpool_process_buffer(wctx);
+			bufferBits = bufferPos = 0;
+		}
+		buffer[bufferPos] = b << (8 - bufferRem);
+		bufferBits += bufferRem;
+		bits_len -= 8;
+		sourcePos++;
+	}
+	if (bits_len > 0) {
+		b = (source[sourcePos] << sourceGap) & 0xff;
+		buffer[bufferPos] |= b >> bufferRem;
+	} else {
+		b = 0;
+	}
+	if (bufferRem + bits_len < 8) {
+		bufferBits += bits_len;
+	} else {
+		bufferPos++;
+		bufferBits += 8 - bufferRem;
+		bits_len -= 8 - bufferRem;
+		if (bufferBits == WHIRLPOOL_DIGEST_SIZE * 8) {
+			whirlpool_process_buffer(wctx);
+			bufferBits = bufferPos = 0;
+		}
+		buffer[bufferPos] = b << (8 - bufferRem);
+		bufferBits += (int)bits_len;
+	}
+
+	wctx->bufferBits   = bufferBits;
+	wctx->bufferPos    = bufferPos;
+
+}
+
+static void whirlpool_final(void *ctx, u8 *out)
+{
+	struct whirlpool_ctx *wctx = ctx;
+	int i;
+   	u8 *buffer      = wctx->buffer;
+   	u8 *bitLength   = wctx->bitLength;
+   	int bufferBits  = wctx->bufferBits;
+   	int bufferPos   = wctx->bufferPos;
+   	u8 *digest      = out;
+
+   	buffer[bufferPos] |= 0x80U >> (bufferBits & 7);
+   	bufferPos++;
+   	if (bufferPos > WHIRLPOOL_BLOCK_SIZE - WHIRLPOOL_LENGTHBYTES) {
+   		if (bufferPos < WHIRLPOOL_BLOCK_SIZE) {
+	   	memset(&buffer[bufferPos], 0, WHIRLPOOL_BLOCK_SIZE - bufferPos);
+   		}
+   		whirlpool_process_buffer(wctx);
+   		bufferPos = 0;
+   	}
+   	if (bufferPos < WHIRLPOOL_BLOCK_SIZE - WHIRLPOOL_LENGTHBYTES) {
+   		memset(&buffer[bufferPos], 0,
+			  (WHIRLPOOL_BLOCK_SIZE - WHIRLPOOL_LENGTHBYTES) - bufferPos);
+   	}
+   	bufferPos = WHIRLPOOL_BLOCK_SIZE - WHIRLPOOL_LENGTHBYTES;
+   	memcpy(&buffer[WHIRLPOOL_BLOCK_SIZE - WHIRLPOOL_LENGTHBYTES],
+		   bitLength, WHIRLPOOL_LENGTHBYTES);
+   	whirlpool_process_buffer(wctx);
+   	for (i = 0; i < WHIRLPOOL_DIGEST_SIZE/8; i++) {
+		digest[0] = (u8)(wctx->hash[i] >> 56);
+		digest[1] = (u8)(wctx->hash[i] >> 48);
+		digest[2] = (u8)(wctx->hash[i] >> 40);
+		digest[3] = (u8)(wctx->hash[i] >> 32);
+		digest[4] = (u8)(wctx->hash[i] >> 24);
+		digest[5] = (u8)(wctx->hash[i] >> 16);
+		digest[6] = (u8)(wctx->hash[i] >>  8);
+		digest[7] = (u8)(wctx->hash[i]      );
+		digest += 8;
+   	}
+   	wctx->bufferBits   = bufferBits;
+   	wctx->bufferPos    = bufferPos;
+}
+
+static struct crypto_alg alg = {
+	.cra_name	=	"whirlpool",
+	.cra_flags	=	CRYPTO_ALG_TYPE_DIGEST,
+	.cra_blocksize	=	WHIRLPOOL_BLOCK_SIZE,
+	.cra_ctxsize	=	sizeof(struct whirlpool_ctx),
+	.cra_module	=	THIS_MODULE,
+	.cra_list       =       LIST_HEAD_INIT(alg.cra_list),	
+	.cra_u		=	{ .digest = {
+	.dia_digestsize	=	WHIRLPOOL_DIGEST_SIZE,
+	.dia_init   	= 	whirlpool_init,
+	.dia_update 	=	whirlpool_update,
+	.dia_final  	=	whirlpool_final } }
+};
+
+static int __init init(void)
+{
+	return crypto_register_alg(&alg);
+}
+
+static void __exit fini(void)
+{
+	crypto_unregister_alg(&alg);
+}
+
+module_init(init);
+module_exit(fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Whirlpool Message Digest Algorithm");
diff -Nru a/drivers/char/synclinkmp.c b/drivers/char/synclinkmp.c
--- a/drivers/char/synclinkmp.c	2003-09-09 09:09:31 -07:00
+++ b/drivers/char/synclinkmp.c	2004-08-26 11:02:57 -07:00
@@ -1,5 +1,5 @@
 /*
- * $Id: synclinkmp.c,v 3.22 2003/09/05 14:04:26 paulkf Exp $
+ * $Id: synclinkmp.c,v 3.23 2004/08/24 19:49:48 paulkf Exp $
  *
  * Device driver for Microgate SyncLink Multiport
  * high speed multiprotocol serial adapter.
@@ -504,7 +504,7 @@
 MODULE_PARM(dosyncppp,"1-" __MODULE_STRING(MAX_DEVICES) "i");
 
 static char *driver_name = "SyncLink MultiPort driver";
-static char *driver_version = "$Revision: 3.22 $";
+static char *driver_version = "$Revision: 3.23 $";
 
 static int __devinit synclinkmp_init_one(struct pci_dev *dev,const struct pci_device_id *ent);
 static void __devexit synclinkmp_remove_one(struct pci_dev *dev);
@@ -681,7 +681,7 @@
 static unsigned char tx_negate_fifo_level = 32;	// tx request FIFO negation level in bytes
 
 static u32 misc_ctrl_value = 0x007e4040;
-static u32 lcr1_brdr_value = 0x0080002d;
+static u32 lcr1_brdr_value = 0x00800029;
 
 static u32 read_ahead_count = 8;
 
@@ -2036,16 +2036,15 @@
 {
  	struct tty_struct *tty = info->tty;
  	struct	mgsl_icount *icount = &info->icount;
-	unsigned char status = read_reg(info, SR1);
-	unsigned char status2 = read_reg(info, SR2);
+	unsigned char status = read_reg(info, SR1) & info->ie1_value & (FLGD + IDLD + CDCD + BRKD);
+	unsigned char status2 = read_reg(info, SR2) & info->ie2_value & OVRN;
 
 	/* clear status bits */
-	if ( status & (FLGD + IDLD + CDCD + BRKD) )
-		write_reg(info, SR1, 
-				(unsigned char)(status & (FLGD + IDLD + CDCD + BRKD)));
+	if (status)
+		write_reg(info, SR1, status);
 
-	if ( status2 & OVRN )
-		write_reg(info, SR2, (unsigned char)(status2 & OVRN));
+	if (status2)
+		write_reg(info, SR2, status2);
 	
 	if ( debug_level >= DEBUG_LEVEL_ISR )
 		printk("%s(%d):%s isr_rxint status=%02X %02x\n",
@@ -2182,15 +2181,22 @@
 		printk("%s(%d):%s isr_txeom status=%02x\n",
 			__FILE__,__LINE__,info->device_name,status);
 
-	/* disable and clear MSCI interrupts */
-	info->ie1_value &= ~(IDLE + UDRN);
-	write_reg(info, IE1, info->ie1_value);
-	write_reg(info, SR1, (unsigned char)(UDRN + IDLE));
-
 	write_reg(info, TXDMA + DIR, 0x00); /* disable Tx DMA IRQs */
 	write_reg(info, TXDMA + DSR, 0xc0); /* clear IRQs and disable DMA */
 	write_reg(info, TXDMA + DCMD, SWABORT);	/* reset/init DMA channel */
 
+	if (status & UDRN) {
+		write_reg(info, CMD, TXRESET);
+		write_reg(info, CMD, TXENABLE);
+	} else
+		write_reg(info, CMD, TXBUFCLR);
+
+	/* disable and clear tx interrupts */
+	info->ie0_value &= ~TXRDYE;
+	info->ie1_value &= ~(IDLE + UDRN);
+	write_reg16(info, IE0, (unsigned short)((info->ie1_value << 8) + info->ie0_value));
+	write_reg(info, SR1, (unsigned char)(UDRN + IDLE));
+
 	if ( info->tx_active ) {
 		if (info->params.mode != MGSL_MODE_ASYNC) {
 			if (status & UDRN)
@@ -2231,10 +2237,10 @@
  */
 void isr_txint(SLMP_INFO * info)
 {
-	unsigned char status = read_reg(info, SR1);
+	unsigned char status = read_reg(info, SR1) & info->ie1_value & (UDRN + IDLE + CCTS);
 
 	/* clear status bits */
-	write_reg(info, SR1, (unsigned char)(status & (UDRN + IDLE + CCTS)));
+	write_reg(info, SR1, status);
 
 	if ( debug_level >= DEBUG_LEVEL_ISR )
 		printk("%s(%d):%s isr_txint status=%02x\n",
@@ -2263,6 +2269,14 @@
 		printk("%s(%d):%s isr_txrdy() tx_count=%d\n",
 			__FILE__,__LINE__,info->device_name,info->tx_count);
 
+	if (info->params.mode != MGSL_MODE_ASYNC) {
+		/* disable TXRDY IRQ, enable IDLE IRQ */
+		info->ie0_value &= ~TXRDYE;
+		info->ie1_value |= IDLE;
+		write_reg16(info, IE0, (unsigned short)((info->ie1_value << 8) + info->ie0_value));
+		return;
+	}
+
 	if (info->tty && (info->tty->stopped || info->tty->hw_stopped)) {
 		tx_stop(info);
 		return;
@@ -2317,13 +2331,6 @@
 
 void isr_txdmaok(SLMP_INFO * info)
 {
-	/* BIT7 = EOT (end of transfer, used for async mode)
-	 * BIT6 = EOM (end of message/frame, used for sync mode)
-	 *
-	 * We don't look at DMA status because only EOT is enabled
-	 * and we always clear and disable all tx DMA IRQs.
-	 */
-//	unsigned char dma_status = read_reg(info,TXDMA + DSR) & 0xc0;
 	unsigned char status_reg1 = read_reg(info, SR1);
 
 	write_reg(info, TXDMA + DIR, 0x00);	/* disable Tx DMA IRQs */
@@ -2334,19 +2341,10 @@
 		printk("%s(%d):%s isr_txdmaok(), status=%02x\n",
 			__FILE__,__LINE__,info->device_name,status_reg1);
 
-	/* If transmitter already idle, do end of frame processing,
-	 * otherwise enable interrupt for tx IDLE.
-	 */
-	if (status_reg1 & IDLE)
-		isr_txeom(info, IDLE);
-	else {
-		/* disable and clear underrun IRQ, enable IDLE interrupt */
-		info->ie1_value |= IDLE;
-		info->ie1_value &= ~UDRN;
-		write_reg(info, IE1, info->ie1_value);
-
-		write_reg(info, SR1, UDRN);
-	}
+	/* program TXRDY as FIFO empty flag, enable TXRDY IRQ */
+	write_reg16(info, TRC0, 0);
+	info->ie0_value |= TXRDYE;
+	write_reg(info, IE0, info->ie0_value);
 }
 
 void isr_txdmaerror(SLMP_INFO * info)
@@ -3037,7 +3035,7 @@
 		unsigned char oldval = info->ie1_value;
 		unsigned char newval = oldval +
 			 (mask & MgslEvent_ExitHuntMode ? FLGD:0) +
-			 (mask & MgslEvent_IdleReceived ? IDLE:0);
+			 (mask & MgslEvent_IdleReceived ? IDLD:0);
 		if ( oldval != newval ) {
 			info->ie1_value = newval;
 			write_reg(info, IE1, info->ie1_value);
@@ -3104,7 +3102,7 @@
 		spin_lock_irqsave(&info->lock,flags);
 		if (!waitqueue_active(&info->event_wait_q)) {
 			/* disable enable exit hunt mode/idle rcvd IRQs */
-			info->ie1_value &= ~(FLGD|IDLE);
+			info->ie1_value &= ~(FLGD|IDLD);
 			write_reg(info, IE1, info->ie1_value);
 		}
 		spin_unlock_irqrestore(&info->lock,flags);
@@ -3554,9 +3552,10 @@
 
 int claim_resources(SLMP_INFO *info)
 {
-	if (request_mem_region(info->phys_memory_base,0x40000,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_memory_base,SCA_MEM_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_memory_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
@@ -3565,22 +3564,25 @@
 	if (request_mem_region(info->phys_lcr_base + info->lcr_offset,128,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s lcr mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_lcr_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
 		info->lcr_mem_requested = 1;
 
-	if (request_mem_region(info->phys_sca_base + info->sca_offset,512,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_sca_base + info->sca_offset,SCA_BASE_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s sca mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_sca_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
 		info->sca_base_requested = 1;
 
-	if (request_mem_region(info->phys_statctrl_base + info->statctrl_offset,16,"synclinkmp") == NULL) {
+	if (request_mem_region(info->phys_statctrl_base + info->statctrl_offset,SCA_REG_SIZE,"synclinkmp") == NULL) {
 		printk( "%s(%d):%s stat/ctrl mem addr conflict, Addr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_statctrl_base);
+		info->init_error = DiagStatus_AddressConflict;
 		goto errout;
 	}
 	else
@@ -3590,33 +3592,41 @@
 	if (!info->memory_base) {
 		printk( "%s(%d):%s Cant map shared memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
 
-	if ( !memory_test(info) ) {
-		printk( "%s(%d):Shared Memory Test failed for device %s MemAddr=%08X\n",
-			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
-		goto errout;
-	}
-
-	info->lcr_base = ioremap(info->phys_lcr_base,PAGE_SIZE) + info->lcr_offset;
+	info->lcr_base = ioremap(info->phys_lcr_base,PAGE_SIZE);
 	if (!info->lcr_base) {
 		printk( "%s(%d):%s Cant map LCR memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_lcr_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
+	info->lcr_base += info->lcr_offset;
 
-	info->sca_base = ioremap(info->phys_sca_base,PAGE_SIZE) + info->sca_offset;
+	info->sca_base = ioremap(info->phys_sca_base,PAGE_SIZE);
 	if (!info->sca_base) {
 		printk( "%s(%d):%s Cant map SCA memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_sca_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
 		goto errout;
 	}
+	info->sca_base += info->sca_offset;
 
-	info->statctrl_base = ioremap(info->phys_statctrl_base,PAGE_SIZE) + info->statctrl_offset;
+	info->statctrl_base = ioremap(info->phys_statctrl_base,PAGE_SIZE);
 	if (!info->statctrl_base) {
 		printk( "%s(%d):%s Cant map SCA Status/Control memory, MemAddr=%08X\n",
 			__FILE__,__LINE__,info->device_name, info->phys_statctrl_base );
+		info->init_error = DiagStatus_CantAssignPciResources;
+		goto errout;
+	}
+	info->statctrl_base += info->statctrl_offset;
+
+	if ( !memory_test(info) ) {
+		printk( "%s(%d):Shared Memory Test failed for device %s MemAddr=%08X\n",
+			__FILE__,__LINE__,info->device_name, info->phys_memory_base );
+		info->init_error = DiagStatus_MemoryError;
 		goto errout;
 	}
 
@@ -3639,7 +3649,7 @@
 	}
 
 	if ( info->shared_mem_requested ) {
-		release_mem_region(info->phys_memory_base,0x40000);
+		release_mem_region(info->phys_memory_base,SCA_MEM_SIZE);
 		info->shared_mem_requested = 0;
 	}
 	if ( info->lcr_mem_requested ) {
@@ -3647,11 +3657,11 @@
 		info->lcr_mem_requested = 0;
 	}
 	if ( info->sca_base_requested ) {
-		release_mem_region(info->phys_sca_base + info->sca_offset,512);
+		release_mem_region(info->phys_sca_base + info->sca_offset,SCA_BASE_SIZE);
 		info->sca_base_requested = 0;
 	}
 	if ( info->sca_statctrl_requested ) {
-		release_mem_region(info->phys_statctrl_base + info->statctrl_offset,16);
+		release_mem_region(info->phys_statctrl_base + info->statctrl_offset,SCA_REG_SIZE);
 		info->sca_statctrl_requested = 0;
 	}
 
@@ -3982,34 +3992,25 @@
 		       __FILE__,__LINE__,rc);
 	restore_flags(flags);
 
+	/* reset devices */
 	info = synclinkmp_device_list;
 	while(info) {
-#ifdef CONFIG_SYNCLINK_SYNCPPP
-		if (info->dosyncppp)
-			sppp_delete(info);
-#endif
 		reset_port(info);
-		if ( info->port_num == 0 ) {
-			if ( info->irq_requested ) {
-				free_irq(info->irq_level, info);
-				info->irq_requested = 0;
-			}
-		}
 		info = info->next_device;
 	}
 
-	/* port 0 of each adapter originally claimed
-	 * all resources, release those now
-	 */
+	/* release devices */
 	info = synclinkmp_device_list;
 	while(info) {
+#ifdef CONFIG_SYNCLINK_SYNCPPP
+		if (info->dosyncppp)
+			sppp_delete(info);
+#endif
 		free_dma_bufs(info);
 		free_tmp_rx_buf(info);
 		if ( info->port_num == 0 ) {
-			spin_lock_irqsave(&info->lock,flags);
-			reset_adapter(info);
-			write_reg(info, LPR, 1);		/* set low power mode */
-			spin_unlock_irqrestore(&info->lock,flags);
+			if (info->sca_base)
+				write_reg(info, LPR, 1); /* set low power mode */
 			release_resources(info);
 		}
 		tmp = info;
@@ -4229,6 +4230,9 @@
 				}
 			}
 
+			write_reg16(info, TRC0,
+				(unsigned short)(((tx_negate_fifo_level-1)<<8) + tx_active_fifo_level));
+
 			write_reg(info, TXDMA + DSR, 0); 		/* disable DMA channel */
 			write_reg(info, TXDMA + DCMD, SWABORT);	/* reset/init DMA channel */
 	
@@ -4240,11 +4244,10 @@
 			write_reg16(info, TXDMA + EDA,
 				info->tx_buf_list_ex[info->last_tx_buf].phys_entry);
 	
-			/* clear IDLE and UDRN status bit */
-			info->ie1_value &= ~(IDLE + UDRN);
-			if (info->params.mode != MGSL_MODE_ASYNC)
-				info->ie1_value |= UDRN;     		/* HDLC, IRQ on underrun */
-			write_reg(info, IE1, info->ie1_value);	/* enable MSCI interrupts */
+			/* enable underrun IRQ */
+			info->ie1_value &= ~IDLE;
+			info->ie1_value |= UDRN;
+			write_reg(info, IE1, info->ie1_value);
 			write_reg(info, SR1, (unsigned char)(IDLE + UDRN));
 	
 			write_reg(info, TXDMA + DIR, 0x40);		/* enable Tx DMA interrupts (EOM) */
diff -Nru a/drivers/char/vc_screen.c b/drivers/char/vc_screen.c
--- a/drivers/char/vc_screen.c	2004-07-30 06:29:39 -07:00
+++ b/drivers/char/vc_screen.c	2004-08-26 06:29:53 -07:00
@@ -136,10 +136,12 @@
 	if (!vc_cons_allocated(currcons))
 		goto unlock_out;
 
-	read = 0;
-	ret = 0;
+	ret = -EINVAL;
 	if (pos != n)
 		goto unlock_out;
+
+	read = 0;
+	ret = 0;
 	while (count) {
 		char *con_buf0, *con_buf_start;
 		long this_round, size;
diff -Nru a/drivers/hotplug/ibmphp_res.c b/drivers/hotplug/ibmphp_res.c
--- a/drivers/hotplug/ibmphp_res.c	2003-08-28 04:21:26 -07:00
+++ b/drivers/hotplug/ibmphp_res.c	2004-08-26 08:10:16 -07:00
@@ -42,7 +42,7 @@
 static int update_bridge_ranges (struct bus_node **);
 static int add_range (int type, struct range_node *, struct bus_node *);
 static void fix_resources (struct bus_node *);
-static inline struct bus_node *find_bus_wprev (u8, struct bus_node **, u8);
+static struct bus_node *find_bus_wprev (u8, struct bus_node **, u8);
 
 static LIST_HEAD(gbuses);
 LIST_HEAD(ibmphp_res_head);
@@ -1757,7 +1757,7 @@
 	return find_bus_wprev (bus_number, NULL, 0);
 }
 
-static inline struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u8 flag)
+static struct bus_node *find_bus_wprev (u8 bus_number, struct bus_node **prev, u8 flag)
 {
 	struct bus_node *bus_cur;
 	struct list_head *tmp;
diff -Nru a/drivers/ide/pci/sc1200.c b/drivers/ide/pci/sc1200.c
--- a/drivers/ide/pci/sc1200.c	2003-03-24 06:43:58 -08:00
+++ b/drivers/ide/pci/sc1200.c	2004-08-28 05:58:54 -07:00
@@ -418,7 +418,7 @@
 			ss = kmalloc(sizeof(sc1200_saved_state_t), GFP_KERNEL);
 			if (ss == NULL)
 				return -ENOMEM;
-			(sc1200_saved_state_t *)hwif->config_data = ss;
+			hwif->config_data = (unsigned long)ss;
 		}
 		ss = (sc1200_saved_state_t *)hwif->config_data;
 		//
diff -Nru a/drivers/isdn/hysdn/hysdn_procconf.c b/drivers/isdn/hysdn/hysdn_procconf.c
--- a/drivers/isdn/hysdn/hysdn_procconf.c	2004-07-30 06:29:39 -07:00
+++ b/drivers/isdn/hysdn/hysdn_procconf.c	2004-08-26 06:42:04 -07:00
@@ -219,20 +219,25 @@
 	if (off != &file->f_pos)	/* fs error check */
 		return -ESPIPE;
 
+	if (!(file->f_mode & FMODE_READ))
+		return -EPERM;
+
 	if (!(cp = file->private_data))
 		return (-EFAULT);	/* should never happen */
+
 	i = strlen(cp);	/* get total string length */
-	if (pos == (unsigned)pos && pos < i) {
-		/* still bytes to transfer */
-		cp += pos;	/* point to desired data offset */
-		i -= pos;	/* remaining length */
-		if (i > count)
-			i = count;	/* limit length to transfer */
-		if (copy_to_user(buf, cp, i))
-			return (-EFAULT);	/* copy error */
-		*off = pos + i;	/* adjust offset */
-	} else
-		return (0);
+
+	if (pos != (unsigned)pos || pos >= i)
+		return 0;
+
+	/* still bytes to transfer */
+	cp += pos;	/* point to desired data offset */
+	i -= pos;	/* remaining length */
+	if (i > count)
+		i = count;	/* limit length to transfer */
+	if (copy_to_user(buf, cp, i))
+		return (-EFAULT);	/* copy error */
+	*off = pos + i;	/* adjust offset */
 
 	return (i);
 }				/* hysdn_conf_read */
diff -Nru a/drivers/md/lvm.c b/drivers/md/lvm.c
--- a/drivers/md/lvm.c	2003-11-17 17:40:24 -08:00
+++ b/drivers/md/lvm.c	2004-08-30 17:24:13 -07:00
@@ -2689,6 +2689,10 @@
 			    (&lv_status_byname_req.lv->lv_current_pe,
 			     &saved_ptr1, sizeof(void *)) != 0)
 				return -EFAULT;
+			if (copy_to_user
+			    (&lv_status_byname_req.lv->lv_block_exception,
+			     &saved_ptr2, sizeof(void *)) != 0)
+				return -EFAULT;
 			return 0;
 		}
 	}
@@ -2743,6 +2747,10 @@
 	    (&lv_status_byindex_req.lv->lv_current_pe, &saved_ptr1,
 	     sizeof(void *)) != 0)
 		return -EFAULT;
+	if (copy_to_user
+	    (&lv_status_byindex_req.lv->lv_block_exception, &saved_ptr2,
+	     sizeof(void *)) != 0)
+		return -EFAULT;
 
 	return 0;
 }				/* lvm_do_lv_status_byindex() */
@@ -2799,6 +2807,10 @@
 	/* Restore usermode pointers */
 	if (copy_to_user
 	    (&lv_status_bydev_req.lv->lv_current_pe, &saved_ptr1,
+	     sizeof(void *)) != 0)
+		return -EFAULT;
+	if (copy_to_user
+	    (&lv_status_bydev_req.lv->lv_block_exception, &saved_ptr2,
 	     sizeof(void *)) != 0)
 		return -EFAULT;
 
diff -Nru a/drivers/net/e1000/e1000_param.c b/drivers/net/e1000/e1000_param.c
--- a/drivers/net/e1000/e1000_param.c	2004-05-25 01:20:23 -07:00
+++ b/drivers/net/e1000/e1000_param.c	2004-08-28 15:57:30 -07:00
@@ -212,7 +212,7 @@
 #define MAX_TXABSDELAY            0xFFFF
 #define MIN_TXABSDELAY                 0
 
-#define DEFAULT_ITR                    1
+#define DEFAULT_ITR                 8000
 #define MAX_ITR                   100000
 #define MIN_ITR                      100
 
diff -Nru a/drivers/net/fealnx.c b/drivers/net/fealnx.c
--- a/drivers/net/fealnx.c	2004-04-05 13:46:30 -07:00
+++ b/drivers/net/fealnx.c	2004-08-28 16:34:37 -07:00
@@ -1796,7 +1796,7 @@
 	unsigned long flags;
 	spin_lock_irqsave(lp, flags);
 	__set_rx_mode(dev);
-	spin_unlock_irqrestore(&lp, flags);
+	spin_unlock_irqrestore(lp, flags);
 }
 
 
diff -Nru a/drivers/net/ne2k-pci.c b/drivers/net/ne2k-pci.c
--- a/drivers/net/ne2k-pci.c	2003-11-18 12:16:28 -08:00
+++ b/drivers/net/ne2k-pci.c	2004-08-26 16:50:38 -07:00
@@ -505,8 +505,12 @@
 		insl(NE_BASE + NE_DATAPORT, buf, count>>2);
 		if (count & 3) {
 			buf += count & ~3;
-			if (count & 2)
-				*((u16*)buf)++ = le16_to_cpu(inw(NE_BASE + NE_DATAPORT));
+			if (count & 2) {
+				u16 *b = (u16 *)buf;
+
+				*b++ = le16_to_cpu(inw(NE_BASE + NE_DATAPORT));
+				buf = (char *)b;
+			}
 			if (count & 1)
 				*buf = inb(NE_BASE + NE_DATAPORT);
 		}
@@ -566,8 +570,12 @@
 		outsl(NE_BASE + NE_DATAPORT, buf, count>>2);
 		if (count & 3) {
 			buf += count & ~3;
-			if (count & 2)
-				outw(cpu_to_le16(*((u16*)buf)++), NE_BASE + NE_DATAPORT);
+			if (count & 2) {
+				u16 *b = (u16 *)buf;
+
+				outw(cpu_to_le16(*b++), NE_BASE + NE_DATAPORT);
+				buf = (char *)b;
+			}
 		}
 	}
 
diff -Nru a/drivers/net/tg3.c b/drivers/net/tg3.c
--- a/drivers/net/tg3.c	2004-07-21 14:14:20 -07:00
+++ b/drivers/net/tg3.c	2004-08-30 15:36:20 -07:00
@@ -4,6 +4,9 @@
  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
  * Copyright (C) 2004 Sun Microsystems Inc.
+ *
+ * Firmware is:
+ * 	Copyright (C) 2000-2003 Broadcom Corporation.
  */
 
 #include <linux/config.h>
@@ -56,8 +59,8 @@
 
 #define DRV_MODULE_NAME		"tg3"
 #define PFX DRV_MODULE_NAME	": "
-#define DRV_MODULE_VERSION	"3.8"
-#define DRV_MODULE_RELDATE	"July 14, 2004"
+#define DRV_MODULE_VERSION	"3.9"
+#define DRV_MODULE_RELDATE	"August 30, 2004"
 
 #define TG3_DEF_MAC_MODE	0
 #define TG3_DEF_RX_MODE		0
@@ -441,9 +444,14 @@
 		       0x1f);
 	tp->pci_clock_ctrl = clock_ctrl;
 
-	if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
-	    GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
-	    (orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
+	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
+	    GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
+		if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
+			tw32_f(TG3PCI_CLOCK_CTRL,
+			       clock_ctrl | CLOCK_CTRL_625_CORE);
+			udelay(40);
+		}
+	} else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 		tw32_f(TG3PCI_CLOCK_CTRL,
 		     clock_ctrl |
 		     (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK));
@@ -979,7 +987,7 @@
 		tp->link_config.orig_autoneg = tp->link_config.autoneg;
 	}
 
-	if (tp->phy_id != PHY_ID_SERDES) {
+	if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
 		tp->link_config.speed = SPEED_10;
 		tp->link_config.duplex = DUPLEX_HALF;
 		tp->link_config.autoneg = AUTONEG_ENABLE;
@@ -991,7 +999,7 @@
 	if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
 		u32 mac_mode;
 
-		if (tp->phy_id != PHY_ID_SERDES) {
+		if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
 			tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
 			udelay(40);
 
@@ -1486,6 +1494,18 @@
 	current_speed = SPEED_INVALID;
 	current_duplex = DUPLEX_INVALID;
 
+	if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
+		u32 val;
+
+		tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
+		tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
+		if (!(val & (1 << 10))) {
+			val |= (1 << 10);
+			tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
+			goto relink;
+		}
+	}
+
 	bmsr = 0;
 	for (i = 0; i < 100; i++) {
 		tg3_readphy(tp, MII_BMSR, &bmsr);
@@ -1565,7 +1585,7 @@
 			tg3_setup_flow_control(tp, local_adv, remote_adv);
 		}
 	}
-
+relink:
 	if (current_link_up == 0) {
 		u32 tmp;
 
@@ -1615,7 +1635,7 @@
 	tw32_f(MAC_MODE, tp->mac_mode);
 	udelay(40);
 
-	if (tp->tg3_flags & (TG3_FLAG_USE_LINKCHG_REG | TG3_FLAG_POLL_SERDES)) {
+	if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
 		/* Polled via timer. */
 		tw32_f(MAC_EVENT, 0);
 	} else {
@@ -1964,62 +1984,264 @@
 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
 {
 	int res = 0;
+	struct tg3_fiber_aneginfo aninfo;
+	int status = ANEG_FAILED;
+	unsigned int tick;
+	u32 tmp;
 
-	if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) {
-		u32 dig_status;
+	tw32_f(MAC_TX_AUTO_NEG, 0);
 
-		dig_status = tr32(SG_DIG_STATUS);
-		*flags = 0;
-		if (dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
-			*flags |= MR_LP_ADV_ASYM_PAUSE;
-		if (dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
-			*flags |= MR_LP_ADV_SYM_PAUSE;
-
-		if ((dig_status & SG_DIG_AUTONEG_COMPLETE) &&
-		    !(dig_status & (SG_DIG_AUTONEG_ERROR |
-				    SG_DIG_PARTNER_FAULT_MASK)))
-			res = 1;
-	} else {
-		struct tg3_fiber_aneginfo aninfo;
-		int status = ANEG_FAILED;
-		unsigned int tick;
-		u32 tmp;
+	tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
+	tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
+	udelay(40);
 
-		tw32_f(MAC_TX_AUTO_NEG, 0);
+	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
+	udelay(40);
 
-		tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
-		tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
-		udelay(40);
+	memset(&aninfo, 0, sizeof(aninfo));
+	aninfo.flags |= MR_AN_ENABLE;
+	aninfo.state = ANEG_STATE_UNKNOWN;
+	aninfo.cur_time = 0;
+	tick = 0;
+	while (++tick < 195000) {
+		status = tg3_fiber_aneg_smachine(tp, &aninfo);
+		if (status == ANEG_DONE || status == ANEG_FAILED)
+			break;
 
-		tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
-		udelay(40);
+		udelay(1);
+	}
 
-		memset(&aninfo, 0, sizeof(aninfo));
-		aninfo.flags |= MR_AN_ENABLE;
-		aninfo.state = ANEG_STATE_UNKNOWN;
-		aninfo.cur_time = 0;
-		tick = 0;
-		while (++tick < 195000) {
-			status = tg3_fiber_aneg_smachine(tp, &aninfo);
-			if (status == ANEG_DONE || status == ANEG_FAILED)
-				break;
+	tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
+	tw32_f(MAC_MODE, tp->mac_mode);
+	udelay(40);
 
-			udelay(1);
+	*flags = aninfo.flags;
+
+	if (status == ANEG_DONE &&
+	    (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
+			     MR_LP_ADV_FULL_DUPLEX)))
+		res = 1;
+
+	return res;
+}
+
+static void tg3_init_bcm8002(struct tg3 *tp)
+{
+	u32 mac_status = tr32(MAC_STATUS);
+	int i;
+
+	/* Reset when initting first time or we have a link. */
+	if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
+	    !(mac_status & MAC_STATUS_PCS_SYNCED))
+		return;
+
+	/* Set PLL lock range. */
+	tg3_writephy(tp, 0x16, 0x8007);
+
+	/* SW reset */
+	tg3_writephy(tp, MII_BMCR, BMCR_RESET);
+
+	/* Wait for reset to complete. */
+	/* XXX schedule_timeout() ... */
+	for (i = 0; i < 500; i++)
+		udelay(10);
+
+	/* Config mode; select PMA/Ch 1 regs. */
+	tg3_writephy(tp, 0x10, 0x8411);
+
+	/* Enable auto-lock and comdet, select txclk for tx. */
+	tg3_writephy(tp, 0x11, 0x0a10);
+
+	tg3_writephy(tp, 0x18, 0x00a0);
+	tg3_writephy(tp, 0x16, 0x41ff);
+
+	/* Assert and deassert POR. */
+	tg3_writephy(tp, 0x13, 0x0400);
+	udelay(40);
+	tg3_writephy(tp, 0x13, 0x0000);
+
+	tg3_writephy(tp, 0x11, 0x0a50);
+	udelay(40);
+	tg3_writephy(tp, 0x11, 0x0a10);
+
+	/* Wait for signal to stabilize */
+	/* XXX schedule_timeout() ... */
+	for (i = 0; i < 15000; i++)
+		udelay(10);
+
+	/* Deselect the channel register so we can read the PHYID
+	 * later.
+	 */
+	tg3_writephy(tp, 0x10, 0x8011);
+}
+
+static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
+{
+	u32 sg_dig_ctrl, sg_dig_status;
+	u32 serdes_cfg, expected_sg_dig_ctrl;
+	int workaround, port_a;
+	int current_link_up;
+
+	serdes_cfg = 0;
+	expected_sg_dig_ctrl = 0;
+	workaround = 0;
+	port_a = 1;
+	current_link_up = 0;
+
+	if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
+	    tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
+		workaround = 1;
+		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
+			port_a = 0;
+
+		serdes_cfg = tr32(MAC_SERDES_CFG) &
+			((1 << 23) | (1 << 22) | (1 << 21) | (1 << 20));
+	}
+
+	sg_dig_ctrl = tr32(SG_DIG_CTRL);
+
+	if (tp->link_config.autoneg != AUTONEG_ENABLE) {
+		if (sg_dig_ctrl & (1 << 31)) {
+			if (workaround) {
+				u32 val = serdes_cfg;
+
+				if (port_a)
+					val |= 0xc010880;
+				else
+					val |= 0x4010880;
+				tw32_f(MAC_SERDES_CFG, val);
+			}
+			tw32_f(SG_DIG_CTRL, 0x01388400);
 		}
+		if (mac_status & MAC_STATUS_PCS_SYNCED) {
+			tg3_setup_flow_control(tp, 0, 0);
+			current_link_up = 1;
+		}
+		goto out;
+	}
 
-		tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
-		tw32_f(MAC_MODE, tp->mac_mode);
-		udelay(40);
+	/* Want auto-negotiation.  */
+	expected_sg_dig_ctrl = 0x81388400;
+
+	/* Pause capability */
+	expected_sg_dig_ctrl |= (1 << 11);
 
-		*flags = aninfo.flags;
+	/* Asymettric pause */
+	expected_sg_dig_ctrl |= (1 << 12);
 
-		if (status == ANEG_DONE &&
-		    (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
-				     MR_LP_ADV_FULL_DUPLEX)))
-			res = 1;
+	if (sg_dig_ctrl != expected_sg_dig_ctrl) {
+		if (workaround)
+			tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011880);
+		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
+		udelay(5);
+		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
+
+		tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
+	} else if (mac_status & (MAC_STATUS_PCS_SYNCED |
+				 MAC_STATUS_SIGNAL_DET)) {
+		sg_dig_status = tr32(SG_DIG_STATUS);
+
+		if ((sg_dig_status & (1 << 1)) &&
+		    (mac_status & MAC_STATUS_PCS_SYNCED)) {
+			u32 local_adv, remote_adv;
+
+			local_adv = ADVERTISE_PAUSE_CAP;
+			remote_adv = 0;
+			if (sg_dig_status & (1 << 19))
+				remote_adv |= LPA_PAUSE_CAP;
+			if (sg_dig_status & (1 << 20))
+				remote_adv |= LPA_PAUSE_ASYM;
+
+			tg3_setup_flow_control(tp, local_adv, remote_adv);
+			current_link_up = 1;
+			tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
+		} else if (!(sg_dig_status & (1 << 1))) {
+			if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED)
+				tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
+			else {
+				if (workaround) {
+					u32 val = serdes_cfg;
+
+					if (port_a)
+						val |= 0xc010880;
+					else
+						val |= 0x4010880;
+
+					tw32_f(MAC_SERDES_CFG, serdes_cfg);
+				}
+
+				tw32_f(SG_DIG_CTRL, 0x01388400);
+				udelay(40);
+
+				mac_status = tr32(MAC_STATUS);
+				if (mac_status & MAC_STATUS_PCS_SYNCED) {
+					tg3_setup_flow_control(tp, 0, 0);
+					current_link_up = 1;
+				}
+			}
+		}
 	}
 
-	return res;
+out:
+	return current_link_up;
+}
+
+static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
+{
+	int current_link_up = 0;
+
+ 	if (!(mac_status & MAC_STATUS_PCS_SYNCED)) {
+		tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
+		goto out;
+	}
+
+	if (tp->link_config.autoneg == AUTONEG_ENABLE) {
+		u32 flags;
+		int i;
+  
+		if (fiber_autoneg(tp, &flags)) {
+			u32 local_adv, remote_adv;
+
+			local_adv = ADVERTISE_PAUSE_CAP;
+			remote_adv = 0;
+			if (flags & MR_LP_ADV_SYM_PAUSE)
+				remote_adv |= LPA_PAUSE_CAP;
+			if (flags & MR_LP_ADV_ASYM_PAUSE)
+				remote_adv |= LPA_PAUSE_ASYM;
+
+			tg3_setup_flow_control(tp, local_adv, remote_adv);
+
+			tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
+			current_link_up = 1;
+		}
+		for (i = 0; i < 30; i++) {
+			udelay(20);
+			tw32_f(MAC_STATUS,
+			       (MAC_STATUS_SYNC_CHANGED |
+				MAC_STATUS_CFG_CHANGED));
+			udelay(40);
+			if ((tr32(MAC_STATUS) &
+			     (MAC_STATUS_SYNC_CHANGED |
+			      MAC_STATUS_CFG_CHANGED)) == 0)
+				break;
+		}
+
+		mac_status = tr32(MAC_STATUS);
+		if (current_link_up == 0 &&
+		    (mac_status & MAC_STATUS_PCS_SYNCED) &&
+		    !(mac_status & MAC_STATUS_RCVD_CFG))
+			current_link_up = 1;
+	} else {
+		/* Forcing 1000FD link up. */
+		current_link_up = 1;
+		tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
+
+		tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
+		udelay(40);
+	}
+
+out:
+	return current_link_up;
 }
 
 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
@@ -2027,6 +2249,7 @@
 	u32 orig_pause_cfg;
 	u16 orig_active_speed;
 	u8 orig_active_duplex;
+	u32 mac_status;
 	int current_link_up;
 	int i;
 
@@ -2036,118 +2259,43 @@
 	orig_active_speed = tp->link_config.active_speed;
 	orig_active_duplex = tp->link_config.active_duplex;
 
-	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
-	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
-	tw32_f(MAC_MODE, tp->mac_mode);
-	udelay(40);
-
-	if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) {
-		/* Allow time for the hardware to auto-negotiate (195ms) */
-		unsigned int tick = 0;
-
-		while (++tick < 195000) { 
-			if (tr32(SG_DIG_STATUS) & SG_DIG_AUTONEG_COMPLETE)
-				break;
-			udelay(1);
+	if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
+	    netif_carrier_ok(tp->dev) &&
+	    (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
+		mac_status = tr32(MAC_STATUS);
+		mac_status &= (MAC_STATUS_PCS_SYNCED |
+			       MAC_STATUS_SIGNAL_DET |
+			       MAC_STATUS_CFG_CHANGED |
+			       MAC_STATUS_RCVD_CFG);
+		if (mac_status == (MAC_STATUS_PCS_SYNCED |
+				   MAC_STATUS_SIGNAL_DET)) {
+			tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
+					    MAC_STATUS_CFG_CHANGED));
+			return 0;
 		}
-		if (tick >= 195000)
-			printk(KERN_INFO PFX "%s: HW autoneg failed !\n",
-			    tp->dev->name);
 	}
 
-	/* Reset when initting first time or we have a link. */
-	if (!(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) ||
-	    (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED)) {
-		/* Set PLL lock range. */
-		tg3_writephy(tp, 0x16, 0x8007);
-
-		/* SW reset */
-		tg3_writephy(tp, MII_BMCR, BMCR_RESET);
-
-		/* Wait for reset to complete. */
-		/* XXX schedule_timeout() ... */
-		for (i = 0; i < 500; i++)
-			udelay(10);
-
-		/* Config mode; select PMA/Ch 1 regs. */
-		tg3_writephy(tp, 0x10, 0x8411);
-
-		/* Enable auto-lock and comdet, select txclk for tx. */
-		tg3_writephy(tp, 0x11, 0x0a10);
-
-		tg3_writephy(tp, 0x18, 0x00a0);
-		tg3_writephy(tp, 0x16, 0x41ff);
-
-		/* Assert and deassert POR. */
-		tg3_writephy(tp, 0x13, 0x0400);
-		udelay(40);
-		tg3_writephy(tp, 0x13, 0x0000);
+	tw32_f(MAC_TX_AUTO_NEG, 0);
 
-		tg3_writephy(tp, 0x11, 0x0a50);
-		udelay(40);
-		tg3_writephy(tp, 0x11, 0x0a10);
+	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
+	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
+	tw32_f(MAC_MODE, tp->mac_mode);
+	udelay(40);
 
-		/* Wait for signal to stabilize */
-		/* XXX schedule_timeout() ... */
-		for (i = 0; i < 15000; i++)
-			udelay(10);
+	if (tp->phy_id == PHY_ID_BCM8002)
+		tg3_init_bcm8002(tp);
 
-		/* Deselect the channel register so we can read the PHYID
-		 * later.
-		 */
-		tg3_writephy(tp, 0x10, 0x8011);
-	}
-
-	/* Enable link change interrupt unless serdes polling.  */
-	if (!(tp->tg3_flags & TG3_FLAG_POLL_SERDES))
-		tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
-	else
-		tw32_f(MAC_EVENT, 0);
+	/* Enable link change event even when serdes polling.  */
+	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
 	udelay(40);
 
 	current_link_up = 0;
- 	if (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) {
-		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
-			u32 flags;
-  
-			if (fiber_autoneg(tp, &flags)) {
-				u32 local_adv, remote_adv;
-
-				local_adv = ADVERTISE_PAUSE_CAP;
-				remote_adv = 0;
-				if (flags & MR_LP_ADV_SYM_PAUSE)
-  					remote_adv |= LPA_PAUSE_CAP;
-				if (flags & MR_LP_ADV_ASYM_PAUSE)
-					remote_adv |= LPA_PAUSE_ASYM;
+	mac_status = tr32(MAC_STATUS);
 
-				tg3_setup_flow_control(tp, local_adv, remote_adv);
-
-				tp->tg3_flags |=
-					TG3_FLAG_GOT_SERDES_FLOWCTL;
-				current_link_up = 1;
-			}
-			for (i = 0; i < 60; i++) {
-				udelay(20);
-				tw32_f(MAC_STATUS,
-				     (MAC_STATUS_SYNC_CHANGED |
-				      MAC_STATUS_CFG_CHANGED));
-				udelay(40);
-				if ((tr32(MAC_STATUS) &
-				     (MAC_STATUS_SYNC_CHANGED |
-				      MAC_STATUS_CFG_CHANGED)) == 0)
-					break;
-			}
-			if (current_link_up == 0 &&
-			    (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED)) {
-				current_link_up = 1;
-			}
-		} else {
-			/* Forcing 1000FD link up. */
-			current_link_up = 1;
-			tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
-		}
-	} else
-		tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
+	if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
+		current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
+	else
+		current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
 
 	tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
 	tw32_f(MAC_MODE, tp->mac_mode);
@@ -2158,19 +2306,24 @@
 		 (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
 
 	for (i = 0; i < 100; i++) {
-		udelay(20);
-		tw32_f(MAC_STATUS,
-		     (MAC_STATUS_SYNC_CHANGED |
-		      MAC_STATUS_CFG_CHANGED));
-		udelay(40);
-		if ((tr32(MAC_STATUS) &
-		     (MAC_STATUS_SYNC_CHANGED |
-		      MAC_STATUS_CFG_CHANGED)) == 0)
+		tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
+				    MAC_STATUS_CFG_CHANGED));
+		udelay(5);
+		if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
+					 MAC_STATUS_CFG_CHANGED)) == 0)
 			break;
 	}
 
-	if ((tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) == 0)
+	mac_status = tr32(MAC_STATUS);
+	if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
 		current_link_up = 0;
+		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
+			tw32_f(MAC_MODE, (tp->mac_mode |
+					  MAC_MODE_SEND_CONFIGS));
+			udelay(1);
+			tw32_f(MAC_MODE, tp->mac_mode);
+		}
+	}
 
 	if (current_link_up == 1) {
 		tp->link_config.active_speed = SPEED_1000;
@@ -2202,15 +2355,6 @@
 			tg3_link_report(tp);
 	}
 
-	if ((tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) == 0) {
-		tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_LINK_POLARITY);
-		udelay(40);
-		if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
-			tw32_f(MAC_MODE, tp->mac_mode);
-			udelay(40);
-		}
-	}
-
 	return 0;
 }
 
@@ -2218,7 +2362,7 @@
 {
 	int err;
 
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		err = tg3_setup_fiber_phy(tp, force_reset);
 	} else {
 		err = tg3_setup_copper_phy(tp, force_reset);
@@ -2736,11 +2880,11 @@
 	tg3_halt(tp);
 	tg3_init_hw(tp);
 
+	tg3_netif_start(tp);
+
 	spin_unlock(&tp->tx_lock);
 	spin_unlock_irq(&tp->lock);
 
-	tg3_netif_start(tp);
-
 	if (restart_timer)
 		mod_timer(&tp->timer, jiffies + 1);
 }
@@ -3100,9 +3244,10 @@
 
 	tg3_init_hw(tp);
 
+	tg3_netif_start(tp);
+
 	spin_unlock(&tp->tx_lock);
 	spin_unlock_irq(&tp->lock);
-	tg3_netif_start(tp);
 
 	return 0;
 }
@@ -3593,6 +3738,8 @@
 	}
 }
 
+static void tg3_stop_fw(struct tg3 *);
+
 /* tp->lock is held. */
 static int tg3_chip_reset(struct tg3 *tp)
 {
@@ -3695,6 +3842,11 @@
 
 	tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
 
+	if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
+		tg3_stop_fw(tp);
+		tw32(0x5000, 0x400);
+	}
+
 	tw32(GRC_MODE, tp->grc_mode);
 
 	if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
@@ -3711,7 +3863,7 @@
 		tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
 	}
 
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
 		tw32_f(MAC_MODE, tp->mac_mode);
 	} else
@@ -5241,14 +5393,14 @@
 	tw32(MAC_LED_CTRL, tp->led_ctrl);
 
 	tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
 		udelay(10);
 	}
 	tw32_f(MAC_RX_MODE, tp->rx_mode);
 	udelay(10);
 
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
 			/* Set drive transmission level to 1.2V  */
 			val = tr32(MAC_SERDES_CFG);
@@ -5266,22 +5418,8 @@
 	tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
 
 	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
-	    tp->phy_id == PHY_ID_SERDES) {
-		/* Enable hardware link auto-negotiation */
-		u32 digctrl, txctrl;
-
-		digctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_CRC16_CLEAR_N |
-		    SG_DIG_LOCAL_DUPLEX_STATUS | SG_DIG_LOCAL_LINK_STATUS |
-		    (2 << SG_DIG_SPEED_STATUS_SHIFT) | SG_DIG_FIBER_MODE |
-		    SG_DIG_GBIC_ENABLE;
-
-		txctrl = tr32(MAC_SERDES_CFG);
-		tw32_f(MAC_SERDES_CFG, txctrl | MAC_SERDES_CFG_EDGE_SELECT);
-		tw32_f(SG_DIG_CTRL, digctrl | SG_DIG_SOFT_RESET);
-		tr32(SG_DIG_CTRL);
-		udelay(5);
-		tw32_f(SG_DIG_CTRL, digctrl);
-
+	    (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
+		/* Use hardware link auto-negotiation */
 		tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
 	}
 
@@ -5289,7 +5427,7 @@
 	if (err)
 		return err;
 
-	if (tp->phy_id != PHY_ID_SERDES) {
+	if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
 		u32 tmp;
 
 		/* Clear CRC stats. */
@@ -5481,7 +5619,8 @@
 				need_setup = 1;
 			}
 			if (! netif_carrier_ok(tp->dev) &&
-			    (mac_stat & MAC_STATUS_PCS_SYNCED)) {
+			    (mac_stat & (MAC_STATUS_PCS_SYNCED |
+					 MAC_STATUS_SIGNAL_DET))) {
 				need_setup = 1;
 			}
 			if (need_setup) {
@@ -5877,7 +6016,7 @@
 {
 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
 
-	if (tp->phy_id != PHY_ID_SERDES &&
+	if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
 	    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
 	     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
 		unsigned long flags;
@@ -6308,7 +6447,7 @@
 		cmd->supported |= (SUPPORTED_1000baseT_Half |
 				   SUPPORTED_1000baseT_Full);
 
-	if (tp->phy_id != PHY_ID_SERDES)
+	if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES))
 		cmd->supported |= (SUPPORTED_100baseT_Half |
 				  SUPPORTED_100baseT_Full |
 				  SUPPORTED_10baseT_Half |
@@ -6337,7 +6476,7 @@
 	    tp->link_config.phy_is_low_power)
 		return -EAGAIN;
 
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		/* These are the only valid advertisement bits allowed.  */
 		if (cmd->autoneg == AUTONEG_ENABLE &&
 		    (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
@@ -6395,7 +6534,7 @@
 	if (wol->wolopts & ~WAKE_MAGIC)
 		return -EINVAL;
 	if ((wol->wolopts & WAKE_MAGIC) &&
-	    tp->phy_id == PHY_ID_SERDES &&
+	    tp->tg3_flags2 & TG3_FLG2_PHY_SERDES &&
 	    !(tp->tg3_flags & TG3_FLAG_SERDES_WOL_CAP))
 		return -EINVAL;
   
@@ -6491,10 +6630,9 @@
 
 	tg3_halt(tp);
 	tg3_init_hw(tp);
-	netif_wake_queue(tp->dev);
+	tg3_netif_start(tp);
 	spin_unlock(&tp->tx_lock);
 	spin_unlock_irq(&tp->lock);
-	tg3_netif_start(tp);
   
 	return 0;
 }
@@ -6529,9 +6667,9 @@
 		tp->tg3_flags &= ~TG3_FLAG_PAUSE_TX;
 	tg3_halt(tp);
 	tg3_init_hw(tp);
+	tg3_netif_start(tp);
 	spin_unlock(&tp->tx_lock);
 	spin_unlock_irq(&tp->lock);
-	tg3_netif_start(tp);
   
 	return 0;
 }
@@ -6618,7 +6756,7 @@
 	case SIOCGMIIREG: {
 		u32 mii_regval;
 
-		if (tp->phy_id == PHY_ID_SERDES)
+		if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
 			break;			/* We have no PHY */
 
 		spin_lock_irq(&tp->lock);
@@ -6631,7 +6769,7 @@
 	}
 
 	case SIOCSMIIREG:
-		if (tp->phy_id == PHY_ID_SERDES)
+		if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
 			break;			/* We have no PHY */
 
 		if (!capable(CAP_NET_ADMIN))
@@ -6868,10 +7006,10 @@
 	{ PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
-	{ PCI_VENDOR_ID_BROADCOM, 0x0003, PHY_ID_SERDES  }, /* BCM95700A9 */
+	{ PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },		    /* BCM95700A9 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
-	{ PCI_VENDOR_ID_BROADCOM, 0x0007, PHY_ID_SERDES  }, /* BCM95701A7 */
+	{ PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },		    /* BCM95701A7 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
 	{ PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
@@ -6880,7 +7018,7 @@
 	/* 3com boards. */
 	{ PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
 	{ PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
-	{ PCI_VENDOR_ID_3COM, 0x1004, PHY_ID_SERDES  }, /* 3C996SX */
+	{ PCI_VENDOR_ID_3COM, 0x1004, 0 },		/* 3C996SX */
 	{ PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
 	{ PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
 
@@ -6893,37 +7031,43 @@
 	/* Compaq boards. */
 	{ PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
 	{ PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
-	{ PCI_VENDOR_ID_COMPAQ, 0x007d, PHY_ID_SERDES  }, /* CHANGELING */
+	{ PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },		  /* CHANGELING */
 	{ PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
 	{ PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
 
 	/* IBM boards. */
-	{ PCI_VENDOR_ID_IBM, 0x0281, PHY_ID_SERDES } /* IBM??? */
+	{ PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
 };
 
-static int __devinit tg3_phy_probe(struct tg3 *tp)
+static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
 {
-	u32 eeprom_phy_id, hw_phy_id_1, hw_phy_id_2;
-	u32 hw_phy_id, hw_phy_id_masked;
-	u32 val;
-	int i, eeprom_signature_found, err;
+	int i;
 
-	tp->phy_id = PHY_ID_INVALID;
 	for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
 		if ((subsys_id_to_phy_id[i].subsys_vendor ==
 		     tp->pdev->subsystem_vendor) &&
 		    (subsys_id_to_phy_id[i].subsys_devid ==
-		     tp->pdev->subsystem_device)) {
-			tp->phy_id = subsys_id_to_phy_id[i].phy_id;
-			break;
-		}
+		     tp->pdev->subsystem_device))
+			return &subsys_id_to_phy_id[i];
 	}
+	return NULL;
+}
+
+static int __devinit tg3_phy_probe(struct tg3 *tp)
+{
+	u32 eeprom_phy_id, hw_phy_id_1, hw_phy_id_2;
+	u32 hw_phy_id, hw_phy_id_masked;
+	u32 val;
+	int eeprom_signature_found, eeprom_phy_serdes, err;
 
+	tp->phy_id = PHY_ID_INVALID;
 	eeprom_phy_id = PHY_ID_INVALID;
+	eeprom_phy_serdes = 0;
 	eeprom_signature_found = 0;
 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
 		u32 nic_cfg, led_cfg;
+		u32 nic_phy_id, cfg2;
 
 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
 		tp->nic_sram_data_cfg = nic_cfg;
@@ -6931,21 +7075,19 @@
 		eeprom_signature_found = 1;
 
 		if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
-		    NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER) {
-			eeprom_phy_id = PHY_ID_SERDES;
-		} else {
-			u32 nic_phy_id;
+		    NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
+			eeprom_phy_serdes = 1;
 
-			tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
-			if (nic_phy_id != 0) {
-				u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
-				u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
-
-				eeprom_phy_id  = (id1 >> 16) << 10;
-				eeprom_phy_id |= (id2 & 0xfc00) << 16;
-				eeprom_phy_id |= (id2 & 0x03ff) <<  0;
-			}
-		}
+		tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
+		if (nic_phy_id != 0) {
+			u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
+			u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
+
+			eeprom_phy_id  = (id1 >> 16) << 10;
+			eeprom_phy_id |= (id2 & 0xfc00) << 16;
+			eeprom_phy_id |= (id2 & 0x03ff) <<  0;
+		} else
+			eeprom_phy_id = 0;
 
 		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &led_cfg);
@@ -7007,6 +7149,10 @@
 		}
 		if (nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL)
 			tp->tg3_flags |= TG3_FLAG_SERDES_WOL_CAP;
+
+		tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &cfg2);
+		if (cfg2 & (1 << 17))
+			tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
 	}
 
 	/* Reading the PHY ID register can conflict with ASF
@@ -7033,20 +7179,31 @@
 
 	if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
 		tp->phy_id = hw_phy_id;
+		if (hw_phy_id_masked == PHY_ID_BCM8002)
+			tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
 	} else {
-		/* phy_id currently holds the value found in the
-		 * subsys_id_to_phy_id[] table or PHY_ID_INVALID
-		 * if a match was not found there.
-		 */
-		if (tp->phy_id == PHY_ID_INVALID) {
-			if (!eeprom_signature_found ||
-			    !KNOWN_PHY_ID(eeprom_phy_id & PHY_ID_MASK))
-				return -ENODEV;
+		if (eeprom_signature_found) {
 			tp->phy_id = eeprom_phy_id;
+			if (eeprom_phy_serdes)
+				tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
+		} else {
+			struct subsys_tbl_ent *p;
+
+			/* No eeprom signature?  Try the hardcoded
+			 * subsys device table.
+			 */
+			p = lookup_by_subsys(tp);
+			if (!p)
+				return -ENODEV;
+
+			tp->phy_id = p->phy_id;
+			if (!tp->phy_id ||
+			    tp->phy_id == PHY_ID_BCM8002)
+				tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
 		}
 	}
 
-	if (tp->phy_id != PHY_ID_SERDES &&
+	if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
 	    !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
 		u32 bmsr, adv_reg, tg3_ctrl;
 
@@ -7103,7 +7260,7 @@
 	if (!eeprom_signature_found)
 		tp->led_ctrl = LED_CTRL_MODE_PHY_1;
 
-	if (tp->phy_id == PHY_ID_SERDES)
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
 		tp->link_config.advertising =
 			(ADVERTISED_1000baseT_Half |
 			 ADVERTISED_1000baseT_Full |
@@ -7490,12 +7647,14 @@
 	grc_misc_cfg = tr32(GRC_MISC_CFG);
 	grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
 
+	/* Broadcom's driver says that CIOBE multisplit has a bug */
+#if 0
 	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
 	    grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5704CIOBE) {
 		tp->tg3_flags |= TG3_FLAG_SPLIT_MODE;
 		tp->split_mode_max_reqs = SPLIT_MODE_5704_MAX_REQ;
 	}
-
+#endif
 	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
 	    (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
 	     grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
@@ -7522,7 +7681,7 @@
 
 	tg3_read_partno(tp);
 
-	if (tp->phy_id == PHY_ID_SERDES) {
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
 		tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
 	} else {
 		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
@@ -7545,13 +7704,13 @@
 	 * upon subsystem IDs.
 	 */
 	if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
-	    tp->phy_id != PHY_ID_SERDES) {
+	    !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
 		tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
 				  TG3_FLAG_USE_LINKCHG_REG);
 	}
 
 	/* For all SERDES we poll the MAC status register. */
-	if (tp->phy_id == PHY_ID_SERDES)
+	if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
 		tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
 	else
 		tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
@@ -7987,8 +8146,8 @@
 	case PHY_ID_BCM5704:	return "5704";
 	case PHY_ID_BCM5705:	return "5705";
 	case PHY_ID_BCM5750:	return "5750";
-	case PHY_ID_BCM8002:	return "8002";
-	case PHY_ID_SERDES:	return "serdes";
+	case PHY_ID_BCM8002:	return "8002/serdes";
+	case 0:			return "serdes";
 	default:		return "unknown";
 	};
 }
@@ -8363,11 +8522,11 @@
 		tp->timer.expires = jiffies + tp->timer_offset;
 		add_timer(&tp->timer);
 
-		spin_unlock(&tp->tx_lock);
-		spin_unlock_irq(&tp->lock);
-
 		netif_device_attach(dev);
 		tg3_netif_start(tp);
+
+		spin_unlock(&tp->tx_lock);
+		spin_unlock_irq(&tp->lock);
 	}
 
 	return err;
@@ -8400,10 +8559,10 @@
 
 	tg3_enable_ints(tp);
 
+	tg3_netif_start(tp);
+
 	spin_unlock(&tp->tx_lock);
 	spin_unlock_irq(&tp->lock);
-
-	tg3_netif_start(tp);
 
 	return 0;
 }
diff -Nru a/drivers/net/tg3.h b/drivers/net/tg3.h
--- a/drivers/net/tg3.h	2004-07-02 09:04:40 -07:00
+++ b/drivers/net/tg3.h	2004-08-30 15:25:29 -07:00
@@ -124,6 +124,7 @@
 #define  CHIPREV_ID_5705_A3		 0x3003
 #define  CHIPREV_ID_5750_A0		 0x4000
 #define  CHIPREV_ID_5750_A1		 0x4001
+#define  CHIPREV_ID_5750_A3		 0x4003
 #define  GET_ASIC_REV(CHIP_REV_ID)	((CHIP_REV_ID) >> 12)
 #define   ASIC_REV_5700			 0x07
 #define   ASIC_REV_5701			 0x00
@@ -2089,6 +2090,9 @@
 #define TG3_FLG2_PCI_EXPRESS		0x00000200
 #define TG3_FLG2_ASF_NEW_HANDSHAKE	0x00000400
 #define TG3_FLG2_HW_AUTONEG		0x00000800
+#define TG3_FLG2_PHY_JUST_INITTED	0x00001000
+#define TG3_FLG2_PHY_SERDES		0x00002000
+#define TG3_FLG2_CAPACITIVE_COUPLING	0x00004000
 
 	u32				split_mode_max_reqs;
 #define SPLIT_MODE_5704_MAX_REQ		3
@@ -2136,7 +2140,6 @@
 #define PHY_ID_BCM5705			0x600081a0
 #define PHY_ID_BCM5750			0x60008180
 #define PHY_ID_BCM8002			0x60010140
-#define PHY_ID_SERDES			0xfeedbee0
 #define PHY_ID_INVALID			0xffffffff
 #define PHY_ID_REV_MASK			0x0000000f
 #define PHY_REV_BCM5401_B0		0x1
@@ -2159,7 +2162,7 @@
 	 (X) == PHY_ID_BCM5411 || (X) == PHY_ID_BCM5701 || \
 	 (X) == PHY_ID_BCM5703 || (X) == PHY_ID_BCM5704 || \
 	 (X) == PHY_ID_BCM5705 || (X) == PHY_ID_BCM5750 || \
-	 (X) == PHY_ID_BCM8002 || (X) == PHY_ID_SERDES)
+	 (X) == PHY_ID_BCM8002)
 
 	struct tg3_hw_stats		*hw_stats;
 	dma_addr_t			stats_mapping;
diff -Nru a/drivers/net/wan/dscc4.c b/drivers/net/wan/dscc4.c
--- a/drivers/net/wan/dscc4.c	2003-10-04 06:02:06 -07:00
+++ b/drivers/net/wan/dscc4.c	2004-08-26 11:18:53 -07:00
@@ -351,8 +351,8 @@
 #endif
 
 /* Functions prototypes */
-static inline void dscc4_rx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
-static inline void dscc4_tx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
+static void dscc4_rx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
+static void dscc4_tx_irq(struct dscc4_pci_priv *, struct dscc4_dev_priv *);
 static int dscc4_found1(struct pci_dev *, unsigned long ioaddr);
 static int dscc4_init_one(struct pci_dev *, const struct pci_device_id *ent);
 static int dscc4_open(struct net_device *);
@@ -366,7 +366,6 @@
 static void dscc4_irq(int irq, void *dev_id, struct pt_regs *ptregs);
 static int dscc4_hdlc_attach(hdlc_device *, unsigned short, unsigned short);
 static int dscc4_set_iface(struct dscc4_dev_priv *, struct net_device *);
-static inline int dscc4_set_quartz(struct dscc4_dev_priv *, int);
 #ifdef DSCC4_POLLING
 static int dscc4_tx_poll(struct dscc4_dev_priv *, struct net_device *);
 #endif
@@ -857,6 +856,18 @@
 	//scc_writel(0x00250008 & ~RxActivate, dpriv, dev, CCR2);
 }
 
+static inline int dscc4_set_quartz(struct dscc4_dev_priv *dpriv, int hz)
+{
+	int ret = 0;
+
+	if ((hz < 0) || (hz > DSCC4_HZ_MAX))
+		ret = -EOPNOTSUPP;
+	else
+		dpriv->pci_priv->xtal_hz = hz;
+
+	return ret;
+}
+
 static int dscc4_found1(struct pci_dev *pdev, unsigned long ioaddr)
 {
 	struct dscc4_pci_priv *ppriv;
@@ -1325,18 +1336,6 @@
 	return ret;
 }
 
-static inline int dscc4_set_quartz(struct dscc4_dev_priv *dpriv, int hz)
-{
-	int ret = 0;
-
-	if ((hz < 0) || (hz > DSCC4_HZ_MAX))
-		ret = -EOPNOTSUPP;
-	else
-		dpriv->pci_priv->xtal_hz = hz;
-
-	return ret;
-}
-
 static int dscc4_match(struct thingie *p, int value)
 {
 	int i;
@@ -1512,7 +1511,7 @@
 	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
-static inline void dscc4_tx_irq(struct dscc4_pci_priv *ppriv,
+static void dscc4_tx_irq(struct dscc4_pci_priv *ppriv,
 				struct dscc4_dev_priv *dpriv)
 {
 	struct net_device *dev = hdlc_to_dev(&dpriv->hdlc);
@@ -1681,7 +1680,7 @@
 	goto try;
 }
 
-static inline void dscc4_rx_irq(struct dscc4_pci_priv *priv,
+static void dscc4_rx_irq(struct dscc4_pci_priv *priv,
 				    struct dscc4_dev_priv *dpriv)
 {
 	struct net_device *dev = hdlc_to_dev(&dpriv->hdlc);
diff -Nru a/drivers/net/wan/lmc/lmc_debug.c b/drivers/net/wan/lmc/lmc_debug.c
--- a/drivers/net/wan/lmc/lmc_debug.c	2002-03-11 06:13:28 -08:00
+++ b/drivers/net/wan/lmc/lmc_debug.c	2004-08-26 09:59:47 -07:00
@@ -66,7 +66,7 @@
 #endif
 }
 
-inline void lmc_trace(struct net_device *dev, char *msg){
+void lmc_trace(struct net_device *dev, char *msg){
 #ifdef LMC_TRACE
     unsigned long j = jiffies + 3; /* Wait for 50 ms */
 
diff -Nru a/drivers/net/wan/lmc/lmc_debug.h b/drivers/net/wan/lmc/lmc_debug.h
--- a/drivers/net/wan/lmc/lmc_debug.h	2002-02-05 09:40:01 -08:00
+++ b/drivers/net/wan/lmc/lmc_debug.h	2004-08-26 09:58:32 -07:00
@@ -47,6 +47,6 @@
 
 void lmcConsoleLog(char *type, unsigned char *ucData, int iLen);
 void lmcEventLog (u_int32_t EventNum, u_int32_t arg2, u_int32_t arg3);
-inline void lmc_trace(struct net_device *dev, char *msg);
+void lmc_trace(struct net_device *dev, char *msg);
 
 #endif
diff -Nru a/drivers/pci/pci.c b/drivers/pci/pci.c
--- a/drivers/pci/pci.c	2004-06-28 15:47:57 -07:00
+++ b/drivers/pci/pci.c	2004-08-21 15:04:58 -07:00
@@ -1238,6 +1238,8 @@
 	 * Allocate a new bus, and inherit stuff from the parent..
 	 */
 	child = pci_alloc_bus();
+	if (!child)
+		return NULL;
 
 	list_add_tail(&child->node, &parent->children);
 	child->self = dev;
@@ -1291,7 +1293,11 @@
 		 */
 		if (pass)
 			return max;
+
 		child = pci_add_new_bus(bus, dev, 0);
+		if (!child)
+			return max;
+
 		child->primary = buses & 0xFF;
 		child->secondary = (buses >> 8) & 0xFF;
 		child->subordinate = (buses >> 16) & 0xFF;
@@ -1316,6 +1322,9 @@
 		pci_write_config_word(dev, PCI_STATUS, 0xffff);
 
 		child = pci_add_new_bus(bus, dev, ++max);
+		if (!child)
+			return max;
+
 		buses = (buses & 0xff000000)
 		      | ((unsigned int)(child->primary)     <<  0)
 		      | ((unsigned int)(child->secondary)   <<  8)
diff -Nru a/drivers/scsi/53c7,8xx.c b/drivers/scsi/53c7,8xx.c
--- a/drivers/scsi/53c7,8xx.c	2003-09-02 11:27:25 -07:00
+++ b/drivers/scsi/53c7,8xx.c	2004-08-26 16:50:38 -07:00
@@ -3926,7 +3926,7 @@
 	restore_flags (flags);
 	cmd->result = le32_to_cpu(0xffff);	/* The NCR will overwrite message
 				       and status with valid data */
-	cmd->host_scribble = (unsigned char *) tmp = create_cmd (cmd);
+	cmd->host_scribble = (unsigned char *) create_cmd (cmd);
     }
     cli();
     /*
diff -Nru a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
--- a/drivers/scsi/advansys.c	2004-02-01 07:56:53 -08:00
+++ b/drivers/scsi/advansys.c	2004-08-26 16:50:38 -07:00
@@ -6118,8 +6118,8 @@
     } else {
         /* Append to 'done_scp' at the end with 'last_scp'. */
         ASC_ASSERT(last_scp != NULL);
-        REQPNEXT(last_scp) = asc_dequeue_list(&boardp->active,
-            &new_last_scp, ASC_TID_ALL);
+        last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->active, &new_last_scp, ASC_TID_ALL);
         if (new_last_scp != NULL) {
             ASC_ASSERT(REQPNEXT(last_scp) != NULL);
             for (tscp = REQPNEXT(last_scp); tscp; tscp = REQPNEXT(tscp)) {
@@ -6141,8 +6141,8 @@
     } else {
         /* Append to 'done_scp' at the end with 'last_scp'. */
         ASC_ASSERT(last_scp != NULL);
-        REQPNEXT(last_scp) = asc_dequeue_list(&boardp->waiting,
-            &new_last_scp, ASC_TID_ALL);
+        last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->waiting, &new_last_scp, ASC_TID_ALL);
         if (new_last_scp != NULL) {
             ASC_ASSERT(REQPNEXT(last_scp) != NULL);
             for (tscp = REQPNEXT(last_scp); tscp; tscp = REQPNEXT(tscp)) {
@@ -6394,8 +6394,8 @@
                     ASC_TID_ALL);
             } else {
                 ASC_ASSERT(last_scp != NULL);
-                REQPNEXT(last_scp) = asc_dequeue_list(&boardp->done,
-                    &new_last_scp, ASC_TID_ALL);
+                last_scp->host_scribble = (unsigned char *)asc_dequeue_list(
+			&boardp->done, &new_last_scp, ASC_TID_ALL);
                 if (new_last_scp != NULL) {
                     ASC_ASSERT(REQPNEXT(last_scp) != NULL);
                     last_scp = new_last_scp;
@@ -6466,7 +6466,7 @@
     while (scp != NULL) {
         ASC_DBG1(3, "asc_scsi_done_list: scp 0x%lx\n", (ulong) scp);
         tscp = REQPNEXT(scp);
-        REQPNEXT(scp) = NULL;
+        scp->host_scribble = NULL;
         ASC_STATS(scp->host, done);
         ASC_ASSERT(scp->scsi_done != NULL);
         scp->scsi_done(scp);
@@ -7511,7 +7511,7 @@
     tid = REQPTID(reqp);
     ASC_ASSERT(tid >= 0 && tid <= ADV_MAX_TID);
     if (flag == ASC_FRONT) {
-        REQPNEXT(reqp) = ascq->q_first[tid];
+        reqp->host_scribble = (unsigned char *)ascq->q_first[tid];
         ascq->q_first[tid] = reqp;
         /* If the queue was empty, set the last pointer. */
         if (ascq->q_last[tid] == NULL) {
@@ -7519,10 +7519,10 @@
         }
     } else { /* ASC_BACK */
         if (ascq->q_last[tid] != NULL) {
-            REQPNEXT(ascq->q_last[tid]) = reqp;
+            ascq->q_last[tid]->host_scribble = (unsigned char *)reqp;
         }
         ascq->q_last[tid] = reqp;
-        REQPNEXT(reqp) = NULL;
+        reqp->host_scribble = NULL;
         /* If the queue was empty, set the first pointer. */
         if (ascq->q_first[tid] == NULL) {
             ascq->q_first[tid] = reqp;
@@ -7643,7 +7643,7 @@
                     lastp = ascq->q_last[i];
                 } else {
                     ASC_ASSERT(lastp != NULL);
-                    REQPNEXT(lastp) = ascq->q_first[i];
+                    lastp->host_scribble = (unsigned char *)ascq->q_first[i];
                     lastp = ascq->q_last[i];
                 }
                 ascq->q_first[i] = ascq->q_last[i] = NULL;
@@ -7721,8 +7721,8 @@
              currp; prevp = currp, currp = REQPNEXT(currp)) {
             if (currp == reqp) {
                 ret = ASC_TRUE;
-                REQPNEXT(prevp) = REQPNEXT(currp);
-                REQPNEXT(reqp) = NULL;
+                prevp->host_scribble = (unsigned char *)REQPNEXT(currp);
+                reqp->host_scribble = NULL;
                 if (ascq->q_last[tid] == reqp) {
                     ascq->q_last[tid] = prevp;
                 }
diff -Nru a/drivers/scsi/ata_piix.c b/drivers/scsi/ata_piix.c
--- a/drivers/scsi/ata_piix.c	2004-08-14 12:38:51 -07:00
+++ b/drivers/scsi/ata_piix.c	2004-08-30 11:43:40 -07:00
@@ -390,7 +390,7 @@
 
 static void piix_set_piomode (struct ata_port *ap, struct ata_device *adev)
 {
-	unsigned int pio	= adev->pio_mode;
+	unsigned int pio	= adev->pio_mode - XFER_PIO_0;
 	struct pci_dev *dev	= ap->host_set->pdev;
 	unsigned int is_slave	= (adev->devno != 0);
 	unsigned int master_port= ap->port_no ? 0x42 : 0x40;
diff -Nru a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
--- a/drivers/scsi/dpt_i2o.c	2003-03-31 05:20:33 -08:00
+++ b/drivers/scsi/dpt_i2o.c	2004-08-26 16:50:38 -07:00
@@ -434,7 +434,7 @@
 			cmd->scsi_done(cmd);
 			return 0;
 		}
-		(struct adpt_device*)(cmd->device->hostdata) = pDev;
+		cmd->device->hostdata = pDev;
 	}
 	pDev->pScsi_dev = cmd->device;
 
@@ -2194,7 +2194,7 @@
 		printk ("%s: scsi_register returned NULL\n",pHba->name);
 		return -1;
 	}
-	(adpt_hba*)(host->hostdata[0]) = pHba;
+	host->hostdata[0] = (unsigned long)pHba;
 	pHba->host = host;
 
 	host->irq = pHba->pDev->irq;;
diff -Nru a/drivers/scsi/ips.c b/drivers/scsi/ips.c
--- a/drivers/scsi/ips.c	2004-08-16 14:09:52 -07:00
+++ b/drivers/scsi/ips.c	2004-08-20 00:18:04 -07:00
@@ -133,6 +133,10 @@
 /* 6.10.00  - Remove 1G Addressing Limitations                               */
 /* 6.11.xx  - Get VersionInfo buffer off the stack !              DDTS 60401 */
 /* 6.11.xx  - Make Logical Drive Info structure safe for DMA      DDTS 60639 */
+/* 7.10.xx  - Add highmem_io flag in SCSI Templete for 2.4 kernels           */
+/*          - Fix path/name for scsi_hosts.h include for 2.6 kernels         */
+/*          - Fix sort order of 7k                                           */
+/*          - Remove 3 unused "inline" functions                             */
 /*****************************************************************************/
 
 /*
@@ -176,7 +180,13 @@
 #include <scsi/sg.h>
 
 #include "scsi.h"
+
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(2,5,0)
 #include "hosts.h"
+#else
+#include <scsi/scsi_host.h>
+#endif
+
 #include "ips.h"
 
 #include <linux/module.h>
@@ -197,8 +207,8 @@
 /*
  * DRIVER_VER
  */
-#define IPS_VERSION_HIGH        "7.00"
-#define IPS_VERSION_LOW         ".15 "
+#define IPS_VERSION_HIGH        "7.10"
+#define IPS_VERSION_LOW         ".18 "
 
 #if !defined(__i386__) && !defined(__ia64__) && !defined(__x86_64__)
 #warning "This driver has only been tested on the x86/ia64/x86_64 platforms"
@@ -277,9 +287,9 @@
 	.use_clustering		= ENABLE_CLUSTERING,
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
 	.use_new_eh_code	= 1,
-#ifdef RHEL3
-    .vary_io            = 1,
 #endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,20)  &&  LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
+    .highmem_io          = 1,   
 #endif
 };
 
@@ -477,21 +487,17 @@
 static uint32_t ips_statupd_copperhead_memio(ips_ha_t *);
 static uint32_t ips_statupd_morpheus(ips_ha_t *);
 static ips_scb_t *ips_getscb(ips_ha_t *);
-static inline void ips_putq_scb_head(ips_scb_queue_t *, ips_scb_t *);
-static inline void ips_putq_scb_tail(ips_scb_queue_t *, ips_scb_t *);
-static inline void ips_putq_wait_head(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline void ips_putq_wait_tail(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline void ips_putq_copp_head(ips_copp_queue_t *,
-				      ips_copp_wait_item_t *);
-static inline void ips_putq_copp_tail(ips_copp_queue_t *,
+static void ips_putq_scb_head(ips_scb_queue_t *, ips_scb_t *);
+static void ips_putq_wait_tail(ips_wait_queue_t *, Scsi_Cmnd *);
+static void ips_putq_copp_tail(ips_copp_queue_t *,
 				      ips_copp_wait_item_t *);
-static inline ips_scb_t *ips_removeq_scb_head(ips_scb_queue_t *);
-static inline ips_scb_t *ips_removeq_scb(ips_scb_queue_t *, ips_scb_t *);
-static inline Scsi_Cmnd *ips_removeq_wait_head(ips_wait_queue_t *);
-static inline Scsi_Cmnd *ips_removeq_wait(ips_wait_queue_t *, Scsi_Cmnd *);
-static inline ips_copp_wait_item_t *ips_removeq_copp(ips_copp_queue_t *,
+static ips_scb_t *ips_removeq_scb_head(ips_scb_queue_t *);
+static ips_scb_t *ips_removeq_scb(ips_scb_queue_t *, ips_scb_t *);
+static Scsi_Cmnd *ips_removeq_wait_head(ips_wait_queue_t *);
+static Scsi_Cmnd *ips_removeq_wait(ips_wait_queue_t *, Scsi_Cmnd *);
+static ips_copp_wait_item_t *ips_removeq_copp(ips_copp_queue_t *,
 						     ips_copp_wait_item_t *);
-static inline ips_copp_wait_item_t *ips_removeq_copp_head(ips_copp_queue_t *);
+static ips_copp_wait_item_t *ips_removeq_copp_head(ips_copp_queue_t *);
 
 static int ips_is_passthru(Scsi_Cmnd *);
 static int ips_make_passthru(ips_ha_t *, Scsi_Cmnd *, ips_scb_t *, int);
@@ -802,168 +808,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_removeq_copp_head                                      */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_copp_wait_item_t *
-ips_removeq_copp_head(ips_copp_queue_t * queue)
-{
-	ips_copp_wait_item_t *item;
-
-	METHOD_TRACE("ips_removeq_copp_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = item->next;
-	item->next = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_copp                                           */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_copp_wait_item_t *
-ips_removeq_copp(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
-{
-	ips_copp_wait_item_t *p;
-
-	METHOD_TRACE("ips_removeq_copp", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_copp_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != p->next))
-		p = p->next;
-
-	if (p) {
-		/* found a match */
-		p->next = item->next;
-
-		if (!item->next)
-			queue->tail = p;
-
-		item->next = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_wait_head                                      */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline Scsi_Cmnd *
-ips_removeq_wait_head(ips_wait_queue_t * queue)
-{
-	Scsi_Cmnd *item;
-
-	METHOD_TRACE("ips_removeq_wait_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = (Scsi_Cmnd *) item->host_scribble;
-	item->host_scribble = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_wait                                           */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline Scsi_Cmnd *
-ips_removeq_wait(ips_wait_queue_t * queue, Scsi_Cmnd * item)
-{
-	Scsi_Cmnd *p;
-
-	METHOD_TRACE("ips_removeq_wait", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_wait_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != (Scsi_Cmnd *) p->host_scribble))
-		p = (Scsi_Cmnd *) p->host_scribble;
-
-	if (p) {
-		/* found a match */
-		p->host_scribble = item->host_scribble;
-
-		if (!item->host_scribble)
-			queue->tail = p;
-
-		item->host_scribble = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_eh_abort                                               */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -1021,87 +865,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_removeq_scb_head                                       */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove the head of the queue                                           */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_scb_t *
-ips_removeq_scb_head(ips_scb_queue_t * queue)
-{
-	ips_scb_t *item;
-
-	METHOD_TRACE("ips_removeq_scb_head", 1);
-
-	item = queue->head;
-
-	if (!item) {
-		return (NULL);
-	}
-
-	queue->head = item->q_next;
-	item->q_next = NULL;
-
-	if (queue->tail == item)
-		queue->tail = NULL;
-
-	queue->count--;
-
-	return (item);
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_removeq_scb                                            */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Remove an item from a queue                                            */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline ips_scb_t *
-ips_removeq_scb(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	ips_scb_t *p;
-
-	METHOD_TRACE("ips_removeq_scb", 1);
-
-	if (!item)
-		return (NULL);
-
-	if (item == queue->head) {
-		return (ips_removeq_scb_head(queue));
-	}
-
-	p = queue->head;
-
-	while ((p) && (item != p->q_next))
-		p = p->q_next;
-
-	if (p) {
-		/* found a match */
-		p->q_next = item->q_next;
-
-		if (!item->q_next)
-			queue->tail = p;
-
-		item->q_next = NULL;
-		queue->count--;
-
-		return (item);
-	}
-
-	return (NULL);
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_eh_reset                                               */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -1301,70 +1064,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_copp_tail                                         */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_copp_tail(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
-{
-	METHOD_TRACE("ips_putq_copp_tail", 1);
-
-	if (!item)
-		return;
-
-	item->next = NULL;
-
-	if (queue->tail)
-		queue->tail->next = item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_putq_wait_tail                                         */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_wait_tail(ips_wait_queue_t * queue, Scsi_Cmnd * item)
-{
-	METHOD_TRACE("ips_putq_wait_tail", 1);
-
-	if (!item)
-		return;
-
-	item->host_scribble = NULL;
-
-	if (queue->tail)
-		queue->tail->host_scribble = (char *) item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_queue                                                  */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -2195,7 +1894,7 @@
 /*   Fill in a single scb sg_list element from an address                   */
 /*   return a -1 if a breakup occurred                                      */
 /****************************************************************************/
-static inline int
+static int
 ips_fill_scb_sg_single(ips_ha_t * ha, dma_addr_t busaddr,
 		       ips_scb_t * scb, int indx, unsigned int e_len)
 {
@@ -3000,66 +2699,6 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_scb_head                                          */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the head of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_scb_head(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	METHOD_TRACE("ips_putq_scb_head", 1);
-
-	if (!item)
-		return;
-
-	item->q_next = queue->head;
-	queue->head = item;
-
-	if (!queue->tail)
-		queue->tail = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
-/* Routine Name: ips_putq_scb_tail                                          */
-/*                                                                          */
-/* Routine Description:                                                     */
-/*                                                                          */
-/*   Add an item to the tail of the queue                                   */
-/*                                                                          */
-/* ASSUMED to be called from within the HA lock                             */
-/*                                                                          */
-/****************************************************************************/
-static inline void
-ips_putq_scb_tail(ips_scb_queue_t * queue, ips_scb_t * item)
-{
-	METHOD_TRACE("ips_putq_scb_tail", 1);
-
-	if (!item)
-		return;
-
-	item->q_next = NULL;
-
-	if (queue->tail)
-		queue->tail->q_next = item;
-
-	queue->tail = item;
-
-	if (!queue->head)
-		queue->head = item;
-
-	queue->count++;
-}
-
-/****************************************************************************/
-/*                                                                          */
 /* Routine Name: ips_next                                                   */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -3311,7 +2950,7 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_wait_head                                         */
+/* Routine Name: ips_putq_scb_head                                          */
 /*                                                                          */
 /* Routine Description:                                                     */
 /*                                                                          */
@@ -3320,15 +2959,15 @@
 /* ASSUMED to be called from within the HA lock                             */
 /*                                                                          */
 /****************************************************************************/
-static inline void
-ips_putq_wait_head(ips_wait_queue_t * queue, Scsi_Cmnd * item)
+static void
+ips_putq_scb_head(ips_scb_queue_t * queue, ips_scb_t * item)
 {
-	METHOD_TRACE("ips_putq_wait_head", 1);
+	METHOD_TRACE("ips_putq_scb_head", 1);
 
 	if (!item)
 		return;
 
-	item->host_scribble = (char *) queue->head;
+	item->q_next = queue->head;
 	queue->head = item;
 
 	if (!queue->tail)
@@ -3339,34 +2978,313 @@
 
 /****************************************************************************/
 /*                                                                          */
-/* Routine Name: ips_putq_copp_head                                         */
+/* Routine Name: ips_removeq_scb_head                                       */
 /*                                                                          */
 /* Routine Description:                                                     */
 /*                                                                          */
-/*   Add an item to the head of the queue                                   */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_scb_t *
+ips_removeq_scb_head(ips_scb_queue_t * queue)
+{
+	ips_scb_t *item;
+
+	METHOD_TRACE("ips_removeq_scb_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = item->q_next;
+	item->q_next = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_scb                                            */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_scb_t *
+ips_removeq_scb(ips_scb_queue_t * queue, ips_scb_t * item)
+{
+	ips_scb_t *p;
+
+	METHOD_TRACE("ips_removeq_scb", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_scb_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != p->q_next))
+		p = p->q_next;
+
+	if (p) {
+		/* found a match */
+		p->q_next = item->q_next;
+
+		if (!item->q_next)
+			queue->tail = p;
+
+		item->q_next = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_putq_wait_tail                                         */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Add an item to the tail of the queue                                   */
 /*                                                                          */
 /* ASSUMED to be called from within the HA lock                             */
 /*                                                                          */
 /****************************************************************************/
-static inline void
-ips_putq_copp_head(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+static void
+ips_putq_wait_tail(ips_wait_queue_t * queue, Scsi_Cmnd * item)
 {
-	METHOD_TRACE("ips_putq_copp_head", 1);
+	METHOD_TRACE("ips_putq_wait_tail", 1);
 
 	if (!item)
 		return;
 
-	item->next = queue->head;
-	queue->head = item;
+	item->host_scribble = NULL;
 
-	if (!queue->tail)
-		queue->tail = item;
+	if (queue->tail)
+		queue->tail->host_scribble = (char *) item;
+
+	queue->tail = item;
+
+	if (!queue->head)
+		queue->head = item;
 
 	queue->count++;
 }
 
 /****************************************************************************/
 /*                                                                          */
+/* Routine Name: ips_removeq_wait_head                                      */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static Scsi_Cmnd *
+ips_removeq_wait_head(ips_wait_queue_t * queue)
+{
+	Scsi_Cmnd *item;
+
+	METHOD_TRACE("ips_removeq_wait_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = (Scsi_Cmnd *) item->host_scribble;
+	item->host_scribble = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_wait                                           */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static Scsi_Cmnd *
+ips_removeq_wait(ips_wait_queue_t * queue, Scsi_Cmnd * item)
+{
+	Scsi_Cmnd *p;
+
+	METHOD_TRACE("ips_removeq_wait", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_wait_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != (Scsi_Cmnd *) p->host_scribble))
+		p = (Scsi_Cmnd *) p->host_scribble;
+
+	if (p) {
+		/* found a match */
+		p->host_scribble = item->host_scribble;
+
+		if (!item->host_scribble)
+			queue->tail = p;
+
+		item->host_scribble = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_putq_copp_tail                                         */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Add an item to the tail of the queue                                   */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static void
+ips_putq_copp_tail(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+{
+	METHOD_TRACE("ips_putq_copp_tail", 1);
+
+	if (!item)
+		return;
+
+	item->next = NULL;
+
+	if (queue->tail)
+		queue->tail->next = item;
+
+	queue->tail = item;
+
+	if (!queue->head)
+		queue->head = item;
+
+	queue->count++;
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_copp_head                                      */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove the head of the queue                                           */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_copp_wait_item_t *
+ips_removeq_copp_head(ips_copp_queue_t * queue)
+{
+	ips_copp_wait_item_t *item;
+
+	METHOD_TRACE("ips_removeq_copp_head", 1);
+
+	item = queue->head;
+
+	if (!item) {
+		return (NULL);
+	}
+
+	queue->head = item->next;
+	item->next = NULL;
+
+	if (queue->tail == item)
+		queue->tail = NULL;
+
+	queue->count--;
+
+	return (item);
+}
+
+/****************************************************************************/
+/*                                                                          */
+/* Routine Name: ips_removeq_copp                                           */
+/*                                                                          */
+/* Routine Description:                                                     */
+/*                                                                          */
+/*   Remove an item from a queue                                            */
+/*                                                                          */
+/* ASSUMED to be called from within the HA lock                             */
+/*                                                                          */
+/****************************************************************************/
+static ips_copp_wait_item_t *
+ips_removeq_copp(ips_copp_queue_t * queue, ips_copp_wait_item_t * item)
+{
+	ips_copp_wait_item_t *p;
+
+	METHOD_TRACE("ips_removeq_copp", 1);
+
+	if (!item)
+		return (NULL);
+
+	if (item == queue->head) {
+		return (ips_removeq_copp_head(queue));
+	}
+
+	p = queue->head;
+
+	while ((p) && (item != p->next))
+		p = p->next;
+
+	if (p) {
+		/* found a match */
+		p->next = item->next;
+
+		if (!item->next)
+			queue->tail = p;
+
+		item->next = NULL;
+		queue->count--;
+
+		return (item);
+	}
+
+	return (NULL);
+}
+
+/****************************************************************************/
+/*                                                                          */
 /* Routine Name: ipsintr_blocking                                           */
 /*                                                                          */
 /* Routine Description:                                                     */
@@ -4014,7 +3932,7 @@
 			scb->cmd.logical_info.reserved2 = 0;
 			scb->cmd.logical_info.reserved3 = 0;
 			scb->data_len = sizeof (IPS_LD_INFO);
-            scb->data_busaddr = ha->logical_drive_info_dma_addr;
+			scb->data_busaddr = ha->logical_drive_info_dma_addr;
 			scb->flags = 0;
 			scb->cmd.logical_info.buffer_addr = scb->data_busaddr;
 			ret = IPS_SUCCESS;
@@ -4340,7 +4258,7 @@
 
 	if ((scb->basic_status & IPS_GSC_STATUS_MASK) > 1) {
 		memset(ha->logical_drive_info, 0, sizeof (IPS_LD_INFO));
-        return (0);
+		return (0);
 	}
 
 	if (ha->logical_drive_info->drive_info[scb->target_id].state !=
@@ -4579,7 +4497,7 @@
 		if (ha->logical_drive_info) {
 			pci_free_consistent(ha->pcidev,
 					    sizeof (IPS_LD_INFO),
-                        ha->logical_drive_info,
+					    ha->logical_drive_info,
 					    ha->logical_drive_info_dma_addr);
 			ha->logical_drive_info = NULL;
 		}
@@ -4773,7 +4691,7 @@
 	}
 
 	ha->scb_freelist = scb->q_next;
-    scb->flags = 0;
+	scb->flags = 0;
 	scb->q_next = NULL;
 
 	ips_init_scb(ha, scb);
@@ -6831,7 +6749,7 @@
 
 	METHOD_TRACE("ips_version_check", 1);
 
-    VersionInfo = ( IPS_VERSION_DATA * ) ha->ioctl_data;
+	VersionInfo = ( IPS_VERSION_DATA * ) ha->ioctl_data;
 
 	memset(FirmwareVersion, 0, IPS_COMPAT_ID_LENGTH + 1);
 	memset(BiosVersion, 0, IPS_COMPAT_ID_LENGTH + 1);
@@ -6843,7 +6761,7 @@
 	rc = IPS_FAILURE;
 	if (ha->subsys->param[4] & IPS_GET_VERSION_SUPPORT) {	/* If Versioning is Supported */
 		/* Get the Version Info with a Get Version Command */
-        memset( VersionInfo, 0, sizeof (IPS_VERSION_DATA));
+		memset( VersionInfo, 0, sizeof (IPS_VERSION_DATA));
 		rc = ips_get_version_info(ha, ha->ioctl_busaddr, intr);
 		if (rc == IPS_SUCCESS)
 			memcpy(FirmwareVersion, VersionInfo->compatibilityId,
@@ -6999,7 +6917,6 @@
 			for (j = position; j < ips_num_controllers; j++) {
 				switch (ips_ha[j]->ad_type) {
 				case IPS_ADTYPE_SERVERAID6M:
-				case IPS_ADTYPE_SERVERAID7k:
 				case IPS_ADTYPE_SERVERAID7M:
 					if (nvram->adapter_order[i] == 'M') {
 						ips_shift_controllers(position,
@@ -7020,6 +6937,7 @@
 				case IPS_ADTYPE_SERVERAID6I:
 				case IPS_ADTYPE_SERVERAID5I2:
 				case IPS_ADTYPE_SERVERAID5I1:
+				case IPS_ADTYPE_SERVERAID7k:
 					if (nvram->adapter_order[i] == 'S') {
 						ips_shift_controllers(position,
 								      j);
@@ -7547,6 +7465,15 @@
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,9)
 MODULE_LICENSE("GPL");
 #endif
+
+#ifdef MODULE_DESCRIPTION
+MODULE_DESCRIPTION("IBM ServeRAID Adapter Driver " IPS_VER_STRING);
+#endif
+
+#ifdef MODULE_VERSION
+MODULE_VERSION(IPS_VER_STRING);
+#endif
+
 
 /*
  * Overrides for Emacs so that we almost follow Linus's tabbing style.
diff -Nru a/drivers/scsi/ips.h b/drivers/scsi/ips.h
--- a/drivers/scsi/ips.h	2004-05-19 12:42:01 -07:00
+++ b/drivers/scsi/ips.h	2004-08-20 00:18:04 -07:00
@@ -95,14 +95,15 @@
       #define scsi_set_pci_device(sh,dev) (0)
    #endif
 
-   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,23)
-      typedef void irqreturn_t;
+   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
+   
+      #ifndef irqreturn_t
+         typedef void irqreturn_t;
+      #endif 
+      
       #define IRQ_NONE
       #define IRQ_HANDLED
       #define IRQ_RETVAL(x)
-   #endif
-
-   #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)
       #define IPS_REGISTER_HOSTS(SHT)      scsi_register_module(MODULE_SCSI_HA,SHT)
       #define IPS_UNREGISTER_HOSTS(SHT)    scsi_unregister_module(MODULE_SCSI_HA,SHT)
       #define IPS_ADD_HOST(shost,device)
@@ -1213,13 +1214,13 @@
 
 #define IPS_VER_MAJOR 7
 #define IPS_VER_MAJOR_STRING "7"
-#define IPS_VER_MINOR 00
-#define IPS_VER_MINOR_STRING "00"
-#define IPS_VER_BUILD 15
-#define IPS_VER_BUILD_STRING "15"
-#define IPS_VER_STRING "7.00.15"
+#define IPS_VER_MINOR 10
+#define IPS_VER_MINOR_STRING "10"
+#define IPS_VER_BUILD 18
+#define IPS_VER_BUILD_STRING "18"
+#define IPS_VER_STRING "7.10.18"
 #define IPS_RELEASE_ID 0x00020000
-#define IPS_BUILD_IDENT 625
+#define IPS_BUILD_IDENT 731
 #define IPS_LEGALCOPYRIGHT_STRING "(C) Copyright IBM Corp. 1994, 2002. All Rights Reserved."
 #define IPS_ADAPTECCOPYRIGHT_STRING "(c) Copyright Adaptec, Inc. 2002 to 2004. All Rights Reserved."
 #define IPS_DELLCOPYRIGHT_STRING "(c) Copyright Dell 2004. All Rights Reserved."
@@ -1230,32 +1231,35 @@
 #define IPS_VER_SERVERAID2 "2.88.13"
 #define IPS_VER_NAVAJO "2.88.13"
 #define IPS_VER_SERVERAID3 "6.10.24"
-#define IPS_VER_SERVERAID4H "7.00.15"
-#define IPS_VER_SERVERAID4MLx "7.00.15"
-#define IPS_VER_SARASOTA "7.00.15"
-#define IPS_VER_MARCO "7.00.15"
-#define IPS_VER_SEBRING "7.00.15"
+#define IPS_VER_SERVERAID4H "7.10.11"
+#define IPS_VER_SERVERAID4MLx "7.10.18"
+#define IPS_VER_SARASOTA "7.10.18"
+#define IPS_VER_MARCO "7.10.18"
+#define IPS_VER_SEBRING "7.10.18"
+#define IPS_VER_KEYWEST "7.10.18"
 
 /* Compatability IDs for various adapters */
 #define IPS_COMPAT_UNKNOWN ""
-#define IPS_COMPAT_CURRENT "SB610"
+#define IPS_COMPAT_CURRENT "KW710"
 #define IPS_COMPAT_SERVERAID1 "2.25.01"
 #define IPS_COMPAT_SERVERAID2 "2.88.13"
 #define IPS_COMPAT_NAVAJO  "2.88.13"
 #define IPS_COMPAT_KIOWA "2.88.13"
 #define IPS_COMPAT_SERVERAID3H  "SB610"
 #define IPS_COMPAT_SERVERAID3L  "SB610"
-#define IPS_COMPAT_SERVERAID4H  "SB610"
-#define IPS_COMPAT_SERVERAID4M  "SB610"
-#define IPS_COMPAT_SERVERAID4L  "SB610"
-#define IPS_COMPAT_SERVERAID4Mx "SB610"
-#define IPS_COMPAT_SERVERAID4Lx "SB610"
-#define IPS_COMPAT_SARASOTA     "SB610"
-#define IPS_COMPAT_MARCO        "SB610"
-#define IPS_COMPAT_SEBRING      "SB610"
-#define IPS_COMPAT_BIOS "SB610"
+#define IPS_COMPAT_SERVERAID4H  "KW710"
+#define IPS_COMPAT_SERVERAID4M  "KW710"
+#define IPS_COMPAT_SERVERAID4L  "KW710"
+#define IPS_COMPAT_SERVERAID4Mx "KW710"
+#define IPS_COMPAT_SERVERAID4Lx "KW710"
+#define IPS_COMPAT_SARASOTA     "KW710"
+#define IPS_COMPAT_MARCO        "KW710"
+#define IPS_COMPAT_SEBRING      "KW710"
+#define IPS_COMPAT_TAMPA        "KW710"
+#define IPS_COMPAT_KEYWEST      "KW710"
+#define IPS_COMPAT_BIOS "KW710"
 
-#define IPS_COMPAT_MAX_ADAPTER_TYPE 16
+#define IPS_COMPAT_MAX_ADAPTER_TYPE 18
 #define IPS_COMPAT_ID_LENGTH 8
 
 #define IPS_DEFINE_COMPAT_TABLE(tablename) \
@@ -1275,7 +1279,9 @@
       IPS_COMPAT_SARASOTA,         /* one-channel variety of SARASOTA */  \
       IPS_COMPAT_SARASOTA,         /* two-channel variety of SARASOTA */  \
       IPS_COMPAT_MARCO, \
-      IPS_COMPAT_SEBRING \
+      IPS_COMPAT_SEBRING, \
+	  IPS_COMPAT_TAMPA, \
+      IPS_COMPAT_KEYWEST \
    }
 
 
diff -Nru a/drivers/scsi/libata-core.c b/drivers/scsi/libata-core.c
--- a/drivers/scsi/libata-core.c	2004-08-16 09:32:03 -07:00
+++ b/drivers/scsi/libata-core.c	2004-08-31 00:01:32 -07:00
@@ -1858,7 +1858,7 @@
 
 	idx = 0;
 	for (nelem = qc->n_elem; nelem; nelem--,sg++) {
-		u32 addr, boundary;
+		u32 addr, offset;
 		u32 sg_len, len;
 
 		/* determine if physical DMA addr spans 64K boundary.
@@ -1869,10 +1869,10 @@
 		sg_len = sg_dma_len(sg);
 
 		while (sg_len) {
-			boundary = (addr & ~0xffff) + (0xffff + 1);
+			offset = addr & 0xffff;
 			len = sg_len;
-			if ((addr + sg_len) > boundary)
-				len = boundary - addr;
+			if ((offset + sg_len) > 0x10000)
+				len = 0x10000 - offset;
 
 			ap->prd[idx].addr = cpu_to_le32(addr);
 			ap->prd[idx].flags_len = cpu_to_le32(len & 0xffff);
@@ -2304,8 +2304,9 @@
 
 	if ((ap->pio_task_state != PIO_ST_IDLE) &&
 	    (ap->pio_task_state != PIO_ST_TMOUT) &&
-	    (ap->pio_task_state != PIO_ST_ERR))
+	    (ap->pio_task_state != PIO_ST_ERR)) {
 		schedule_task(&ap->pio_task);
+	}
 }
 
 /**
@@ -3596,7 +3597,7 @@
 EXPORT_SYMBOL_GPL(ata_scsi_error);
 EXPORT_SYMBOL_GPL(ata_scsi_detect);
 EXPORT_SYMBOL_GPL(ata_add_to_probe_list);
-EXPORT_SYMBOL_GPL(ata_scsi_release);
 EXPORT_SYMBOL_GPL(libata_msleep);
+EXPORT_SYMBOL_GPL(ata_scsi_release);
 EXPORT_SYMBOL_GPL(ata_host_intr);
 EXPORT_SYMBOL_GPL(ata_dev_id_string);
diff -Nru a/drivers/scsi/libata-scsi.c b/drivers/scsi/libata-scsi.c
--- a/drivers/scsi/libata-scsi.c	2004-08-17 22:52:31 -07:00
+++ b/drivers/scsi/libata-scsi.c	2004-08-31 00:01:32 -07:00
@@ -46,8 +46,8 @@
 /**
  *	ata_std_bios_param - generic bios head/sector/cylinder calculator used by sd.
  *	@disk: SCSI device for which BIOS geometry is to be determined
- *	@dev: device major/minor
- *	@ip: location to which geometry will be output
+ *	@bdev: device major/minor
+ *	@geom: location to which geometry will be output
  *
  *	Generic bios head/sector/cylinder calculator
  *	used by sd. Most BIOSes nowadays expect a XXX/255/16  (CHS)
@@ -61,7 +61,7 @@
  *	Zero.
  */
 int ata_std_bios_param(Disk * disk,	/* SCSI disk */
-		       kdev_t dev,	/* Device major, minor */
+		       kdev_t bdev,	/* Device major, minor */
 		  int *geom /* Heads, sectors, cylinders in that order */ )
 {
 	geom[0] = 255;
@@ -95,7 +95,7 @@
 		return 0;
 
 	case ATA_IOC_SET_IO32:
-		val = (long) arg;
+		val = (unsigned long) arg;
 		if (val != 0)
 			return -EINVAL;
 		return 0;
diff -Nru a/drivers/scsi/sata_nv.c b/drivers/scsi/sata_nv.c
--- a/drivers/scsi/sata_nv.c	2004-08-18 17:50:30 -07:00
+++ b/drivers/scsi/sata_nv.c	2004-08-28 16:29:46 -07:00
@@ -20,6 +20,10 @@
  *  If you do not delete the provisions above, a recipient may use your
  *  version of this file under either the OSL or the GPL.
  *
+ *  0.03
+ *     - Fixed a bug where the hotplug handlers for non-CK804/MCP04 were using
+ *       mmio_base, which is only set for the CK804/MCP04 case.
+ *
  *  0.02
  *     - Added support for CK804 SATA controller.
  *
@@ -40,7 +44,7 @@
 #include <linux/libata.h>
 
 #define DRV_NAME			"sata_nv"
-#define DRV_VERSION			"0.02"
+#define DRV_VERSION			"0.03"
 
 #define NV_PORTS			2
 #define NV_PIO_MASK			0x1f
@@ -416,33 +420,33 @@
 	u8 intr_mask;
 
 	outb(NV_INT_STATUS_HOTPLUG,
-		(unsigned long)probe_ent->mmio_base + NV_INT_STATUS);
+		probe_ent->port[0].scr_addr + NV_INT_STATUS);
 
-	intr_mask = inb((unsigned long)probe_ent->mmio_base + NV_INT_ENABLE);
+	intr_mask = inb(probe_ent->port[0].scr_addr + NV_INT_ENABLE);
 	intr_mask |= NV_INT_ENABLE_HOTPLUG;
 
-	outb(intr_mask, (unsigned long)probe_ent->mmio_base + NV_INT_ENABLE);
+	outb(intr_mask, probe_ent->port[0].scr_addr + NV_INT_ENABLE);
 }
 
 static void nv_disable_hotplug(struct ata_host_set *host_set)
 {
 	u8 intr_mask;
 
-	intr_mask = inb((unsigned long)host_set->mmio_base + NV_INT_ENABLE);
+	intr_mask = inb(host_set->ports[0]->ioaddr.scr_addr + NV_INT_ENABLE);
 
 	intr_mask &= ~(NV_INT_ENABLE_HOTPLUG);
 
-	outb(intr_mask, (unsigned long)host_set->mmio_base + NV_INT_ENABLE);
+	outb(intr_mask, host_set->ports[0]->ioaddr.scr_addr + NV_INT_ENABLE);
 }
 
 static void nv_check_hotplug(struct ata_host_set *host_set)
 {
 	u8 intr_status;
 
-	intr_status = inb((unsigned long)host_set->mmio_base + NV_INT_STATUS);
+	intr_status = inb(host_set->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
 
 	// Clear interrupt status.
-	outb(0xff, (unsigned long)host_set->mmio_base + NV_INT_STATUS);
+	outb(0xff, host_set->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
 
 	if (intr_status & NV_INT_STATUS_HOTPLUG) {
 		if (intr_status & NV_INT_STATUS_PDEV_ADDED)
diff -Nru a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
--- a/drivers/scsi/scsi_debug.c	2002-08-15 04:21:54 -07:00
+++ b/drivers/scsi/scsi_debug.c	2004-08-31 03:01:41 -07:00
@@ -48,7 +48,7 @@
 
 #include "scsi_debug.h"
 
-static const char * scsi_debug_version_str = "Version: 0.61 (20020815)";
+static const char * scsi_debug_version_str = "Version: 0.63 (20040831)";
 
 
 #ifndef SCSI_CMD_READ_16
@@ -73,7 +73,7 @@
 #define SCSI_DEBUG_OPT_MEDIUM_ERR   2
 #define SCSI_DEBUG_OPT_EVERY_NTH   4
 
-#define OPT_MEDIUM_ERR_ADDR   0x1234
+#define OPT_MEDIUM_ERR_ADDR   0x1234  /* that's 4660 in decimal */
 
 static int scsi_debug_num_devs = DEF_NR_FAKE_DEVS;
 static int scsi_debug_opts = DEF_OPTS;
@@ -124,7 +124,7 @@
 
 static unsigned char * fake_storep;	/* ramdisk storage */
 
-static unsigned char broken_buff[SDEBUG_SENSE_LEN];
+static unsigned char spare_buff[SDEBUG_SENSE_LEN];
 
 static int num_aborts = 0;
 static int num_dev_resets = 0;
@@ -193,8 +193,7 @@
 	else 
 		buff = (unsigned char *) SCpnt->request_buffer;
 	if (NULL == buff) {
-		printk(KERN_WARNING "scsi_debug:qc: buff was NULL??\n");
-		buff = broken_buff;	/* just point at dummy */
+		buff = spare_buff;	/* just point at dummy */
 		bufflen = SDEBUG_SENSE_LEN;
 	}
 
@@ -370,7 +369,7 @@
 		if ((errsts = check_reset(SCpnt, devip)))
 			break;
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x20, 0, 14);
-		errsts = (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		errsts = (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 		break;
 	}
 	return schedule_resp(SCpnt, devip, done, errsts, scsi_debug_delay);
@@ -390,7 +389,7 @@
 	if (devip->reset) {
 		devip->reset = 0;
 		mk_sense_buffer(devip, UNIT_ATTENTION, 0x29, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	return 0;
 }
@@ -451,7 +450,7 @@
 	arr[0] = pq_pdt;
 	if (0x2 & cmd[1]) {  /* CMDDT bit set */
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x24, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	} else if (0x1 & cmd[1]) {  /* EVPD bit set */
 		int dev_id_num, len;
 		char dev_id_str[6];
@@ -476,7 +475,7 @@
 		} else {
 			/* Illegal request, invalid field in cdb */
 			mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x24, 0, 14);
-			return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+			return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 		}
 		memcpy(buff, arr, min_len); 
 		return 0;
@@ -589,7 +588,7 @@
 	memset(arr, 0, SDEBUG_MAX_MSENSE_SZ);
 	if (0x3 == pcontrol) {  /* Saving values not supported */
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x39, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	dev_spec = DEV_READONLY(target) ? 0x80 : 0x0;
 	if (msense_6) {
@@ -631,7 +630,7 @@
 		break;
 	default:
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x24, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	if (msense_6)
 		arr[0] = offset - 1;
@@ -655,14 +654,14 @@
 
 	if (upper_blk || (block + num > CAPACITY)) {
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x21, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	if ((SCSI_DEBUG_OPT_MEDIUM_ERR & scsi_debug_opts) &&
-	    (block >= OPT_MEDIUM_ERR_ADDR) && 
-	    (block < (OPT_MEDIUM_ERR_ADDR + num))) {
+	    (block <= OPT_MEDIUM_ERR_ADDR) &&
+            ((block + num) > OPT_MEDIUM_ERR_ADDR)) {
 		mk_sense_buffer(devip, MEDIUM_ERROR, 0x11, 0, 14);
 		/* claim unrecoverable read error */
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	read_lock_irqsave(&atomic_rw, iflags);
         sgcount = 0;
@@ -704,7 +703,7 @@
 
 	if (upper_blk || (block + num > CAPACITY)) {
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x21, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 
 	write_lock_irqsave(&atomic_rw, iflags);
@@ -744,7 +743,7 @@
 	alloc_len = cmd[9] + (cmd[8] << 8) + (cmd[7] << 16) + (cmd[6] << 24);
 	if ((alloc_len < 16) || (select_report > 2)) {
 		mk_sense_buffer(devip, ILLEGAL_REQUEST, 0x24, 0, 14);
-		return (COMMAND_COMPLETE << 8) | (CHECK_CONDITION << 1);
+		return (DRIVER_SENSE << 24) | (CHECK_CONDITION << 1);
 	}
 	if (bufflen > 3) {
 		memset(buff, 0, bufflen);
@@ -773,8 +772,10 @@
 		return;
 	}
 	sqcp->in_use = 0;
-	if (sqcp->done_funct)
+	if (sqcp->done_funct) {
+		sqcp->a_cmnd->result = sqcp->scsi_result;
 		sqcp->done_funct(sqcp->a_cmnd); /* callback to mid level */
+	}
 	sqcp->done_funct = NULL;
 	spin_unlock_irqrestore(&queued_arr_lock, iflags);
 }
diff -Nru a/drivers/scsi/seagate.c b/drivers/scsi/seagate.c
--- a/drivers/scsi/seagate.c	2003-06-29 08:10:06 -07:00
+++ b/drivers/scsi/seagate.c	2004-08-26 16:50:38 -07:00
@@ -698,7 +698,7 @@
 	done_fn = done;
 	current_target = SCpnt->target;
 	current_lun = SCpnt->lun;
-	(const void *) current_cmnd = SCpnt->cmnd;
+	current_cmnd = SCpnt->cmnd;
 	current_data = (unsigned char *) SCpnt->request_buffer;
 	current_bufflen = SCpnt->request_bufflen;
 	SCint = SCpnt;
diff -Nru a/drivers/sound/ac97_codec.c b/drivers/sound/ac97_codec.c
--- a/drivers/sound/ac97_codec.c	2004-08-04 09:07:52 -07:00
+++ b/drivers/sound/ac97_codec.c	2004-08-28 16:40:14 -07:00
@@ -155,6 +155,7 @@
 	{0x43525931, "Cirrus Logic CS4299 rev A", &crystal_digital_ops},
 	{0x43525933, "Cirrus Logic CS4299 rev C", &crystal_digital_ops},
 	{0x43525934, "Cirrus Logic CS4299 rev D", &crystal_digital_ops},
+	{0x43585430, "CXT48",			&default_ops,		AC97_DELUDED_MODEM },
 	{0x43585442, "CXT66",			&default_ops,		AC97_DELUDED_MODEM },
 	{0x44543031, "Diamond Technology DT0893", &default_ops},
 	{0x45838308, "ESS Allegro ES1988",	&null_ops},
diff -Nru a/drivers/sound/i810_audio.c b/drivers/sound/i810_audio.c
--- a/drivers/sound/i810_audio.c	2004-05-14 22:57:53 -07:00
+++ b/drivers/sound/i810_audio.c	2004-08-30 12:42:09 -07:00
@@ -304,6 +304,14 @@
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
 	{PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_MCP3_AUDIO,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
+	{PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_MCP2S_AUDIO,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
+	{PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_CK8S_AUDIO,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
+	{PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_CK804_AUDIO,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
+	{PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_MCP04_AUDIO,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, NVIDIA_NFORCE},
 	{PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_OPUS_7445,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, AMD768},
 	{PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8111_AUDIO,
diff -Nru a/fs/affs/super.c b/fs/affs/super.c
--- a/fs/affs/super.c	2004-06-18 02:07:09 -07:00
+++ b/fs/affs/super.c	2004-08-26 16:50:38 -07:00
@@ -130,7 +130,7 @@
 					printk("AFFS: Argument for set[ug]id option missing\n");
 					return 0;
 				} else {
-					(f ? *uid : *gid) = simple_strtoul(value,&value,0);
+					*(f ? uid : gid) = simple_strtoul(value,&value,0);
 					if (*value) {
 						printk("AFFS: Bad set[ug]id argument\n");
 						return 0;
diff -Nru a/fs/buffer.c b/fs/buffer.c
--- a/fs/buffer.c	2004-08-15 19:28:55 -07:00
+++ b/fs/buffer.c	2004-08-29 23:00:51 -07:00
@@ -371,6 +371,7 @@
 	}
 	return err;
 }
+EXPORT_SYMBOL(sync_buffers);
 
 int fsync_super(struct super_block *sb)
 {
diff -Nru a/fs/ext3/super.c b/fs/ext3/super.c
--- a/fs/ext3/super.c	2004-05-20 13:50:09 -07:00
+++ b/fs/ext3/super.c	2004-08-15 18:18:18 -07:00
@@ -1302,6 +1302,7 @@
 	if (!journal) {
 		printk(KERN_ERR "EXT3-fs: Could not load journal inode\n");
 		iput(journal_inode);
+		return NULL;
 	}
 	ext3_init_journal_params(EXT3_SB(sb), journal);
 	return journal;
diff -Nru a/fs/hfs/file_hdr.c b/fs/hfs/file_hdr.c
--- a/fs/hfs/file_hdr.c	2004-08-08 01:59:53 -07:00
+++ b/fs/hfs/file_hdr.c	2004-08-26 06:28:10 -07:00
@@ -642,17 +642,18 @@
         int left, lcv, written = 0;
 	struct hdr_hdr meta;
 	int built_meta = 0;
-        off_t pos;
+        loff_t pos;
 
 	if (!S_ISREG(inode->i_mode)) {
 		hfs_warn("hfs_hdr_write: mode = %07o\n", inode->i_mode);
 		return -EINVAL;
 	}
+
+	pos = (filp->f_flags & O_APPEND) ? inode->i_size : *ppos;
+
 	if (count <= 0 || pos != (unsigned)pos) {
 		return 0;
 	}
-
-	pos = (filp->f_flags & O_APPEND) ? inode->i_size : *ppos;
 
 	if (!HFS_I(inode)->layout) {
 		HFS_I(inode)->layout = dup_layout(HFS_I(inode)->default_layout);
diff -Nru a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c
--- a/fs/jfs/jfs_metapage.c	2004-06-14 13:26:55 -07:00
+++ b/fs/jfs/jfs_metapage.c	2004-08-28 13:15:40 -07:00
@@ -606,6 +606,7 @@
 			if (page) {
 				block_flushpage(page, 0);
 				UnlockPage(page);
+				page_cache_release(page);
 			}
 		}
 	}
diff -Nru a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c
--- a/fs/jfs/jfs_mount.c	2003-08-25 11:19:31 -07:00
+++ b/fs/jfs/jfs_mount.c	2004-08-20 13:44:15 -07:00
@@ -197,9 +197,6 @@
 	/*
 	 *      unwind on error
 	 */
-//errout42: /* close fileset inode allocation map */
-	diUnmount(ipimap, 1);
-
       errout41:		/* close fileset inode allocation map inode */
 	diFreeSpecial(ipimap);
 
diff -Nru a/fs/xfs/linux-2.4/xfs_aops.c b/fs/xfs/linux-2.4/xfs_aops.c
--- a/fs/xfs/linux-2.4/xfs_aops.c	2004-06-17 20:56:25 -07:00
+++ b/fs/xfs/linux-2.4/xfs_aops.c	2004-08-18 19:08:12 -07:00
@@ -209,6 +209,7 @@
 	sector_shift = block_bits - BBSHIFT;
 	bn = iomapp->iomap_bn >> sector_shift;
 	bn += delta;
+	BUG_ON(!bn && !(iomapp->iomap_flags & IOMAP_REALTIME));
 	ASSERT((bn << sector_shift) >= iomapp->iomap_bn);
 
 	lock_buffer(bh);
@@ -887,7 +888,7 @@
 
 			bn = iomap.iomap_bn >> (inode->i_blkbits - BBSHIFT);
 			bn += delta;
-
+			BUG_ON(!bn && !(iomap.iomap_flags & IOMAP_REALTIME));
 			bh_result->b_blocknr = bn;
 			set_buffer_mapped(bh_result);
 		}
@@ -911,10 +912,10 @@
 	}
 
 	if (iomap.iomap_flags & IOMAP_DELAY) {
-		if (unlikely(direct))
-			BUG();
+		BUG_ON(direct);
 		if (create) {
 			set_buffer_mapped(bh_result);
+			set_buffer_uptodate(bh_result);
 		}
 		set_buffer_delay(bh_result);
 	}
@@ -934,17 +935,6 @@
 }
 
 STATIC int
-linvfs_get_block_sync(
-	struct inode		*inode,
-	long			iblock,
-	struct buffer_head	*bh_result,
-	int			create)
-{
-	return linvfs_get_block_core(inode, iblock, bh_result,
-					create, 0, BMAPI_SYNC|BMAPI_WRITE);
-}
-
-STATIC int
 linvfs_get_block_direct(
 	struct inode		*inode,
 	long			iblock,
@@ -1155,13 +1145,7 @@
 	unsigned int		from,
 	unsigned int		to)
 {
-	if (file && (file->f_flags & O_SYNC)) {
-		return block_prepare_write(page, from, to,
-						linvfs_get_block_sync);
-	} else {
-		return block_prepare_write(page, from, to,
-						linvfs_get_block);
-	}
+	return block_prepare_write(page, from, to, linvfs_get_block);
 }
 
 /*
diff -Nru a/fs/xfs/linux-2.4/xfs_buf.c b/fs/xfs/linux-2.4/xfs_buf.c
--- a/fs/xfs/linux-2.4/xfs_buf.c	2004-08-02 19:11:04 -07:00
+++ b/fs/xfs/linux-2.4/xfs_buf.c	2004-08-18 19:11:25 -07:00
@@ -1445,6 +1445,7 @@
 	xfs_flush_buftarg(btp, 1);
 	if (external)
 		xfs_blkdev_put(btp->pbr_bdev);
+	iput(btp->pbr_mapping->host);
 	kmem_free(btp, sizeof(*btp));
 }
 
@@ -1458,7 +1459,7 @@
 	truncate_inode_pages(btp->pbr_mapping, 0LL);
 }
 
-void
+int
 xfs_setsize_buftarg(
 	xfs_buftarg_t		*btp,
 	unsigned int		blocksize,
@@ -1472,7 +1473,40 @@
 		printk(KERN_WARNING
 			"XFS: Cannot set_blocksize to %u on device 0x%x\n",
 			sectorsize, kdev_t_to_nr(btp->pbr_kdev));
+		return EINVAL;
+	}
+	return 0;
+}
+
+STATIC int
+xfs_mapping_buftarg(
+	xfs_buftarg_t		*btp,
+	struct block_device	*bdev)
+{
+	kdev_t			kdev;
+	struct inode		*inode;
+	struct address_space	*mapping;
+	static struct address_space_operations mapping_aops = {
+		.sync_page = block_sync_page,
+	};
+
+	kdev = to_kdev_t(bdev->bd_dev);
+	inode = new_inode(bdev->bd_inode->i_sb);
+	if (!inode) {
+		printk(KERN_WARNING
+			"XFS: Cannot allocate mapping inode for device %s\n",
+			XFS_BUFTARG_NAME(btp));
+		return ENOMEM;
 	}
+	inode->i_mode = S_IFBLK;
+	inode->i_dev  = kdev;
+	inode->i_rdev = kdev;
+	inode->i_bdev = bdev;
+	mapping = &inode->i_data;
+	mapping->a_ops = &mapping_aops;
+	mapping->gfp_mask = GFP_KERNEL;
+	btp->pbr_mapping = mapping;
+	return 0;
 }
 
 xfs_buftarg_t *
@@ -1480,16 +1514,14 @@
 	struct block_device	*bdev)
 {
 	xfs_buftarg_t		*btp;
+	kdev_t			kdev;
 
 	btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
 
+	kdev = to_kdev_t(bdev->bd_dev);
 	btp->pbr_dev =  bdev->bd_dev;
-	btp->pbr_kdev = to_kdev_t(btp->pbr_dev);
+	btp->pbr_kdev = kdev;
 	btp->pbr_bdev = bdev;
-	btp->pbr_mapping = bdev->bd_inode->i_mapping;
-	xfs_setsize_buftarg(btp, PAGE_CACHE_SIZE,
-			    get_hardsect_size(btp->pbr_kdev));
-
 	switch (MAJOR(btp->pbr_dev)) {
 	case MD_MAJOR:
 	case EVMS_MAJOR:
@@ -1499,8 +1531,15 @@
 		btp->pbr_flags = PBR_SECTOR_ONLY;
 		break;
 	}
-
+	if (xfs_setsize_buftarg(btp, PAGE_CACHE_SIZE, get_hardsect_size(kdev)))
+		goto error;
+	if (xfs_mapping_buftarg(btp, bdev))
+		goto error;
 	return btp;
+
+error:
+	kmem_free(btp, sizeof(*btp));
+	return NULL;
 }
 
 /*
diff -Nru a/fs/xfs/linux-2.4/xfs_buf.h b/fs/xfs/linux-2.4/xfs_buf.h
--- a/fs/xfs/linux-2.4/xfs_buf.h	2004-06-17 19:08:05 -07:00
+++ b/fs/xfs/linux-2.4/xfs_buf.h	2004-08-18 16:24:20 -07:00
@@ -617,7 +617,7 @@
  
 extern xfs_buftarg_t *xfs_alloc_buftarg(struct block_device *);
 extern void xfs_free_buftarg(xfs_buftarg_t *, int);
-extern void xfs_setsize_buftarg(xfs_buftarg_t *, unsigned int, unsigned int);
+extern int xfs_setsize_buftarg(xfs_buftarg_t *, unsigned int, unsigned int);
 extern void xfs_incore_relse(xfs_buftarg_t *, int, int);
 extern int xfs_flush_buftarg(xfs_buftarg_t *, int);
 
diff -Nru a/fs/xfs/linux-2.4/xfs_ioctl.c b/fs/xfs/linux-2.4/xfs_ioctl.c
--- a/fs/xfs/linux-2.4/xfs_ioctl.c	2004-05-24 21:20:46 -07:00
+++ b/fs/xfs/linux-2.4/xfs_ioctl.c	2004-08-18 21:10:23 -07:00
@@ -71,14 +71,6 @@
 #include <linux/iobuf.h>
 
 /*
- * ioctl commands that are used by Linux filesystems
- */
-#define XFS_IOC_GETXFLAGS	_IOR('f', 1, long)
-#define XFS_IOC_SETXFLAGS	_IOW('f', 2, long)
-#define XFS_IOC_GETVERSION	_IOR('v', 1, long)
-
-
-/*
  * xfs_find_handle maps from userspace xfs_fsop_handlereq structure to
  * a file or fs handle.
  *
@@ -827,13 +819,23 @@
 	case XFS_IOC_FREEZE:
 		if (!capable(CAP_SYS_ADMIN))
 			return -EPERM;
-		xfs_fs_freeze(mp);
+
+		if (vp->v_vfsp->vfs_frozen == SB_UNFROZEN) {
+			freeze_bdev(mp->m_ddev_targp->pbr_bdev);
+			if (mp->m_rtdev_targp)
+				freeze_bdev(mp->m_rtdev_targp->pbr_bdev);
+		}
 		return 0;
 
 	case XFS_IOC_THAW:
 		if (!capable(CAP_SYS_ADMIN))
 			return -EPERM;
-		xfs_fs_thaw(mp);
+
+		if (vp->v_vfsp->vfs_frozen != SB_UNFROZEN) {
+			thaw_bdev(mp->m_ddev_targp->pbr_bdev, inode->i_sb);
+			if (mp->m_rtdev_targp)
+				thaw_bdev(mp->m_ddev_targp->pbr_bdev, NULL);
+		}
 		return 0;
 
 	case XFS_IOC_GOINGDOWN: {
diff -Nru a/fs/xfs/linux-2.4/xfs_linux.h b/fs/xfs/linux-2.4/xfs_linux.h
--- a/fs/xfs/linux-2.4/xfs_linux.h	2004-05-24 21:20:46 -07:00
+++ b/fs/xfs/linux-2.4/xfs_linux.h	2004-08-18 16:55:19 -07:00
@@ -35,6 +35,9 @@
 #include <linux/types.h>
 #include <linux/config.h>
 
+#undef __user
+#define __user
+
 /*
  * Some types are conditional depending on the target system.
  * XFS_BIG_BLKNOS needs block layer disk addresses to be 64 bits.
diff -Nru a/fs/xfs/linux-2.4/xfs_lrw.c b/fs/xfs/linux-2.4/xfs_lrw.c
--- a/fs/xfs/linux-2.4/xfs_lrw.c	2004-08-02 19:22:53 -07:00
+++ b/fs/xfs/linux-2.4/xfs_lrw.c	2004-08-18 16:16:24 -07:00
@@ -604,7 +604,7 @@
 	io = &xip->i_iocore;
 	mp = io->io_mount;
 
-	xfs_check_frozen(mp, bdp, XFS_FREEZE_WRITE);
+	fs_check_frozen(vp->v_vfsp, SB_FREEZE_WRITE);
 
 	if (XFS_FORCED_SHUTDOWN(xip->i_mount)) {
 		return -EIO;
@@ -619,8 +619,6 @@
 		iolock = XFS_IOLOCK_SHARED;
 		locktype = VRWLOCK_WRITE_DIRECT;
 	} else {
-		if (io->io_flags & XFS_IOCORE_RT)
-			return XFS_ERROR(-EINVAL);
 		iolock = XFS_IOLOCK_EXCL;
 		locktype = VRWLOCK_WRITE;
 	}
diff -Nru a/fs/xfs/linux-2.4/xfs_super.c b/fs/xfs/linux-2.4/xfs_super.c
--- a/fs/xfs/linux-2.4/xfs_super.c	2004-06-17 21:09:30 -07:00
+++ b/fs/xfs/linux-2.4/xfs_super.c	2004-08-18 15:54:56 -07:00
@@ -568,32 +568,73 @@
 	return -error;
 }
 
+struct super_block *freeze_bdev(struct block_device *bdev)
+{
+	struct super_block *sb;
+	struct vfs *vfsp;
+	int error;
+
+	sb = get_super(to_kdev_t(bdev->bd_dev));
+	if (sb && !(sb->s_flags & MS_RDONLY)) {
+		vfsp = LINVFS_GET_VFS(sb);
+
+		/* Stop new writers */
+		vfsp->vfs_frozen = SB_FREEZE_WRITE;
+		wmb();
+
+		/* Flush the refcache */
+		VFS_SYNC(vfsp, SYNC_REFCACHE|SYNC_WAIT, NULL, error);
+
+		/* Flush delalloc and delwri data */
+		VFS_SYNC(vfsp, SYNC_DELWRI|SYNC_WAIT, NULL, error);
+
+		/* Pause transaction subsystem */
+		vfsp->vfs_frozen = SB_FREEZE_TRANS;
+		wmb();
+
+		/* Flush any remaining inodes into buffers */
+		VFS_SYNC(vfsp, SYNC_ATTR|SYNC_WAIT, NULL, error);
+
+		 /* Push all buffers out to disk */
+		sync_buffers(sb->s_dev, 1);
+
+		/* Push the superblock and write an unmount record */
+		VFS_FREEZE(vfsp);
+	}
+
+	sync_buffers(to_kdev_t(bdev->bd_dev), 1);
+	return sb;      /* thaw_bdev releases sb->s_umount */
+}
+
+void thaw_bdev(struct block_device *bdev, struct super_block *sb)
+{
+	if (sb) {
+		struct vfs *vfsp = LINVFS_GET_VFS(sb);
+
+		BUG_ON(sb->s_bdev != bdev);
+
+		vfsp->vfs_frozen = SB_UNFROZEN;
+		wmb();
+		wake_up(&vfsp->vfs_wait_unfrozen);
+
+		drop_super(sb);
+	}
+}
+
 STATIC void
 linvfs_freeze_fs(
 	struct super_block	*sb)
 {
-	vfs_t			*vfsp = LINVFS_GET_VFS(sb);
-	vnode_t			*vp;
-	int			error;
-
 	if (sb->s_flags & MS_RDONLY)
 		return;
-	VFS_ROOT(vfsp, &vp, error);
-	VOP_IOCTL(vp, LINVFS_GET_IP(vp), NULL, 0, XFS_IOC_FREEZE, 0, error);
-	VN_RELE(vp);
+	freeze_bdev(sb->s_bdev);
 }
 
 STATIC void
 linvfs_unfreeze_fs(
 	struct super_block	*sb)
 {
-	vfs_t			*vfsp = LINVFS_GET_VFS(sb);
-	vnode_t			*vp;
-	int			error;
-
-	VFS_ROOT(vfsp, &vp, error);
-	VOP_IOCTL(vp, LINVFS_GET_IP(vp), NULL, 0, XFS_IOC_THAW, 0, error);
-	VN_RELE(vp);
+	thaw_bdev(sb->s_bdev, sb);
 }
 
 STATIC int
diff -Nru a/fs/xfs/linux-2.4/xfs_super.h b/fs/xfs/linux-2.4/xfs_super.h
--- a/fs/xfs/linux-2.4/xfs_super.h	2004-06-17 17:39:24 -07:00
+++ b/fs/xfs/linux-2.4/xfs_super.h	2004-08-18 15:54:56 -07:00
@@ -136,5 +136,7 @@
 
 /* matching a 2.6 kernel export, thus no xfs_ prefix */
 extern struct dentry *d_alloc_anon(struct inode *inode);
+extern struct super_block *freeze_bdev(struct block_device *bdev);
+extern void thaw_bdev(struct block_device *bdev, struct super_block *sb);
 
 #endif	/* __XFS_SUPER_H__ */
diff -Nru a/fs/xfs/linux-2.4/xfs_vfs.c b/fs/xfs/linux-2.4/xfs_vfs.c
--- a/fs/xfs/linux-2.4/xfs_vfs.c	2004-05-24 21:20:47 -07:00
+++ b/fs/xfs/linux-2.4/xfs_vfs.c	2004-08-18 15:54:56 -07:00
@@ -243,6 +243,18 @@
 	((*bhvtovfsops(next)->vfs_force_shutdown)(next, fl, file, line));
 }
 
+void
+vfs_freeze(
+	struct bhv_desc		*bdp)
+{
+	struct bhv_desc		*next = bdp;
+
+	ASSERT(next);
+	while (! (bhvtovfsops(next))->vfs_freeze)
+		next = BHV_NEXT(next);
+	((*bhvtovfsops(next)->vfs_freeze)(next));
+}
+
 vfs_t *
 vfs_allocate( void )
 {
@@ -251,6 +263,7 @@
 	vfsp = kmem_zalloc(sizeof(vfs_t), KM_SLEEP);
 	bhv_head_init(VFS_BHVHEAD(vfsp), "vfs");
 	init_waitqueue_head(&vfsp->vfs_wait_sync_task);
+	init_waitqueue_head(&vfsp->vfs_wait_unfrozen);
 	return vfsp;
 }
 
diff -Nru a/fs/xfs/linux-2.4/xfs_vfs.h b/fs/xfs/linux-2.4/xfs_vfs.h
--- a/fs/xfs/linux-2.4/xfs_vfs.h	2004-05-24 21:20:47 -07:00
+++ b/fs/xfs/linux-2.4/xfs_vfs.h	2004-08-18 15:54:56 -07:00
@@ -53,6 +53,8 @@
 	struct super_block	*vfs_super;	/* Linux superblock structure */
 	struct task_struct	*vfs_sync_task;
 	wait_queue_head_t	vfs_wait_sync_task;
+	int			vfs_frozen;
+	wait_queue_head_t	vfs_wait_unfrozen;
 } vfs_t;
 
 #define vfs_fbhv		vfs_bh.bh_first	/* 1st on vfs behavior chain */
@@ -114,6 +116,7 @@
 typedef void	(*vfs_init_vnode_t)(bhv_desc_t *,
 				struct vnode *, bhv_desc_t *, int);
 typedef void	(*vfs_force_shutdown_t)(bhv_desc_t *, int, char *, int);
+typedef void	(*vfs_freeze_t)(bhv_desc_t *);
 typedef	struct inode * (*vfs_get_inode_t)(bhv_desc_t *, xfs_ino_t, int);
 
 typedef struct vfsops {
@@ -132,6 +135,7 @@
 	vfs_get_inode_t		vfs_get_inode;	/* bhv specific iget */
 	vfs_init_vnode_t	vfs_init_vnode;	/* initialize a new vnode */
 	vfs_force_shutdown_t	vfs_force_shutdown;	/* crash and burn */
+	vfs_freeze_t		vfs_freeze;	/* freeze fs for snapshot */
 } vfsops_t;
 
 /*
@@ -152,6 +156,7 @@
 #define VFS_GET_INODE(v, ino, fl)	( vfs_get_inode(VHEAD(v), ino,fl) )
 #define VFS_INIT_VNODE(v, vp,b,ul)	( vfs_init_vnode(VHEAD(v), vp,b,ul) )
 #define VFS_FORCE_SHUTDOWN(v, fl,f,l)	( vfs_force_shutdown(VHEAD(v), fl,f,l) )
+#define VFS_FREEZE(v)			( vfs_freeze(VHEAD(v)) )
 
 /*
  * PVFS's.  Operates on behavior descriptor pointers.
@@ -170,6 +175,7 @@
 #define PVFS_GET_INODE(b, ino,fl)	( vfs_get_inode(b, ino,fl) )
 #define PVFS_INIT_VNODE(b, vp,b2,ul)	( vfs_init_vnode(b, vp,b2,ul) )
 #define PVFS_FORCE_SHUTDOWN(b, fl,f,l)	( vfs_force_shutdown(b, fl,f,l) )
+#define PVFS_FREEZE(b)			( vfs_freeze(b) )
 
 extern int vfs_mount(bhv_desc_t *, struct xfs_mount_args *, struct cred *);
 extern int vfs_parseargs(bhv_desc_t *, char *, struct xfs_mount_args *, int);
@@ -185,6 +191,7 @@
 extern struct inode *vfs_get_inode(bhv_desc_t *, xfs_ino_t, int);
 extern void vfs_init_vnode(bhv_desc_t *, struct vnode *, bhv_desc_t *, int);
 extern void vfs_force_shutdown(bhv_desc_t *, int, char *, int);
+extern void vfs_freeze(bhv_desc_t *);
 
 typedef struct bhv_vfsops {
 	struct vfsops		bhv_common;
@@ -204,5 +211,15 @@
 extern void bhv_insert_all_vfsops(struct vfs *);
 extern void bhv_remove_all_vfsops(struct vfs *, int);
 extern void bhv_remove_vfsops(struct vfs *, int);
+
+enum {
+	SB_UNFROZEN = 0,
+	SB_FREEZE_WRITE	= 1,
+	SB_FREEZE_TRANS = 2,
+};
+
+#define fs_frozen(vfsp)		((vfsp)->vfs_frozen)
+#define fs_check_frozen(vfsp, level) \
+	wait_event((vfsp)->vfs_wait_unfrozen, ((vfsp)->vfs_frozen < (level)))
 
 #endif	/* __XFS_VFS_H__ */
diff -Nru a/fs/xfs/quota/xfs_dquot.c b/fs/xfs/quota/xfs_dquot.c
--- a/fs/xfs/quota/xfs_dquot.c	2003-12-16 18:40:56 -08:00
+++ b/fs/xfs/quota/xfs_dquot.c	2004-08-26 23:21:36 -07:00
@@ -145,7 +145,7 @@
 		 dqp->q_res_icount = 0;
 		 dqp->q_res_rtbcount = 0;
 		 dqp->q_pincount = 0;
-		 dqp->q_hash = 0;
+		 dqp->q_hash = NULL;
 		 ASSERT(dqp->dq_flnext == dqp->dq_flprev);
 
 #ifdef XFS_DQUOT_TRACE
@@ -187,9 +187,9 @@
  */
 STATIC void
 xfs_qm_dqinit_core(
-	xfs_dqid_t	 id,
-	uint		 type,
-	xfs_dqblk_t	 *d)
+	xfs_dqid_t	id,
+	uint		type,
+	xfs_dqblk_t	*d)
 {
 	/*
 	 * Caller has zero'd the entire dquot 'chunk' already.
@@ -250,6 +250,36 @@
 
 
 /*
+ * If default limits are in force, push them into the dquot now.
+ * We overwrite the dquot limits only if they are zero and this
+ * is not the root dquot.
+ */
+void
+xfs_qm_adjust_dqlimits(
+	xfs_mount_t		*mp,
+	xfs_disk_dquot_t	*d)
+{
+	xfs_quotainfo_t		*q = mp->m_quotainfo;
+
+	ASSERT(!INT_ISZERO(d->d_id, ARCH_CONVERT));
+
+	if (q->qi_bsoftlimit && INT_ISZERO(d->d_blk_softlimit, ARCH_CONVERT))
+		INT_SET(d->d_blk_softlimit, ARCH_CONVERT, q->qi_bsoftlimit);
+	if (q->qi_bhardlimit && INT_ISZERO(d->d_blk_hardlimit, ARCH_CONVERT))
+		INT_SET(d->d_blk_hardlimit, ARCH_CONVERT, q->qi_bhardlimit);
+	if (q->qi_isoftlimit && INT_ISZERO(d->d_ino_softlimit, ARCH_CONVERT))
+		INT_SET(d->d_ino_softlimit, ARCH_CONVERT, q->qi_isoftlimit);
+	if (q->qi_ihardlimit && INT_ISZERO(d->d_ino_hardlimit, ARCH_CONVERT))
+		INT_SET(d->d_ino_hardlimit, ARCH_CONVERT, q->qi_ihardlimit);
+	if (q->qi_rtbsoftlimit &&
+	    INT_ISZERO(d->d_rtb_softlimit, ARCH_CONVERT))
+		INT_SET(d->d_rtb_softlimit, ARCH_CONVERT, q->qi_rtbsoftlimit);
+	if (q->qi_rtbhardlimit &&
+	    INT_ISZERO(d->d_rtb_hardlimit, ARCH_CONVERT))
+		INT_SET(d->d_rtb_hardlimit, ARCH_CONVERT, q->qi_rtbhardlimit);
+}
+
+/*
  * Check the limits and timers of a dquot and start or reset timers
  * if necessary.
  * This gets called even when quota enforcement is OFF, which makes our
@@ -265,51 +295,79 @@
 	xfs_mount_t		*mp,
 	xfs_disk_dquot_t	*d)
 {
-	/*
-	 * The dquot had better be locked. We are modifying it here.
-	 */
-
-	/*
-	 * root's limits are not real limits.
-	 */
-	if (INT_ISZERO(d->d_id, ARCH_CONVERT))
-		return;
+	ASSERT(!INT_ISZERO(d->d_id, ARCH_CONVERT));
 
 #ifdef QUOTADEBUG
 	if (INT_GET(d->d_blk_hardlimit, ARCH_CONVERT))
-		ASSERT(INT_GET(d->d_blk_softlimit, ARCH_CONVERT) <= INT_GET(d->d_blk_hardlimit, ARCH_CONVERT));
+		ASSERT(INT_GET(d->d_blk_softlimit, ARCH_CONVERT) <=
+			INT_GET(d->d_blk_hardlimit, ARCH_CONVERT));
 	if (INT_GET(d->d_ino_hardlimit, ARCH_CONVERT))
-		ASSERT(INT_GET(d->d_ino_softlimit, ARCH_CONVERT) <= INT_GET(d->d_ino_hardlimit, ARCH_CONVERT));
+		ASSERT(INT_GET(d->d_ino_softlimit, ARCH_CONVERT) <=
+			INT_GET(d->d_ino_hardlimit, ARCH_CONVERT));
+	if (INT_GET(d->d_rtb_hardlimit, ARCH_CONVERT))
+		ASSERT(INT_GET(d->d_rtb_softlimit, ARCH_CONVERT) <=
+			INT_GET(d->d_rtb_hardlimit, ARCH_CONVERT));
 #endif
 	if (INT_ISZERO(d->d_btimer, ARCH_CONVERT)) {
 		if ((INT_GET(d->d_blk_softlimit, ARCH_CONVERT) &&
-		    (INT_GET(d->d_bcount, ARCH_CONVERT) >= INT_GET(d->d_blk_softlimit, ARCH_CONVERT))) ||
+		    (INT_GET(d->d_bcount, ARCH_CONVERT) >=
+				INT_GET(d->d_blk_softlimit, ARCH_CONVERT))) ||
 		    (INT_GET(d->d_blk_hardlimit, ARCH_CONVERT) &&
-		    (INT_GET(d->d_bcount, ARCH_CONVERT) >= INT_GET(d->d_blk_hardlimit, ARCH_CONVERT)))) {
-			INT_SET(d->d_btimer, ARCH_CONVERT, get_seconds() + XFS_QI_BTIMELIMIT(mp));
+		    (INT_GET(d->d_bcount, ARCH_CONVERT) >=
+				INT_GET(d->d_blk_hardlimit, ARCH_CONVERT)))) {
+			INT_SET(d->d_btimer, ARCH_CONVERT,
+				get_seconds() + XFS_QI_BTIMELIMIT(mp));
 		}
 	} else {
 		if ((INT_ISZERO(d->d_blk_softlimit, ARCH_CONVERT) ||
-		    (INT_GET(d->d_bcount, ARCH_CONVERT) < INT_GET(d->d_blk_softlimit, ARCH_CONVERT))) &&
+		    (INT_GET(d->d_bcount, ARCH_CONVERT) <
+				INT_GET(d->d_blk_softlimit, ARCH_CONVERT))) &&
 		    (INT_ISZERO(d->d_blk_hardlimit, ARCH_CONVERT) ||
-		    (INT_GET(d->d_bcount, ARCH_CONVERT) < INT_GET(d->d_blk_hardlimit, ARCH_CONVERT)))) {
+		    (INT_GET(d->d_bcount, ARCH_CONVERT) <
+				INT_GET(d->d_blk_hardlimit, ARCH_CONVERT)))) {
 			INT_ZERO(d->d_btimer, ARCH_CONVERT);
 		}
 	}
 
 	if (INT_ISZERO(d->d_itimer, ARCH_CONVERT)) {
 		if ((INT_GET(d->d_ino_softlimit, ARCH_CONVERT) &&
-		    (INT_GET(d->d_icount, ARCH_CONVERT) >= INT_GET(d->d_ino_softlimit, ARCH_CONVERT))) ||
+		    (INT_GET(d->d_icount, ARCH_CONVERT) >=
+				INT_GET(d->d_ino_softlimit, ARCH_CONVERT))) ||
 		    (INT_GET(d->d_ino_hardlimit, ARCH_CONVERT) &&
-		    (INT_GET(d->d_icount, ARCH_CONVERT) >= INT_GET(d->d_ino_hardlimit, ARCH_CONVERT)))) {
-			INT_SET(d->d_itimer, ARCH_CONVERT, get_seconds() + XFS_QI_ITIMELIMIT(mp));
+		    (INT_GET(d->d_icount, ARCH_CONVERT) >=
+				INT_GET(d->d_ino_hardlimit, ARCH_CONVERT)))) {
+			INT_SET(d->d_itimer, ARCH_CONVERT,
+				get_seconds() + XFS_QI_ITIMELIMIT(mp));
 		}
 	} else {
 		if ((INT_ISZERO(d->d_ino_softlimit, ARCH_CONVERT) ||
-		    (INT_GET(d->d_icount, ARCH_CONVERT) < INT_GET(d->d_ino_softlimit, ARCH_CONVERT)))  &&
+		    (INT_GET(d->d_icount, ARCH_CONVERT) <
+				INT_GET(d->d_ino_softlimit, ARCH_CONVERT)))  &&
 		    (INT_ISZERO(d->d_ino_hardlimit, ARCH_CONVERT) ||
-		    (INT_GET(d->d_icount, ARCH_CONVERT) < INT_GET(d->d_ino_hardlimit, ARCH_CONVERT)))) {
+		    (INT_GET(d->d_icount, ARCH_CONVERT) <
+				INT_GET(d->d_ino_hardlimit, ARCH_CONVERT)))) {
 			INT_ZERO(d->d_itimer, ARCH_CONVERT);
+		}
+	}
+
+	if (INT_ISZERO(d->d_rtbtimer, ARCH_CONVERT)) {
+		if ((INT_GET(d->d_rtb_softlimit, ARCH_CONVERT) &&
+		    (INT_GET(d->d_rtbcount, ARCH_CONVERT) >=
+				INT_GET(d->d_rtb_softlimit, ARCH_CONVERT))) ||
+		    (INT_GET(d->d_rtb_hardlimit, ARCH_CONVERT) &&
+		    (INT_GET(d->d_rtbcount, ARCH_CONVERT) >=
+				INT_GET(d->d_rtb_hardlimit, ARCH_CONVERT)))) {
+			INT_SET(d->d_rtbtimer, ARCH_CONVERT,
+				get_seconds() + XFS_QI_RTBTIMELIMIT(mp));
+		}
+	} else {
+		if ((INT_ISZERO(d->d_rtb_softlimit, ARCH_CONVERT) ||
+		    (INT_GET(d->d_rtbcount, ARCH_CONVERT) <
+				INT_GET(d->d_rtb_softlimit, ARCH_CONVERT))) &&
+		    (INT_ISZERO(d->d_rtb_hardlimit, ARCH_CONVERT) ||
+		    (INT_GET(d->d_rtbcount, ARCH_CONVERT) <
+				INT_GET(d->d_rtb_hardlimit, ARCH_CONVERT)))) {
+			INT_ZERO(d->d_rtbtimer, ARCH_CONVERT);
 		}
 	}
 }
diff -Nru a/fs/xfs/quota/xfs_dquot.h b/fs/xfs/quota/xfs_dquot.h
--- a/fs/xfs/quota/xfs_dquot.h	2003-10-29 10:49:07 -08:00
+++ b/fs/xfs/quota/xfs_dquot.h	2004-08-26 23:21:36 -07:00
@@ -209,6 +209,8 @@
 extern void		xfs_qm_dqflock_pushbuf_wait(xfs_dquot_t *dqp);
 extern void		xfs_qm_adjust_dqtimers(xfs_mount_t *,
 					xfs_disk_dquot_t *);
+extern void		xfs_qm_adjust_dqlimits(xfs_mount_t *,
+					xfs_disk_dquot_t *);
 extern int		xfs_qm_dqwarn(xfs_disk_dquot_t *, uint);
 extern int		xfs_qm_dqget(xfs_mount_t *, xfs_inode_t *,
 					xfs_dqid_t, uint, uint, xfs_dquot_t **);
diff -Nru a/fs/xfs/quota/xfs_qm.c b/fs/xfs/quota/xfs_qm.c
--- a/fs/xfs/quota/xfs_qm.c	2004-05-20 19:59:18 -07:00
+++ b/fs/xfs/quota/xfs_qm.c	2004-08-26 23:21:36 -07:00
@@ -229,11 +229,8 @@
 	 */
 	XFS_QM_LOCK(xfs_Gqm);
 
-	if (xfs_Gqm == NULL) {
-		if ((xfs_Gqm = xfs_Gqm_init()) == NULL) {
-			return (XFS_ERROR(EINVAL));
-		}
-	}
+	if (xfs_Gqm == NULL)
+		xfs_Gqm = xfs_Gqm_init();
 	/*
 	 * We can keep a list of all filesystems with quotas mounted for
 	 * debugging and statistical purposes, but ...
@@ -1216,22 +1213,46 @@
 			     XFS_QMOPT_DQSUSER|XFS_QMOPT_DOWARN,
 			     &dqp);
 	if (! error) {
+		xfs_disk_dquot_t	*ddqp = &dqp->q_core;
+
 		/*
 		 * The warnings and timers set the grace period given to
 		 * a user or group before he or she can not perform any
 		 * more writing. If it is zero, a default is used.
 		 */
-		qinf->qi_btimelimit = INT_GET(dqp->q_core.d_btimer, ARCH_CONVERT) ?
-			INT_GET(dqp->q_core.d_btimer, ARCH_CONVERT) : XFS_QM_BTIMELIMIT;
-		qinf->qi_itimelimit = INT_GET(dqp->q_core.d_itimer, ARCH_CONVERT) ?
-			INT_GET(dqp->q_core.d_itimer, ARCH_CONVERT) : XFS_QM_ITIMELIMIT;
-		qinf->qi_rtbtimelimit = INT_GET(dqp->q_core.d_rtbtimer, ARCH_CONVERT) ?
-			INT_GET(dqp->q_core.d_rtbtimer, ARCH_CONVERT) : XFS_QM_RTBTIMELIMIT;
-		qinf->qi_bwarnlimit = INT_GET(dqp->q_core.d_bwarns, ARCH_CONVERT) ?
-			INT_GET(dqp->q_core.d_bwarns, ARCH_CONVERT) : XFS_QM_BWARNLIMIT;
-		qinf->qi_iwarnlimit = INT_GET(dqp->q_core.d_iwarns, ARCH_CONVERT) ?
-			INT_GET(dqp->q_core.d_iwarns, ARCH_CONVERT) : XFS_QM_IWARNLIMIT;
-
+		qinf->qi_btimelimit =
+				INT_GET(ddqp->d_btimer, ARCH_CONVERT) ?
+				INT_GET(ddqp->d_btimer, ARCH_CONVERT) :
+				XFS_QM_BTIMELIMIT;
+		qinf->qi_itimelimit =
+				INT_GET(ddqp->d_itimer, ARCH_CONVERT) ?
+				INT_GET(ddqp->d_itimer, ARCH_CONVERT) :
+				XFS_QM_ITIMELIMIT;
+		qinf->qi_rtbtimelimit =
+				INT_GET(ddqp->d_rtbtimer, ARCH_CONVERT) ?
+				INT_GET(ddqp->d_rtbtimer, ARCH_CONVERT) :
+				XFS_QM_RTBTIMELIMIT;
+		qinf->qi_bwarnlimit =
+				INT_GET(ddqp->d_bwarns, ARCH_CONVERT) ?
+				INT_GET(ddqp->d_bwarns, ARCH_CONVERT) :
+				XFS_QM_BWARNLIMIT;
+		qinf->qi_iwarnlimit =
+				INT_GET(ddqp->d_iwarns, ARCH_CONVERT) ?
+				INT_GET(ddqp->d_iwarns, ARCH_CONVERT) :
+				XFS_QM_IWARNLIMIT;
+		qinf->qi_bhardlimit =
+				INT_GET(ddqp->d_blk_hardlimit, ARCH_CONVERT);
+		qinf->qi_bsoftlimit =
+				INT_GET(ddqp->d_blk_softlimit, ARCH_CONVERT);
+		qinf->qi_ihardlimit =
+				INT_GET(ddqp->d_ino_hardlimit, ARCH_CONVERT);
+		qinf->qi_isoftlimit =
+				INT_GET(ddqp->d_ino_softlimit, ARCH_CONVERT);
+		qinf->qi_rtbhardlimit =
+				INT_GET(ddqp->d_rtb_hardlimit, ARCH_CONVERT);
+		qinf->qi_rtbsoftlimit =
+				INT_GET(ddqp->d_rtb_softlimit, ARCH_CONVERT);
+ 
 		/*
 		 * We sent the XFS_QMOPT_DQSUSER flag to dqget because
 		 * we don't want this dquot cached. We haven't done a
@@ -1691,10 +1712,12 @@
 	}
 
 	/*
-	 * Adjust the timers since we just changed usages
+	 * Set default limits, adjust timers (since we changed usages)
 	 */
-	if (! XFS_IS_SUSER_DQUOT(dqp))
+	if (! XFS_IS_SUSER_DQUOT(dqp)) {
+		xfs_qm_adjust_dqlimits(dqp->q_mount, &dqp->q_core);
 		xfs_qm_adjust_dqtimers(dqp->q_mount, &dqp->q_core);
+	}
 
 	dqp->dq_flags |= XFS_DQ_DIRTY;
 }
@@ -1735,7 +1758,7 @@
 xfs_qm_dqusage_adjust(
 	xfs_mount_t	*mp,		/* mount point for filesystem */
 	xfs_ino_t	ino,		/* inode number to get data for */
-	void		*buffer,	/* not used */
+	void		__user *buffer,	/* not used */
 	int		ubsize,		/* not used */
 	void		*private_data,	/* not used */
 	xfs_daddr_t	bno,		/* starting block of inode cluster */
diff -Nru a/fs/xfs/quota/xfs_qm.h b/fs/xfs/quota/xfs_qm.h
--- a/fs/xfs/quota/xfs_qm.h	2003-10-06 08:47:28 -07:00
+++ b/fs/xfs/quota/xfs_qm.h	2004-08-26 23:21:36 -07:00
@@ -136,9 +136,14 @@
 	xfs_qwarncnt_t	 qi_bwarnlimit;	 /* limit for num warnings */
 	xfs_qwarncnt_t	 qi_iwarnlimit;	 /* limit for num warnings */
 	mutex_t		 qi_quotaofflock;/* to serialize quotaoff */
-	/* Some useful precalculated constants */
 	xfs_filblks_t	 qi_dqchunklen;	 /* # BBs in a chunk of dqs */
 	uint		 qi_dqperchunk;	 /* # ondisk dqs in above chunk */
+	xfs_qcnt_t	 qi_bhardlimit;	 /* default data blk hard limit */
+	xfs_qcnt_t	 qi_bsoftlimit;	 /* default data blk soft limit */
+	xfs_qcnt_t	 qi_ihardlimit;	 /* default inode count hard limit */
+	xfs_qcnt_t	 qi_isoftlimit;	 /* default inode count soft limit */
+	xfs_qcnt_t	 qi_rtbhardlimit;/* default realtime blk hard limit */
+	xfs_qcnt_t	 qi_rtbsoftlimit;/* default realtime blk soft limit */
 } xfs_quotainfo_t;
 
 
diff -Nru a/fs/xfs/quota/xfs_qm_stats.c b/fs/xfs/quota/xfs_qm_stats.c
--- a/fs/xfs/quota/xfs_qm_stats.c	2003-06-27 11:04:26 -07:00
+++ b/fs/xfs/quota/xfs_qm_stats.c	2004-08-18 18:13:41 -07:00
@@ -137,8 +137,8 @@
 void
 xfs_qm_init_procfs(void)
 {
-	create_proc_read_entry("fs/xfs/xqmstat", 0, 0, xfs_qm_read_stats, NULL);
-	create_proc_read_entry("fs/xfs/xqm", 0, 0, xfs_qm_read_xfsquota, NULL);
+	create_proc_read_entry("fs/xfs/xqmstat", 0, NULL, xfs_qm_read_stats, NULL);
+	create_proc_read_entry("fs/xfs/xqm", 0, NULL, xfs_qm_read_xfsquota, NULL);
 }
 
 void
diff -Nru a/fs/xfs/quota/xfs_qm_syscalls.c b/fs/xfs/quota/xfs_qm_syscalls.c
--- a/fs/xfs/quota/xfs_qm_syscalls.c	2004-05-20 19:59:18 -07:00
+++ b/fs/xfs/quota/xfs_qm_syscalls.c	2004-08-26 23:21:36 -07:00
@@ -648,8 +648,11 @@
 	if (hard == 0 || hard >= soft) {
 		INT_SET(ddq->d_blk_hardlimit, ARCH_CONVERT, hard);
 		INT_SET(ddq->d_blk_softlimit, ARCH_CONVERT, soft);
-	}
-	else {
+		if (id == 0) {
+			mp->m_quotainfo->qi_bhardlimit = hard;
+			mp->m_quotainfo->qi_bsoftlimit = soft;
+		}
+	} else {
 		qdprintk("blkhard %Ld < blksoft %Ld\n", hard, soft);
 	}
 	hard = (newlim->d_fieldmask & FS_DQ_RTBHARD) ?
@@ -661,40 +664,49 @@
 	if (hard == 0 || hard >= soft) {
 		INT_SET(ddq->d_rtb_hardlimit, ARCH_CONVERT, hard);
 		INT_SET(ddq->d_rtb_softlimit, ARCH_CONVERT, soft);
-	}
-	else
+		if (id == 0) {
+			mp->m_quotainfo->qi_rtbhardlimit = hard;
+			mp->m_quotainfo->qi_rtbsoftlimit = soft;
+		}
+	} else {
 		qdprintk("rtbhard %Ld < rtbsoft %Ld\n", hard, soft);
+	}
 
 	hard = (newlim->d_fieldmask & FS_DQ_IHARD) ?
 		(xfs_qcnt_t) newlim->d_ino_hardlimit :
-		INT_GET(ddq->d_ino_hardlimit, ARCH_CONVERT);
+			INT_GET(ddq->d_ino_hardlimit, ARCH_CONVERT);
 	soft = (newlim->d_fieldmask & FS_DQ_ISOFT) ?
 		(xfs_qcnt_t) newlim->d_ino_softlimit :
-		INT_GET(ddq->d_ino_softlimit, ARCH_CONVERT);
+			INT_GET(ddq->d_ino_softlimit, ARCH_CONVERT);
 	if (hard == 0 || hard >= soft) {
 		INT_SET(ddq->d_ino_hardlimit, ARCH_CONVERT, hard);
 		INT_SET(ddq->d_ino_softlimit, ARCH_CONVERT, soft);
-	}
-	else
+		if (id == 0) {
+			mp->m_quotainfo->qi_ihardlimit = hard;
+			mp->m_quotainfo->qi_isoftlimit = soft;
+		}
+	} else {
 		qdprintk("ihard %Ld < isoft %Ld\n", hard, soft);
+	}
 
 	if (id == 0) {
 		/*
 		 * Timelimits for the super user set the relative time
 		 * the other users can be over quota for this file system.
-		 * If it is zero a default is used.
+		 * If it is zero a default is used.  Ditto for the default
+		 * soft and hard limit values (already done, above).
 		 */
 		if (newlim->d_fieldmask & FS_DQ_BTIMER) {
 			mp->m_quotainfo->qi_btimelimit = newlim->d_btimer;
-			INT_SET(dqp->q_core.d_btimer, ARCH_CONVERT, newlim->d_btimer);
+			INT_SET(ddq->d_btimer, ARCH_CONVERT, newlim->d_btimer);
 		}
 		if (newlim->d_fieldmask & FS_DQ_ITIMER) {
 			mp->m_quotainfo->qi_itimelimit = newlim->d_itimer;
-			INT_SET(dqp->q_core.d_itimer, ARCH_CONVERT, newlim->d_itimer);
+			INT_SET(ddq->d_itimer, ARCH_CONVERT, newlim->d_itimer);
 		}
 		if (newlim->d_fieldmask & FS_DQ_RTBTIMER) {
 			mp->m_quotainfo->qi_rtbtimelimit = newlim->d_rtbtimer;
-			INT_SET(dqp->q_core.d_rtbtimer, ARCH_CONVERT, newlim->d_rtbtimer);
+			INT_SET(ddq->d_rtbtimer, ARCH_CONVERT, newlim->d_rtbtimer);
 		}
 	} else /* if (XFS_IS_QUOTA_ENFORCED(mp)) */ {
 		/*
@@ -1114,7 +1126,7 @@
 	  cmn_err(CE_DEBUG, "%s (#%d)", title, (int) (l)->qh_nelems); \
 	  for (dqp = (xfs_dqtest_t *)(l)->qh_next; dqp != NULL; \
 	       dqp = (xfs_dqtest_t *)dqp->NXT) { \
-		cmn_err(CE_DEBUG, "  %d\. \"%d (%s)\"  bcnt = %d, icnt = %d", \
+		cmn_err(CE_DEBUG, "  %d. \"%d (%s)\"  bcnt = %d, icnt = %d", \
 			 ++i, dqp->d_id, DQFLAGTO_TYPESTR(dqp),	     \
 			 dqp->d_bcount, dqp->d_icount); } \
 }
@@ -1299,7 +1311,7 @@
 xfs_qm_internalqcheck_adjust(
 	xfs_mount_t	*mp,		/* mount point for filesystem */
 	xfs_ino_t	ino,		/* inode number to get data for */
-	void		*buffer,	/* not used */
+	void		__user *buffer,	/* not used */
 	int		ubsize,		/* not used */
 	void		*private_data,	/* not used */
 	xfs_daddr_t	bno,		/* starting block of inode cluster */
diff -Nru a/fs/xfs/quota/xfs_trans_dquot.c b/fs/xfs/quota/xfs_trans_dquot.c
--- a/fs/xfs/quota/xfs_trans_dquot.c	2003-11-24 09:41:25 -08:00
+++ b/fs/xfs/quota/xfs_trans_dquot.c	2004-08-26 23:21:36 -07:00
@@ -452,9 +452,13 @@
 				INT_MOD(d->d_rtbcount, ARCH_CONVERT, (xfs_qcnt_t)totalrtbdelta);
 
 			/*
+			 * Get any default limits in use.
 			 * Start/reset the timer(s) if needed.
 			 */
-			xfs_qm_adjust_dqtimers(tp->t_mountp, d);
+			if (!INT_ISZERO(d->d_id, ARCH_CONVERT)) {
+				xfs_qm_adjust_dqlimits(tp->t_mountp, d);
+				xfs_qm_adjust_dqtimers(tp->t_mountp, d);
+			}
 
 			dqp->dq_flags |= XFS_DQ_DIRTY;
 			/*
@@ -625,6 +629,7 @@
 STATIC int
 xfs_trans_dqresv(
 	xfs_trans_t	*tp,
+	xfs_mount_t	*mp,
 	xfs_dquot_t	*dqp,
 	long		nblks,
 	long		ninos,
@@ -635,6 +640,7 @@
 	xfs_qcnt_t	softlimit;
 	time_t		btimer;
 	xfs_qcnt_t	*resbcountp;
+	xfs_quotainfo_t	*q = mp->m_quotainfo;
 
 	if (! (flags & XFS_QMOPT_DQLOCK)) {
 		xfs_dqlock(dqp);
@@ -642,13 +648,21 @@
 	ASSERT(XFS_DQ_IS_LOCKED(dqp));
 	if (flags & XFS_TRANS_DQ_RES_BLKS) {
 		hardlimit = INT_GET(dqp->q_core.d_blk_hardlimit, ARCH_CONVERT);
+		if (!hardlimit)
+			hardlimit = q->qi_bhardlimit;
 		softlimit = INT_GET(dqp->q_core.d_blk_softlimit, ARCH_CONVERT);
+		if (!softlimit)
+			softlimit = q->qi_bsoftlimit;
 		btimer = INT_GET(dqp->q_core.d_btimer, ARCH_CONVERT);
 		resbcountp = &dqp->q_res_bcount;
 	} else {
 		ASSERT(flags & XFS_TRANS_DQ_RES_RTBLKS);
 		hardlimit = INT_GET(dqp->q_core.d_rtb_hardlimit, ARCH_CONVERT);
+		if (!hardlimit)
+			hardlimit = q->qi_rtbhardlimit;
 		softlimit = INT_GET(dqp->q_core.d_rtb_softlimit, ARCH_CONVERT);
+		if (!softlimit)
+			softlimit = q->qi_rtbsoftlimit;
 		btimer = INT_GET(dqp->q_core.d_rtbtimer, ARCH_CONVERT);
 		resbcountp = &dqp->q_res_rtbcount;
 	}
@@ -689,14 +703,18 @@
 			}
 		}
 		if (ninos > 0) {
-			if (INT_GET(dqp->q_core.d_ino_hardlimit, ARCH_CONVERT) > 0ULL &&
-			    INT_GET(dqp->q_core.d_icount, ARCH_CONVERT) >=
-			    INT_GET(dqp->q_core.d_ino_hardlimit, ARCH_CONVERT)) {
+			hardlimit = INT_GET(dqp->q_core.d_ino_hardlimit, ARCH_CONVERT);
+			if (!hardlimit)
+				hardlimit = q->qi_ihardlimit;
+			softlimit = INT_GET(dqp->q_core.d_ino_softlimit, ARCH_CONVERT);
+			if (!softlimit)
+				softlimit = q->qi_isoftlimit;
+			if (hardlimit > 0ULL &&
+			    INT_GET(dqp->q_core.d_icount, ARCH_CONVERT) >= hardlimit) {
 				error = EDQUOT;
 				goto error_return;
-			} else if (INT_GET(dqp->q_core.d_ino_softlimit, ARCH_CONVERT) > 0ULL &&
-				   INT_GET(dqp->q_core.d_icount, ARCH_CONVERT) >=
-				   INT_GET(dqp->q_core.d_ino_softlimit, ARCH_CONVERT)) {
+			} else if (softlimit > 0ULL &&
+				   INT_GET(dqp->q_core.d_icount, ARCH_CONVERT) >= softlimit) {
 				/*
 				 * If timer or warnings has expired,
 				 * return EDQUOT
@@ -786,19 +804,20 @@
 	resvd = 0;
 
 	if (udqp) {
-		if (xfs_trans_dqresv(tp, udqp, nblks, ninos, flags))
+		if (xfs_trans_dqresv(tp, mp, udqp, nblks, ninos, flags))
 			return (EDQUOT);
 		resvd = 1;
 	}
 
 	if (gdqp) {
-		if (xfs_trans_dqresv(tp, gdqp, nblks, ninos, flags)) {
+		if (xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, flags)) {
 			/*
 			 * can't do it, so backout previous reservation
 			 */
 			if (resvd) {
-				xfs_trans_dqresv(tp, udqp,  -nblks, -ninos,
-						 flags);
+				flags |= XFS_QMOPT_FORCE_RES;
+				xfs_trans_dqresv(tp, mp, udqp,
+						 -nblks, -ninos, flags);
 			}
 			return (EDQUOT);
 		}
diff -Nru a/fs/xfs/support/ktrace.c b/fs/xfs/support/ktrace.c
--- a/fs/xfs/support/ktrace.c	2004-02-01 21:56:07 -08:00
+++ b/fs/xfs/support/ktrace.c	2004-08-18 18:07:09 -07:00
@@ -30,14 +30,7 @@
  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  */
 
-#include <linux/types.h>
-#include <linux/slab.h>
-
-#include <xfs_types.h>
-#include "kmem.h"
-#include "spin.h"
-#include "debug.h"
-#include "ktrace.h"
+#include <xfs.h>
 
 static kmem_zone_t *ktrace_hdr_zone;
 static kmem_zone_t *ktrace_ent_zone;
diff -Nru a/fs/xfs/support/move.c b/fs/xfs/support/move.c
--- a/fs/xfs/support/move.c	2003-10-07 18:34:06 -07:00
+++ b/fs/xfs/support/move.c	2004-08-18 18:07:09 -07:00
@@ -30,13 +30,7 @@
  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  */
 
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <asm/uaccess.h>
-
-#include <xfs_types.h>
-#include "debug.h"
-#include "move.h"
+#include <xfs.h>
 
 /* Read from kernel buffer at src to user/kernel buffer defined
  * by the uio structure. Advance the pointer in the uio struct
diff -Nru a/fs/xfs/xfs_acl.h b/fs/xfs/xfs_acl.h
--- a/fs/xfs/xfs_acl.h	2004-06-17 17:52:33 -07:00
+++ b/fs/xfs/xfs_acl.h	2004-08-18 16:55:19 -07:00
@@ -95,7 +95,7 @@
 #define _ACL_XFS_IACCESS(i,m,c) (XFS_IFORK_Q(i) ? xfs_acl_iaccess(i,m,c) : -1)
 
 #define _ACL_ALLOC(a)		((a) = kmem_zone_alloc(xfs_acl_zone, KM_SLEEP))
-#define _ACL_FREE(a)		((a)? kmem_zone_free(xfs_acl_zone, (a)) : 0)
+#define _ACL_FREE(a)		((a)? kmem_zone_free(xfs_acl_zone, (a)):(void)0)
 
 #else
 #define xfs_acl_zone_init(zone,name)
diff -Nru a/fs/xfs/xfs_alloc.c b/fs/xfs/xfs_alloc.c
--- a/fs/xfs/xfs_alloc.c	2004-03-02 18:33:14 -08:00
+++ b/fs/xfs/xfs_alloc.c	2004-08-18 18:13:41 -07:00
@@ -665,7 +665,7 @@
 	 * Allocate/initialize a cursor for the by-number freespace btree.
 	 */
 	bno_cur = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_BNO, 0, 0);
+		args->agno, XFS_BTNUM_BNO, NULL, 0);
 	/*
 	 * Lookup bno and minlen in the btree (minlen is irrelevant, really).
 	 * Look for the closest free block <= bno, it must contain bno
@@ -721,7 +721,7 @@
 	 * Allocate/initialize a cursor for the by-size btree.
 	 */
 	cnt_cur = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_CNT, 0, 0);
+		args->agno, XFS_BTNUM_CNT, NULL, 0);
 	ASSERT(args->agbno + args->len <=
 		INT_GET(XFS_BUF_TO_AGF(args->agbp)->agf_length,
 			ARCH_CONVERT));
@@ -788,7 +788,7 @@
 	 * Get a cursor for the by-size btree.
 	 */
 	cnt_cur = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_CNT, 0, 0);
+		args->agno, XFS_BTNUM_CNT, NULL, 0);
 	ltlen = 0;
 	bno_cur_lt = bno_cur_gt = NULL;
 	/*
@@ -916,7 +916,7 @@
 		 * Set up a cursor for the by-bno tree.
 		 */
 		bno_cur_lt = xfs_btree_init_cursor(args->mp, args->tp,
-			args->agbp, args->agno, XFS_BTNUM_BNO, 0, 0);
+			args->agbp, args->agno, XFS_BTNUM_BNO, NULL, 0);
 		/*
 		 * Fix up the btree entries.
 		 */
@@ -944,7 +944,7 @@
 	 * Allocate and initialize the cursor for the leftward search.
 	 */
 	bno_cur_lt = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_BNO, 0, 0);
+		args->agno, XFS_BTNUM_BNO, NULL, 0);
 	/*
 	 * Lookup <= bno to find the leftward search's starting point.
 	 */
@@ -956,7 +956,7 @@
 		 * search.
 		 */
 		bno_cur_gt = bno_cur_lt;
-		bno_cur_lt = 0;
+		bno_cur_lt = NULL;
 	}
 	/*
 	 * Found something.  Duplicate the cursor for the rightward search.
@@ -1301,7 +1301,7 @@
 	 * Allocate and initialize a cursor for the by-size btree.
 	 */
 	cnt_cur = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_CNT, 0, 0);
+		args->agno, XFS_BTNUM_CNT, NULL, 0);
 	bno_cur = NULL;
 	/*
 	 * Look for an entry >= maxlen+alignment-1 blocks.
@@ -1406,7 +1406,7 @@
 	 * Allocate and initialize a cursor for the by-block tree.
 	 */
 	bno_cur = xfs_btree_init_cursor(args->mp, args->tp, args->agbp,
-		args->agno, XFS_BTNUM_BNO, 0, 0);
+		args->agno, XFS_BTNUM_BNO, NULL, 0);
 	if ((error = xfs_alloc_fixup_trees(cnt_cur, bno_cur, fbno, flen,
 			rbno, rlen, XFSA_FIXUP_CNT_OK)))
 		goto error0;
@@ -1553,7 +1553,7 @@
 	/*
 	 * Allocate and initialize a cursor for the by-block btree.
 	 */
-	bno_cur = xfs_btree_init_cursor(mp, tp, agbp, agno, XFS_BTNUM_BNO, 0,
+	bno_cur = xfs_btree_init_cursor(mp, tp, agbp, agno, XFS_BTNUM_BNO, NULL,
 		0);
 	cnt_cur = NULL;
 	/*
@@ -1613,7 +1613,7 @@
 	/*
 	 * Now allocate and initialize a cursor for the by-size tree.
 	 */
-	cnt_cur = xfs_btree_init_cursor(mp, tp, agbp, agno, XFS_BTNUM_CNT, 0,
+	cnt_cur = xfs_btree_init_cursor(mp, tp, agbp, agno, XFS_BTNUM_CNT, NULL,
 		0);
 	/*
 	 * Have both left and right contiguous neighbors.
diff -Nru a/fs/xfs/xfs_alloc_btree.c b/fs/xfs/xfs_alloc_btree.c
--- a/fs/xfs/xfs_alloc_btree.c	2003-09-22 12:49:14 -07:00
+++ b/fs/xfs/xfs_alloc_btree.c	2004-08-18 18:13:41 -07:00
@@ -263,7 +263,7 @@
 			/*
 			 * Update the cursor so there's one fewer level.
 			 */
-			xfs_btree_setbuf(cur, level, 0);
+			xfs_btree_setbuf(cur, level, NULL);
 			cur->bc_nlevels--;
 		} else if (level > 0 &&
 			   (error = xfs_alloc_decrement(cur, level, &i)))
diff -Nru a/fs/xfs/xfs_attr.c b/fs/xfs/xfs_attr.c
--- a/fs/xfs/xfs_attr.c	2004-06-17 18:47:32 -07:00
+++ b/fs/xfs/xfs_attr.c	2004-08-18 20:26:29 -07:00
@@ -104,7 +104,6 @@
 STATIC int xfs_attr_rmtval_remove(xfs_da_args_t *args);
 
 #define ATTR_RMTVALUE_MAPSIZE	1	/* # of map entries at once */
-#define ATTR_RMTVALUE_TRANSBLKS	8	/* max # of blks in a transaction */
 
 #if defined(XFS_ATTR_TRACE)
 ktrace_t *xfs_attr_trace_buf;
diff -Nru a/fs/xfs/xfs_bmap.c b/fs/xfs/xfs_bmap.c
--- a/fs/xfs/xfs_bmap.c	2004-05-20 23:43:19 -07:00
+++ b/fs/xfs/xfs_bmap.c	2004-08-26 23:24:46 -07:00
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -203,8 +203,6 @@
 xfs_bmap_check_extents(
 	xfs_inode_t		*ip,		/* incore inode pointer */
 	int			whichfork);	/* data or attr fork */
-#else
-#define	xfs_bmap_check_extents(ip,w)
 #endif
 
 /*
@@ -3429,6 +3427,20 @@
 	int		high;		/* high index of binary search */
 	int		low;		/* low index of binary search */
 
+	/*
+	 * Initialize the extent entry structure to catch access to
+	 * uninitialized br_startblock field.
+	 */
+	got.br_startoff = 0xffa5a5a5a5a5a5a5LL;
+	got.br_blockcount = 0xa55a5a5a5a5a5a5aLL;
+	got.br_state = XFS_EXT_INVALID;
+
+#if XFS_BIG_BLKNOS
+	got.br_startblock = 0xffffa5a5a5a5a5a5LL;
+#else
+	got.br_startblock = 0xffffa5a5;
+#endif
+
 	if (lastx != NULLEXTNUM && lastx < nextents)
 		ep = base + lastx;
 	else
@@ -3527,6 +3539,8 @@
 	xfs_bmbt_rec_t  *base;          /* base of extent list */
 	xfs_extnum_t    lastx;          /* last extent index used */
 	xfs_extnum_t    nextents;       /* extent list size */
+	xfs_bmbt_rec_t  *ep;            /* extent list entry pointer */
+	int		rt;		/* realtime flag    */
 
 	XFS_STATS_INC(xs_look_exlist);
 	ifp = XFS_IFORK_PTR(ip, whichfork);
@@ -3534,8 +3548,18 @@
 	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
 	base = &ifp->if_u1.if_extents[0];
 
-	return xfs_bmap_do_search_extents(base, lastx, nextents, bno, eofp,
+	ep = xfs_bmap_do_search_extents(base, lastx, nextents, bno, eofp,
 					  lastxp, gotp, prevp);
+	rt = ip->i_d.di_flags & XFS_DIFLAG_REALTIME;
+	if(!rt && !gotp->br_startblock && (*lastxp != NULLEXTNUM)) {
+                cmn_err(CE_PANIC,"Access to block zero: fs: <%s> inode: %lld "
+			"start_block : %llx start_off : %llx blkcnt : %llx "
+			"extent-state : %x \n",
+			(ip->i_mount)->m_fsname,(long long)ip->i_ino,
+			gotp->br_startblock, gotp->br_startoff,
+			gotp->br_blockcount,gotp->br_state);
+        }
+        return ep;
 }
 
 
@@ -4689,8 +4713,41 @@
 					}
 					break;
 				}
+
+				/*
+				 * Split changing sb for alen and indlen since
+				 * they could be coming from different places.
+				 */
+				if (ip->i_d.di_flags & XFS_DIFLAG_REALTIME) {
+					xfs_extlen_t	extsz;
+					xfs_extlen_t	ralen;
+					if (!(extsz = ip->i_d.di_extsize))
+						extsz = mp->m_sb.sb_rextsize;
+					ralen = roundup(alen, extsz);
+					ralen = ralen / mp->m_sb.sb_rextsize;
+					if (xfs_mod_incore_sb(mp,
+						XFS_SBS_FREXTENTS,
+						-(ralen), rsvd)) {
+						if (XFS_IS_QUOTA_ON(ip->i_mount))
+							XFS_TRANS_UNRESERVE_BLKQUOTA(
+						     		mp, NULL, ip,
+								(long)alen);
+						break;
+					}
+				} else {
+					if (xfs_mod_incore_sb(mp,
+							      XFS_SBS_FDBLOCKS,
+							      -(alen), rsvd)) {
+						if (XFS_IS_QUOTA_ON(ip->i_mount))
+							XFS_TRANS_UNRESERVE_BLKQUOTA(
+								mp, NULL, ip,
+								(long)alen);
+						break;
+					}
+				}
+
 				if (xfs_mod_incore_sb(mp, XFS_SBS_FDBLOCKS,
-						-(alen + indlen), rsvd)) {
+						-(indlen), rsvd)) {
 					XFS_TRANS_UNRESERVE_BLKQUOTA(
 						mp, NULL, ip, (long)alen);
 					break;
@@ -5445,7 +5502,7 @@
 xfs_getbmap(
 	bhv_desc_t		*bdp,		/* XFS behavior descriptor*/
 	struct getbmap		*bmv,		/* user bmap structure */
-	void			*ap,		/* pointer to user's array */
+	void			__user *ap,	/* pointer to user's array */
 	int			interface)	/* interface flags */
 {
 	__int64_t		bmvend;		/* last block requested */
@@ -5634,8 +5691,10 @@
 					(__int64_t)(bmvend - bmv->bmv_offset));
 				bmv->bmv_entries++;
 				ap = (interface & BMV_IF_EXTENDED) ?
-					(void *)((struct getbmapx *)ap + 1) :
-					(void *)((struct getbmap *)ap + 1);
+						(void __user *)
+					((struct getbmapx __user *)ap + 1) :
+						(void __user *)
+					((struct getbmap __user *)ap + 1);
 			}
 		}
 	} while (nmap && nexleft && bmv->bmv_length);
diff -Nru a/fs/xfs/xfs_bmap.h b/fs/xfs/xfs_bmap.h
--- a/fs/xfs/xfs_bmap.h	2004-05-20 23:43:19 -07:00
+++ b/fs/xfs/xfs_bmap.h	2004-08-18 20:37:21 -07:00
@@ -328,7 +328,7 @@
 xfs_getbmap(
 	bhv_desc_t		*bdp,		/* XFS behavior descriptor*/
 	struct getbmap		*bmv,		/* user bmap structure */
-	void			*ap,		/* pointer to user's array */
+	void			__user *ap,	/* pointer to user's array */
 	int			iflags);	/* interface flags */
 
 /*
diff -Nru a/fs/xfs/xfs_bmap_btree.c b/fs/xfs/xfs_bmap_btree.c
--- a/fs/xfs/xfs_bmap_btree.c	2004-05-20 21:57:05 -07:00
+++ b/fs/xfs/xfs_bmap_btree.c	2004-08-18 18:13:41 -07:00
@@ -1953,7 +1953,7 @@
 		*bpp = cur->bc_bufs[level];
 		rval = XFS_BUF_TO_BMBT_BLOCK(*bpp);
 	} else {
-		*bpp = 0;
+		*bpp = NULL;
 		ifp = XFS_IFORK_PTR(cur->bc_private.b.ip,
 			cur->bc_private.b.whichfork);
 		rval = ifp->if_broot;
diff -Nru a/fs/xfs/xfs_bmap_btree.h b/fs/xfs/xfs_bmap_btree.h
--- a/fs/xfs/xfs_bmap_btree.h	2004-05-20 21:57:05 -07:00
+++ b/fs/xfs/xfs_bmap_btree.h	2004-08-18 19:01:39 -07:00
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 2000,2002-2004 Silicon Graphics, Inc.  All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -153,7 +153,7 @@
  */
 typedef enum {
 	XFS_EXT_NORM, XFS_EXT_UNWRITTEN,
-	XFS_EXT_DMAPI_OFFLINE
+	XFS_EXT_DMAPI_OFFLINE, XFS_EXT_INVALID
 } xfs_exntst_t;
 
 /*
diff -Nru a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
--- a/fs/xfs/xfs_buf_item.c	2004-05-20 20:48:49 -07:00
+++ b/fs/xfs/xfs_buf_item.c	2004-08-18 20:26:29 -07:00
@@ -55,8 +55,6 @@
 #include "xfs_error.h"
 
 
-#define	ROUNDUPNBWORD(x)	(((x) + (NBWORD - 1)) & ~(NBWORD - 1))
-
 kmem_zone_t	*xfs_buf_item_zone;
 
 #ifdef XFS_TRANS_DEBUG
diff -Nru a/fs/xfs/xfs_da_btree.c b/fs/xfs/xfs_da_btree.c
--- a/fs/xfs/xfs_da_btree.c	2004-05-20 20:48:49 -07:00
+++ b/fs/xfs/xfs_da_btree.c	2004-08-18 18:13:41 -07:00
@@ -2092,7 +2092,7 @@
 	int		caller,
 	inst_t		*ra)
 {
-	xfs_buf_t	*bp = 0;
+	xfs_buf_t	*bp = NULL;
 	xfs_buf_t	**bplist;
 	int		error=0;
 	int		i;
diff -Nru a/fs/xfs/xfs_dfrag.c b/fs/xfs/xfs_dfrag.c
--- a/fs/xfs/xfs_dfrag.c	2004-03-31 14:22:54 -08:00
+++ b/fs/xfs/xfs_dfrag.c	2004-08-18 20:37:21 -07:00
@@ -65,7 +65,7 @@
  */
 int
 xfs_swapext(
-	xfs_swapext_t   *sxp)
+	xfs_swapext_t	__user *sxp)
 {
 	xfs_swapext_t	sx;
 	xfs_inode_t     *ip=NULL, *tip=NULL, *ips[2];
diff -Nru a/fs/xfs/xfs_dfrag.h b/fs/xfs/xfs_dfrag.h
--- a/fs/xfs/xfs_dfrag.h	2003-06-27 11:04:26 -07:00
+++ b/fs/xfs/xfs_dfrag.h	2004-08-18 16:55:19 -07:00
@@ -60,7 +60,7 @@
 /*
  * Syscall interface for xfs_swapext
  */
-int	xfs_swapext(struct xfs_swapext *sx);
+int	xfs_swapext(struct xfs_swapext __user *sx);
 
 #endif	/* __KERNEL__ */
 
diff -Nru a/fs/xfs/xfs_dinode.h b/fs/xfs/xfs_dinode.h
--- a/fs/xfs/xfs_dinode.h	2003-10-07 17:24:32 -07:00
+++ b/fs/xfs/xfs_dinode.h	2004-08-18 21:05:36 -07:00
@@ -456,6 +456,7 @@
 #define XFS_DIFLAG_SYNC_BIT      5	/* inode is written synchronously */
 #define XFS_DIFLAG_NOATIME_BIT   6	/* do not update atime */
 #define XFS_DIFLAG_NODUMP_BIT    7	/* do not dump */
+#define XFS_DIFLAG_RTINHERIT_BIT 8	/* create with realtime bit set */
 #define XFS_DIFLAG_REALTIME      (1 << XFS_DIFLAG_REALTIME_BIT)
 #define XFS_DIFLAG_PREALLOC      (1 << XFS_DIFLAG_PREALLOC_BIT)
 #define XFS_DIFLAG_NEWRTBM       (1 << XFS_DIFLAG_NEWRTBM_BIT)
@@ -464,5 +465,11 @@
 #define XFS_DIFLAG_SYNC          (1 << XFS_DIFLAG_SYNC_BIT)
 #define XFS_DIFLAG_NOATIME       (1 << XFS_DIFLAG_NOATIME_BIT)
 #define XFS_DIFLAG_NODUMP        (1 << XFS_DIFLAG_NODUMP_BIT)
+#define XFS_DIFLAG_RTINHERIT     (1 << XFS_DIFLAG_RTINHERIT_BIT)
+
+#define XFS_DIFLAG_ANY \
+	(XFS_DIFLAG_REALTIME | XFS_DIFLAG_PREALLOC | XFS_DIFLAG_NEWRTBM | \
+	 XFS_DIFLAG_IMMUTABLE | XFS_DIFLAG_APPEND | XFS_DIFLAG_SYNC | \
+	 XFS_DIFLAG_NOATIME | XFS_DIFLAG_NODUMP | XFS_DIFLAG_RTINHERIT)
 
 #endif	/* __XFS_DINODE_H__ */
diff -Nru a/fs/xfs/xfs_dir2_trace.c b/fs/xfs/xfs_dir2_trace.c
--- a/fs/xfs/xfs_dir2_trace.c	2004-01-29 22:22:45 -08:00
+++ b/fs/xfs/xfs_dir2_trace.c	2004-08-18 18:13:41 -07:00
@@ -211,7 +211,7 @@
 		(void *)((unsigned long)(args->inumber >> 32)),
 		(void *)((unsigned long)(args->inumber & 0xFFFFFFFF)),
 		(void *)args->dp, (void *)args->trans,
-		(void *)(unsigned long)args->justcheck, (void *)(long)s, 0);
+		(void *)(unsigned long)args->justcheck, (void *)(long)s, NULL);
 }
 
 void
diff -Nru a/fs/xfs/xfs_fs.h b/fs/xfs/xfs_fs.h
--- a/fs/xfs/xfs_fs.h	2004-03-02 17:23:38 -08:00
+++ b/fs/xfs/xfs_fs.h	2004-08-18 21:27:23 -07:00
@@ -76,6 +76,7 @@
 #define XFS_XFLAG_SYNC		0x00000020	/* all writes synchronous */
 #define XFS_XFLAG_NOATIME	0x00000040	/* do not update access time */
 #define XFS_XFLAG_NODUMP	0x00000080	/* do not include in backups */
+#define XFS_XFLAG_RTINHERIT	0x00000100	/* create with rt bit set */
 #define XFS_XFLAG_HASATTR	0x80000000	/* no DIFLAG for this	*/
 
 /*
@@ -164,7 +165,7 @@
 	__s64		l_start;
 	__s64		l_len;		/* len == 0 means until end of file */
 	__s32		l_sysid;
-	pid_t		l_pid;
+	__u32		l_pid;
 	__s32		l_pad[4];	/* reserve area			    */
 } xfs_flock64_t;
 
@@ -313,10 +314,10 @@
  * The user-level BulkStat Request interface structure.
  */
 typedef struct xfs_fsop_bulkreq {
-	__u64		*lastip;	/* last inode # pointer		*/
+	__u64		__user *lastip;	/* last inode # pointer		*/
 	__s32		icount;		/* count of entries in buffer	*/
-	void		*ubuffer;	/* user buffer for inode desc.	*/
-	__s32		*ocount;	/* output count pointer		*/
+	void		__user *ubuffer;/* user buffer for inode desc.	*/
+	__s32		__user *ocount;	/* output count pointer		*/
 } xfs_fsop_bulkreq_t;
 
 
@@ -344,12 +345,12 @@
  */
 typedef struct xfs_fsop_handlereq {
 	__u32		fd;		/* fd for FD_TO_HANDLE		*/
-	void		*path;		/* user pathname		*/
+	void		__user *path;	/* user pathname		*/
 	__u32		oflags;		/* open flags			*/
-	void		*ihandle;	/* user supplied handle		*/
+	void		__user *ihandle;/* user supplied handle		*/
 	__u32		ihandlen;	/* user supplied length		*/
-	void		*ohandle;	/* user buffer for handle	*/
-	__u32		*ohandlen;	/* user buffer length		*/
+	void		__user *ohandle;/* user buffer for handle	*/
+	__u32		__user *ohandlen;/* user buffer length		*/
 } xfs_fsop_handlereq_t;
 
 /*
@@ -360,35 +361,35 @@
  */
 
 typedef struct xfs_fsop_setdm_handlereq {
-	struct xfs_fsop_handlereq hreq; /* handle interface structure */
-	struct fsdmidata *data;		/* DMAPI data to set	      */
+	struct xfs_fsop_handlereq	hreq;	/* handle information	*/
+	struct fsdmidata		__user *data;	/* DMAPI data	*/
 } xfs_fsop_setdm_handlereq_t;
 
 typedef struct xfs_attrlist_cursor {
-	__u32	opaque[4];
+	__u32		opaque[4];
 } xfs_attrlist_cursor_t;
 
 typedef struct xfs_fsop_attrlist_handlereq {
-	struct xfs_fsop_handlereq hreq; /* handle interface structure */
-	struct xfs_attrlist_cursor pos; /* opaque cookie, list offset */
-	__u32 flags;			/* flags, use ROOT/USER names */
-	__u32 buflen;			/* length of buffer supplied  */
-	void *buffer;			/* attrlist data to return    */
+	struct xfs_fsop_handlereq	hreq; /* handle interface structure */
+	struct xfs_attrlist_cursor	pos; /* opaque cookie, list offset */
+	__u32				flags;	/* which namespace to use */
+	__u32				buflen;	/* length of buffer supplied */
+	void				__user *buffer;	/* returned names */
 } xfs_fsop_attrlist_handlereq_t;
 
 typedef struct xfs_attr_multiop {
-	__u32	am_opcode;
-	__s32	am_error;
-	void	*am_attrname;
-	void	*am_attrvalue;
-	__u32	am_length;
-	__u32	am_flags;
+	__u32		am_opcode;
+	__s32		am_error;
+	void		__user *am_attrname;
+	void		__user *am_attrvalue;
+	__u32		am_length;
+	__u32		am_flags;
 } xfs_attr_multiop_t;
 
 typedef struct xfs_fsop_attrmulti_handlereq {
-	struct xfs_fsop_handlereq hreq; /* handle interface structure */
-	__u32 opcount;			/* count of following multiop */
-	struct xfs_attr_multiop *ops;	/* attr_multi data to get/set */
+	struct xfs_fsop_handlereq	hreq; /* handle interface structure */
+	__u32				opcount;/* count of following multiop */
+	struct xfs_attr_multiop		__user *ops; /* attr_multi data */
 } xfs_fsop_attrmulti_handlereq_t;
 
 /*
@@ -443,6 +444,13 @@
 #define XFS_FSOP_GOING_FLAGS_DEFAULT		0x0	/* going down */
 #define XFS_FSOP_GOING_FLAGS_LOGFLUSH		0x1	/* flush log but not data */
 #define XFS_FSOP_GOING_FLAGS_NOLOGFLUSH		0x2	/* don't flush log nor data */
+
+/*
+ * ioctl commands that are used by Linux filesystems
+ */
+#define XFS_IOC_GETXFLAGS	_IOR('f', 1, long)
+#define XFS_IOC_SETXFLAGS	_IOW('f', 2, long)
+#define XFS_IOC_GETVERSION	_IOR('v', 1, long)
 
 /*
  * ioctl commands that replace IRIX fcntl()'s
diff -Nru a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c
--- a/fs/xfs/xfs_fsops.c	2004-05-20 20:53:30 -07:00
+++ b/fs/xfs/xfs_fsops.c	2004-08-26 23:27:59 -07:00
@@ -142,6 +142,7 @@
 	int			dpct;
 	int			error;
 	xfs_agnumber_t		nagcount;
+	xfs_agnumber_t		nagimax = 0;
 	xfs_rfsblock_t		nb, nb_mod;
 	xfs_rfsblock_t		new;
 	xfs_rfsblock_t		nfree;
@@ -183,7 +184,7 @@
 		memset(&mp->m_perag[oagcount], 0,
 			(nagcount - oagcount) * sizeof(xfs_perag_t));
 		mp->m_flags |= XFS_MOUNT_32BITINODES;
-		xfs_initialize_perag(mp, nagcount);
+		nagimax = xfs_initialize_perag(mp, nagcount);
 		up_write(&mp->m_peraglock);
 	}
 	tp = xfs_trans_alloc(mp, XFS_TRANS_GROWFS);
@@ -372,6 +373,9 @@
 	if (error) {
 		return error;
 	}
+	/* New allocation groups fully initialized, so update mount struct */
+	if (nagimax)
+		mp->m_maxagi = nagimax;
 	if (mp->m_sb.sb_imax_pct) {
 		__uint64_t icount = mp->m_sb.sb_dblocks * mp->m_sb.sb_imax_pct;
 		do_div(icount, 100);
@@ -507,9 +511,9 @@
 	__uint64_t              *inval,
 	xfs_fsop_resblks_t      *outval)
 {
-	__uint64_t              lcounter, delta;
-	__uint64_t              request;
-	unsigned long s;
+	__int64_t		lcounter, delta;
+	__uint64_t		request;
+	unsigned long		s;
 
 	/* If inval is null, report current values and return */
 
@@ -536,8 +540,7 @@
 		mp->m_resblks = request;
 	} else {
 		delta = request - mp->m_resblks;
-		lcounter = mp->m_sb.sb_fdblocks;
-		lcounter -= delta;
+		lcounter = mp->m_sb.sb_fdblocks - delta;
 		if (lcounter < 0) {
 			/* We can't satisfy the request, just get what we can */
 			mp->m_resblks += mp->m_sb.sb_fdblocks;
@@ -583,63 +586,22 @@
 }
 
 int
-xfs_fs_freeze(
-	xfs_mount_t	*mp)
-{
-	vfs_t		*vfsp;
-	/*REFERENCED*/
-	int		error;
-
-	vfsp = XFS_MTOVFS(mp);
-
-	/* Stop new writers */
-	xfs_start_freeze(mp, XFS_FREEZE_WRITE);
-
-	/* Flush the refcache */
-	xfs_refcache_purge_mp(mp);
-
-	/* Flush delalloc and delwri data */
-	VFS_SYNC(vfsp, SYNC_DELWRI|SYNC_WAIT, NULL, error);
-
-	/* Pause transaction subsystem */
-	xfs_start_freeze(mp, XFS_FREEZE_TRANS);
-
-	/* Flush any remaining inodes into buffers */
-	VFS_SYNC(vfsp, SYNC_ATTR|SYNC_WAIT, NULL, error);
-
-	/* Push all buffers out to disk */
-	xfs_binval(mp->m_ddev_targp);
-	if (mp->m_rtdev_targp) {
-		xfs_binval(mp->m_rtdev_targp);
-	}
-
-	/* Push the superblock and write an unmount record */
-	xfs_log_unmount_write(mp);
-	xfs_unmountfs_writesb(mp);
-
-	return 0;
-}
-
-int
-xfs_fs_thaw(
-	xfs_mount_t	*mp)
-{
-	xfs_finish_freeze(mp);
-	return 0;
-}
-
-int
 xfs_fs_goingdown(
 	xfs_mount_t	*mp,
 	__uint32_t	inflags)
 {
-	switch (inflags)
-	{
-	case XFS_FSOP_GOING_FLAGS_DEFAULT:
-		xfs_fs_freeze(mp);
-		xfs_force_shutdown(mp, XFS_FORCE_UMOUNT);
-		xfs_fs_thaw(mp);
+	switch (inflags) {
+	case XFS_FSOP_GOING_FLAGS_DEFAULT: {
+		struct vfs *vfsp = XFS_MTOVFS(mp);
+		struct super_block *sb = freeze_bdev(vfsp->vfs_super->s_bdev);
+
+		if (sb) {
+			xfs_force_shutdown(mp, XFS_FORCE_UMOUNT);
+			thaw_bdev(sb->s_bdev, sb);
+		}
+	
 		break;
+	}
 	case XFS_FSOP_GOING_FLAGS_LOGFLUSH:
 		xfs_force_shutdown(mp, XFS_FORCE_UMOUNT);
 		break;
diff -Nru a/fs/xfs/xfs_fsops.h b/fs/xfs/xfs_fsops.h
--- a/fs/xfs/xfs_fsops.h	2004-03-02 17:23:38 -08:00
+++ b/fs/xfs/xfs_fsops.h	2004-08-18 15:54:56 -07:00
@@ -60,14 +60,6 @@
 	xfs_fsop_resblks_t	*outval);
 
 int
-xfs_fs_freeze(
-	xfs_mount_t		*mp);
-
-int
-xfs_fs_thaw(
-	xfs_mount_t		*mp);
-
-int
 xfs_fs_goingdown(
 	xfs_mount_t		*mp,
 	__uint32_t		inflags);
diff -Nru a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
--- a/fs/xfs/xfs_inode.c	2004-06-17 17:59:16 -07:00
+++ b/fs/xfs/xfs_inode.c	2004-08-18 21:05:36 -07:00
@@ -860,25 +860,28 @@
 	xfs_arch_t		arch)
 {
 	__uint16_t		di_flags;
-	uint			flags = 0;
+	uint			flags;
 
 	di_flags = INT_GET(dic->di_flags, arch);
-	if (di_flags & XFS_DIFLAG_REALTIME)
-		flags |= XFS_XFLAG_REALTIME;
-	if (di_flags & XFS_DIFLAG_PREALLOC)
-		flags |= XFS_XFLAG_PREALLOC;
-	if (di_flags & XFS_DIFLAG_IMMUTABLE)
-		flags |= XFS_XFLAG_IMMUTABLE;
-	if (di_flags & XFS_DIFLAG_APPEND)
-		flags |= XFS_XFLAG_APPEND;
-	if (di_flags & XFS_DIFLAG_SYNC)
-		flags |= XFS_XFLAG_SYNC;
-	if (di_flags & XFS_DIFLAG_NOATIME)
-		flags |= XFS_XFLAG_NOATIME;
-	if (di_flags & XFS_DIFLAG_NODUMP)
-		flags |= XFS_XFLAG_NODUMP;
-	if (XFS_CFORK_Q_ARCH(dic, arch))
-		flags |= XFS_XFLAG_HASATTR;
+	flags = XFS_CFORK_Q_ARCH(dic, arch) ? XFS_XFLAG_HASATTR : 0;
+	if (di_flags & XFS_DIFLAG_ANY) {
+		if (di_flags & XFS_DIFLAG_REALTIME)
+			flags |= XFS_XFLAG_REALTIME;
+		if (di_flags & XFS_DIFLAG_PREALLOC)
+			flags |= XFS_XFLAG_PREALLOC;
+		if (di_flags & XFS_DIFLAG_IMMUTABLE)
+			flags |= XFS_XFLAG_IMMUTABLE;
+		if (di_flags & XFS_DIFLAG_APPEND)
+			flags |= XFS_XFLAG_APPEND;
+		if (di_flags & XFS_DIFLAG_SYNC)
+			flags |= XFS_XFLAG_SYNC;
+		if (di_flags & XFS_DIFLAG_NOATIME)
+			flags |= XFS_XFLAG_NOATIME;
+		if (di_flags & XFS_DIFLAG_NODUMP)
+			flags |= XFS_XFLAG_NODUMP;
+		if (di_flags & XFS_DIFLAG_RTINHERIT)
+			flags |= XFS_XFLAG_RTINHERIT;
+	}
 	return flags;
 }
 
@@ -1236,8 +1239,15 @@
 		break;
 	case S_IFREG:
 	case S_IFDIR:
-		if (pip->i_d.di_flags &
-		    (XFS_DIFLAG_NOATIME|XFS_DIFLAG_NODUMP|XFS_DIFLAG_SYNC)) {
+		if (unlikely(pip->i_d.di_flags & XFS_DIFLAG_ANY)) {
+			if (pip->i_d.di_flags & XFS_DIFLAG_RTINHERIT) {
+				if ((mode & S_IFMT) == S_IFDIR) {
+					ip->i_d.di_flags |= XFS_DIFLAG_RTINHERIT;
+				} else {
+					ip->i_d.di_flags |= XFS_DIFLAG_REALTIME;
+					ip->i_iocore.io_flags |= XFS_IOCORE_RT;
+				}
+			}
 			if ((pip->i_d.di_flags & XFS_DIFLAG_NOATIME) &&
 			    xfs_inherit_noatime)
 				ip->i_d.di_flags |= XFS_DIFLAG_NOATIME;
@@ -1248,6 +1258,7 @@
 			    xfs_inherit_sync)
 				ip->i_d.di_flags |= XFS_DIFLAG_SYNC;
 		}
+		/* FALLTHROUGH */
 	case S_IFLNK:
 		ip->i_d.di_format = XFS_DINODE_FMT_EXTENTS;
 		ip->i_df.if_flags = XFS_IFEXTENTS;
@@ -3579,6 +3590,7 @@
 	return XFS_ERROR(EFSCORRUPTED);
 }
 
+
 /*
  * Flush all inactive inodes in mp.  Return true if no user references
  * were found, false otherwise.
@@ -3634,7 +3646,6 @@
 					continue;
 				}
 				if (!(flag & XFS_FLUSH_ALL)) {
-					ASSERT(0);
 					busy = 1;
 					done = 1;
 					break;
@@ -3851,6 +3862,6 @@
 		     (void *)ra,		/* caller of ilock */
 		     (void *)(unsigned long)current_cpu(),
 		     (void *)(unsigned long)current_pid(),
-		     0,0,0,0,0,0,0,0,0,0);
+		     NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
 }
 #endif
diff -Nru a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
--- a/fs/xfs/xfs_iomap.c	2004-05-20 21:16:53 -07:00
+++ b/fs/xfs/xfs_iomap.c	2004-08-18 19:08:12 -07:00
@@ -165,20 +165,24 @@
 		nisize = io->io_new_size;
 
 	for (pbm = 0; imaps && pbm < iomaps; imaps--, iomapp++, imap++, pbm++) {
-		iomapp->iomap_target = io->io_flags & XFS_IOCORE_RT ?
-			mp->m_rtdev_targp : mp->m_ddev_targp;
 		iomapp->iomap_offset = XFS_FSB_TO_B(mp, imap->br_startoff);
 		iomapp->iomap_delta = offset - iomapp->iomap_offset;
 		iomapp->iomap_bsize = XFS_FSB_TO_B(mp, imap->br_blockcount);
 		iomapp->iomap_flags = flags;
 
+		if (io->io_flags & XFS_IOCORE_RT) {
+			iomapp->iomap_flags |= IOMAP_REALTIME;
+			iomapp->iomap_target = mp->m_rtdev_targp;
+		} else {
+			iomapp->iomap_target = mp->m_ddev_targp;
+		}
 		start_block = imap->br_startblock;
 		if (start_block == HOLESTARTBLOCK) {
 			iomapp->iomap_bn = IOMAP_DADDR_NULL;
-			iomapp->iomap_flags = IOMAP_HOLE;
+			iomapp->iomap_flags |= IOMAP_HOLE;
 		} else if (start_block == DELAYSTARTBLOCK) {
 			iomapp->iomap_bn = IOMAP_DADDR_NULL;
-			iomapp->iomap_flags = IOMAP_DELAY;
+			iomapp->iomap_flags |= IOMAP_DELAY;
 		} else {
 			iomapp->iomap_bn = XFS_FSB_TO_DB_IO(io, start_block);
 			if (ISUNWRITTEN(imap))
@@ -512,6 +516,15 @@
 
 	*ret_imap = imap[0];
 	*nmaps = 1;
+	if ( !(io->io_flags & XFS_IOCORE_RT)  && !ret_imap->br_startblock) {
+                cmn_err(CE_PANIC,"Access to block zero:  fs <%s> inode: %lld "
+                        "start_block : %llx start_off : %llx blkcnt : %llx "
+                        "extent-state : %x \n",
+                        (ip->i_mount)->m_fsname,
+                        (long long)ip->i_ino,
+                        ret_imap->br_startblock, ret_imap->br_startoff,
+                        ret_imap->br_blockcount,ret_imap->br_state);
+        }
 	return 0;
 
  error0:	/* Cancel bmap, unlock inode, and cancel trans */
@@ -572,8 +585,10 @@
 	 * out to the file system's write iosize.  We clean up any extra
 	 * space left over when the file is closed in xfs_inactive().
 	 *
-	 * We don't bother with this for sync writes, because we need
-	 * to minimize the amount we write for good performance.
+	 * For sync writes, we are flushing delayed allocate space to
+	 * try to make additional space available for allocation near
+	 * the filesystem full boundary - preallocation hurts in that
+	 * situation, of course.
 	 */
 	if (!(ioflag & BMAPI_SYNC) && ((offset + count) > ip->i_d.di_size)) {
 		xfs_off_t	aligned_offset;
@@ -598,6 +613,20 @@
 				return error;
 			}
 			for (n = 0; n < nimaps; n++) {
+				if ( !(io->io_flags & XFS_IOCORE_RT)  && 
+					!imap[n].br_startblock) {
+					cmn_err(CE_PANIC,"Access to block "
+						"zero:  fs <%s> inode: %lld "
+						"start_block : %llx start_off "
+						": %llx blkcnt : %llx "
+						"extent-state : %x \n",
+						(ip->i_mount)->m_fsname,
+						(long long)ip->i_ino,
+						imap[n].br_startblock,
+						imap[n].br_startoff,
+						imap[n].br_blockcount,
+						imap[n].br_state);
+        			}
 				if ((imap[n].br_startblock != HOLESTARTBLOCK) &&
 				    (imap[n].br_startblock != DELAYSTARTBLOCK)) {
 					goto write_map;
@@ -621,7 +650,8 @@
 	 * request to a stripe width boundary if the file size is >=
 	 * stripe width and we are allocating past the allocation eof.
 	 */
-	if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC)
+	if (!(io->io_flags & XFS_IOCORE_RT) && mp->m_swidth 
+	    && (mp->m_flags & XFS_MOUNT_SWALLOC)
 	    && (isize >= XFS_FSB_TO_B(mp, mp->m_swidth)) && aeof) {
 		int eof;
 		xfs_fileoff_t new_last_fsb;
@@ -639,8 +669,8 @@
 	 * if the file size is >= stripe unit size, and we are allocating past
 	 * the allocation eof.
 	 */
-	} else if (mp->m_dalign && (isize >= XFS_FSB_TO_B(mp, mp->m_dalign))
-		   && aeof) {
+	} else if (!(io->io_flags & XFS_IOCORE_RT) && mp->m_dalign &&
+		   (isize >= XFS_FSB_TO_B(mp, mp->m_dalign)) && aeof) {
 		int eof;
 		xfs_fileoff_t new_last_fsb;
 		new_last_fsb = roundup_64(last_fsb, mp->m_dalign);
@@ -651,8 +681,22 @@
 		if (eof) {
 			last_fsb = new_last_fsb;
 		}
-	}
+	/*
+	 * Round up the allocation request to a real-time extent boundary
+	 * if the file is on the real-time subvolume.
+	 */
+	} else if (io->io_flags & XFS_IOCORE_RT && aeof) {
+		int eof;
+		xfs_fileoff_t new_last_fsb;
 
+		new_last_fsb = roundup_64(last_fsb, mp->m_sb.sb_rextsize);
+		error = XFS_BMAP_EOF(mp, io, new_last_fsb, XFS_DATA_FORK, &eof);
+		if (error) {
+			return error;
+		}
+		if (eof)
+			last_fsb = new_last_fsb;
+	}
 	error = xfs_bmapi(NULL, ip, offset_fsb,
 			  (xfs_filblks_t)(last_fsb - offset_fsb),
 			  XFS_BMAPI_DELAY | XFS_BMAPI_WRITE |
@@ -680,6 +724,15 @@
 
 	*ret_imap = imap[0];
 	*nmaps = 1;
+	if ( !(io->io_flags & XFS_IOCORE_RT)  && !ret_imap->br_startblock) {
+		cmn_err(CE_PANIC,"Access to block zero:  fs <%s> inode: %lld "
+                        "start_block : %llx start_off : %llx blkcnt : %llx "
+                        "extent-state : %x \n",
+                        (ip->i_mount)->m_fsname,
+                        (long long)ip->i_ino,
+                        ret_imap->br_startblock, ret_imap->br_startoff,
+                        ret_imap->br_blockcount,ret_imap->br_state);
+	}
 	return 0;
 }
 
@@ -697,6 +750,7 @@
 	int		*retmap)
 {
 	xfs_mount_t	*mp = ip->i_mount;
+	xfs_iocore_t    *io = &ip->i_iocore;
 	xfs_fileoff_t	offset_fsb, last_block;
 	xfs_fileoff_t	end_fsb, map_start_fsb;
 	xfs_fsblock_t	first_block;
@@ -802,6 +856,18 @@
 		 */
 
 		for (i = 0; i < nimaps; i++) {
+			if ( !(io->io_flags & XFS_IOCORE_RT)  && 
+				!imap[i].br_startblock) {
+				cmn_err(CE_PANIC,"Access to block zero:  "
+					"fs <%s> inode: %lld "
+					"start_block : %llx start_off : %llx " 
+					"blkcnt : %llx extent-state : %x \n",
+					(ip->i_mount)->m_fsname,
+					(long long)ip->i_ino,
+					imap[i].br_startblock,
+					imap[i].br_startoff,
+				        imap[i].br_blockcount,imap[i].br_state);
+                        }
 			if ((map->br_startoff >= imap[i].br_startoff) &&
 			    (map->br_startoff < (imap[i].br_startoff +
 						 imap[i].br_blockcount))) {
@@ -837,6 +903,7 @@
 	size_t		count)
 {
 	xfs_mount_t	*mp = ip->i_mount;
+	xfs_iocore_t    *io = &ip->i_iocore;
 	xfs_trans_t	*tp;
 	xfs_fileoff_t	offset_fsb;
 	xfs_filblks_t	count_fsb;
@@ -853,7 +920,8 @@
 				&ip->i_iocore, offset, count);
 
 	offset_fsb = XFS_B_TO_FSBT(mp, offset);
-	count_fsb = XFS_B_TO_FSB(mp, count);
+	count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
+	count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
 
 	do {
 		nres = XFS_DIOSTRAT_SPACE_RES(mp, 0);
@@ -898,6 +966,16 @@
 		xfs_iunlock(ip, XFS_ILOCK_EXCL);
 		if (error)
 			goto error0;
+		
+		if ( !(io->io_flags & XFS_IOCORE_RT)  && !imap.br_startblock) {
+			cmn_err(CE_PANIC,"Access to block zero:  fs <%s> "
+				"inode: %lld start_block : %llx start_off : "
+				"%llx blkcnt : %llx extent-state : %x \n",
+				(ip->i_mount)->m_fsname,
+				(long long)ip->i_ino,
+				imap.br_startblock,imap.br_startoff,
+				imap.br_blockcount,imap.br_state);
+        	}
 
 		if ((numblks_fsb = imap.br_blockcount) == 0) {
 			/*
diff -Nru a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h
--- a/fs/xfs/xfs_iomap.h	2004-01-18 16:23:13 -08:00
+++ b/fs/xfs/xfs_iomap.h	2004-08-18 19:08:12 -07:00
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 2003,2004 Silicon Graphics, Inc.  All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -42,6 +42,7 @@
 	IOMAP_EOF =		0x01,	/* mapping contains EOF   */
 	IOMAP_HOLE =		0x02,	/* mapping covers a hole  */
 	IOMAP_DELAY =		0x04,	/* mapping covers delalloc region  */
+	IOMAP_REALTIME =	0x10,	/* mapping on the realtime device  */
 	IOMAP_UNWRITTEN =	0x20,	/* mapping covers allocated */
 					/* but uninitialized file data  */
 	IOMAP_NEW =		0x40	/* just allocate */
@@ -57,7 +58,7 @@
 	BMAPI_IGNSTATE = (1 << 4),	/* ignore unwritten state on read */
 	BMAPI_DIRECT = (1 << 5),		/* direct instead of buffered write */
 	BMAPI_MMAP = (1 << 6),		/* allocate for mmap write */
-	BMAPI_SYNC = (1 << 7),		/* sync write */
+	BMAPI_SYNC = (1 << 7),		/* sync write to flush delalloc space */
 	BMAPI_TRYLOCK = (1 << 8),	/* non-blocking request */
 	BMAPI_DEVICE = (1 << 9),	/* we only want to know the device */
 } bmapi_flags_t;
diff -Nru a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c
--- a/fs/xfs/xfs_itable.c	2004-05-20 20:43:07 -07:00
+++ b/fs/xfs/xfs_itable.c	2004-08-18 20:37:21 -07:00
@@ -68,7 +68,7 @@
 xfs_bulkstat_one(
 	xfs_mount_t	*mp,		/* mount point for filesystem */
 	xfs_ino_t	ino,		/* inode number to get data for */
-	void		*buffer,	/* buffer to place output in */
+	void		__user *buffer,	/* buffer to place output in */
 	int		ubsize,		/* size of buffer */
 	void		*private_data,	/* my private data */
 	xfs_daddr_t	bno,		/* starting bno of inode cluster */
@@ -231,7 +231,7 @@
 	bulkstat_one_pf		formatter, /* func that'd fill a single buf */
 	void			*private_data,/* private data for formatter */
 	size_t			statstruct_size, /* sizeof struct filling */
-	xfs_caddr_t		ubuffer, /* buffer with inode stats */
+	char			__user *ubuffer, /* buffer with inode stats */
 	int			flags,	/* defined in xfs_itable.h */
 	int			*done)	/* 1 if there're more stats to get */
 {
@@ -265,7 +265,7 @@
 	int			tmp;	/* result value from btree calls */
 	int			ubcount; /* size of user's buffer */
 	int			ubleft;	/* bytes left in user's buffer */
-	xfs_caddr_t		ubufp;	/* current pointer into user's buffer */
+	char			__user *ubufp;	/* pointer into user's buffer */
 	int			ubelem;	/* spaces used in user's buffer */
 	int			ubused;	/* bytes used by formatter */
 	xfs_buf_t		*bp;	/* ptr to on-disk inode cluster buf */
@@ -633,7 +633,7 @@
 xfs_bulkstat_single(
 	xfs_mount_t		*mp,	/* mount point for filesystem */
 	xfs_ino_t		*lastinop, /* inode to return */
-	xfs_caddr_t		buffer,	/* buffer with inode stats */
+	char			__user *buffer, /* buffer with inode stats */
 	int			*done)	/* 1 if there're more stats to get */
 {
 	int			count;	/* count value for bulkstat call */
@@ -682,7 +682,7 @@
 	xfs_mount_t	*mp,		/* mount point for filesystem */
 	xfs_ino_t	*lastino,	/* last inode returned */
 	int		*count,		/* size of buffer/count returned */
-	xfs_caddr_t	ubuffer)	/* buffer with inode descriptions */
+	xfs_inogrp_t	__user *ubuffer)/* buffer with inode descriptions */
 {
 	xfs_buf_t	*agbp;
 	xfs_agino_t	agino;
@@ -766,7 +766,7 @@
 				error = XFS_ERROR(EFAULT);
 				break;
 			}
-			ubuffer += bufidx * sizeof(*buffer);
+			ubuffer += bufidx;
 			*count += bufidx;
 			bufidx = 0;
 		}
diff -Nru a/fs/xfs/xfs_itable.h b/fs/xfs/xfs_itable.h
--- a/fs/xfs/xfs_itable.h	2004-05-20 19:59:18 -07:00
+++ b/fs/xfs/xfs_itable.h	2004-08-18 20:37:21 -07:00
@@ -40,7 +40,7 @@
  */
 typedef int (*bulkstat_one_pf)(struct xfs_mount	*mp,
 			       xfs_ino_t	ino,
-			       void		*buffer,
+			       void		__user *buffer,
 			       int		ubsize,
 			       void		*private_data,
 			       xfs_daddr_t	bno,
@@ -73,7 +73,7 @@
 	bulkstat_one_pf formatter,	/* func that'd fill a single buf */
 	void		*private_data,	/* private data for formatter */
 	size_t		statstruct_size,/* sizeof struct that we're filling */
-	xfs_caddr_t	ubuffer,	/* buffer with inode stats */
+	char		__user *ubuffer,/* buffer with inode stats */
 	int		flags,		/* flag to control access method */
 	int		*done);		/* 1 if there're more stats to get */
 
@@ -81,14 +81,14 @@
 xfs_bulkstat_single(
 	xfs_mount_t		*mp,
 	xfs_ino_t		*lastinop,
-	xfs_caddr_t		buffer,
+	char			__user *buffer,
 	int			*done);
 
 int
 xfs_bulkstat_one(
 	xfs_mount_t		*mp,
 	xfs_ino_t		ino,
-	void			*buffer,
+	void			__user *buffer,
 	int			ubsize,
 	void			*private_data,
 	xfs_daddr_t		bno,
@@ -101,6 +101,6 @@
 	xfs_mount_t		*mp,	/* mount point for filesystem */
 	xfs_ino_t		*last,	/* last inode returned */
 	int			*count,	/* size of buffer/count returned */
-	xfs_caddr_t		buffer);/* buffer with inode descriptions */
+	xfs_inogrp_t		__user *buffer);/* buffer with inode info */
 
 #endif	/* __XFS_ITABLE_H__ */
diff -Nru a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
--- a/fs/xfs/xfs_log.c	2004-05-20 23:48:10 -07:00
+++ b/fs/xfs/xfs_log.c	2004-08-18 20:26:29 -07:00
@@ -122,7 +122,6 @@
 /* local debug functions */
 #if defined(DEBUG) && !defined(XLOG_NOLOG)
 STATIC void	xlog_verify_dest_ptr(xlog_t *log, __psint_t ptr);
-STATIC void	xlog_verify_disk_cycle_no(xlog_t *log, xlog_in_core_t *iclog);
 STATIC void	xlog_verify_grant_head(xlog_t *log, int equals);
 STATIC void	xlog_verify_iclog(xlog_t *log, xlog_in_core_t *iclog,
 				  int count, boolean_t syncing);
@@ -130,7 +129,6 @@
 				     xfs_lsn_t tail_lsn);
 #else
 #define xlog_verify_dest_ptr(a,b)
-#define xlog_verify_disk_cycle_no(a,b)
 #define xlog_verify_grant_head(a,b)
 #define xlog_verify_iclog(a,b,c,d)
 #define xlog_verify_tail_lsn(a,b,c)
@@ -356,13 +354,13 @@
 	if (!xlog_debug && xlog_target == log->l_targ)
 		return 0;
 #endif
-	cb->cb_next = 0;
+	cb->cb_next = NULL;
 	spl = LOG_LOCK(log);
 	abortflg = (iclog->ic_state & XLOG_STATE_IOERROR);
 	if (!abortflg) {
 		ASSERT_ALWAYS((iclog->ic_state == XLOG_STATE_ACTIVE) ||
 			      (iclog->ic_state == XLOG_STATE_WANT_SYNC));
-		cb->cb_next = 0;
+		cb->cb_next = NULL;
 		*(iclog->ic_callback_tail) = cb;
 		iclog->ic_callback_tail = &(cb->cb_next);
 	}
@@ -564,7 +562,7 @@
 	xlog_in_core_t	 *first_iclog;
 #endif
 	xfs_log_iovec_t  reg[1];
-	xfs_log_ticket_t tic = 0;
+	xfs_log_ticket_t tic = NULL;
 	xfs_lsn_t	 lsn;
 	int		 error;
 	SPLDECL(s);
@@ -813,7 +811,7 @@
 	xlog_t		*log = mp->m_log;
 	vfs_t		*vfsp = XFS_MTOVFS(mp);
 
-	if (mp->m_frozen || XFS_FORCED_SHUTDOWN(mp) ||
+	if (fs_frozen(vfsp) || XFS_FORCED_SHUTDOWN(mp) ||
 	    (vfsp->vfs_flag & VFS_RDONLY))
 		return 0;
 
@@ -1277,7 +1275,7 @@
 	int		error;
 	xfs_log_iovec_t	reg[1];
 
-	reg[0].i_addr = 0;
+	reg[0].i_addr = NULL;
 	reg[0].i_len = 0;
 
 	ASSERT_ALWAYS(iclog);
@@ -1857,7 +1855,7 @@
 		if (iclog->ic_state == XLOG_STATE_DIRTY) {
 			iclog->ic_state	= XLOG_STATE_ACTIVE;
 			iclog->ic_offset       = 0;
-			iclog->ic_callback	= 0;   /* don't need to free */
+			iclog->ic_callback	= NULL;   /* don't need to free */
 			/*
 			 * If the number of ops in this iclog indicate it just
 			 * contains the dummy transaction, we can
@@ -2080,7 +2078,7 @@
 
 			while (cb != 0) {
 				iclog->ic_callback_tail = &(iclog->ic_callback);
-				iclog->ic_callback = 0;
+				iclog->ic_callback = NULL;
 				LOG_UNLOCK(log, s);
 
 				/* perform callbacks in the order given */
@@ -3098,7 +3096,7 @@
 		log->l_ticket_cnt++;
 		log->l_ticket_tcnt++;
 	}
-	t_list->t_next = 0;
+	t_list->t_next = NULL;
 	log->l_tail = t_list;
 	LOG_UNLOCK(log, s);
 }	/* xlog_state_ticket_alloc */
@@ -3126,7 +3124,7 @@
 	/* no need to clear fields */
 #else
 	/* When we debug, it is easier if tickets are cycled */
-	ticket->t_next     = 0;
+	ticket->t_next     = NULL;
 	if (log->l_tail != 0) {
 		log->l_tail->t_next = ticket;
 	} else {
@@ -3241,33 +3239,6 @@
 	if (! good_ptr)
 		xlog_panic("xlog_verify_dest_ptr: invalid ptr");
 }	/* xlog_verify_dest_ptr */
-
-
-#ifdef DEBUG
-/* check split LR write */
-STATIC void
-xlog_verify_disk_cycle_no(xlog_t	 *log,
-			  xlog_in_core_t *iclog)
-{
-    xfs_buf_t	*bp;
-    uint	cycle_no;
-    xfs_caddr_t ptr;
-    xfs_daddr_t	i;
-
-    if (BLOCK_LSN(iclog->ic_header.h_lsn, ARCH_CONVERT) < 10) {
-	cycle_no = CYCLE_LSN(iclog->ic_header.h_lsn, ARCH_CONVERT);
-	bp = xlog_get_bp(log, 1);
-	ASSERT(bp);
-	for (i = 0; i < BLOCK_LSN(iclog->ic_header.h_lsn, ARCH_CONVERT); i++) {
-	    xlog_bread(log, i, 1, bp);
-	    ptr = xlog_align(log, i, 1, bp);
-	    if (GET_CYCLE(ptr, ARCH_CONVERT) != cycle_no)
-		xlog_warn("XFS: xlog_verify_disk_cycle_no: bad cycle no");
-	}
-	xlog_put_bp(bp);
-    }
-}	/* xlog_verify_disk_cycle_no */
-#endif
 
 STATIC void
 xlog_verify_grant_head(xlog_t *log, int equals)
diff -Nru a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
--- a/fs/xfs/xfs_log_recover.c	2004-05-20 23:23:09 -07:00
+++ b/fs/xfs/xfs_log_recover.c	2004-08-26 23:21:36 -07:00
@@ -2047,12 +2047,11 @@
 		errs++;
 	}
 
-	if (! errs) {
+	if (! errs && !INT_ISZERO(ddq->d_id, ARCH_CONVERT)) {
 		if (INT_GET(ddq->d_blk_softlimit, ARCH_CONVERT) &&
 		    INT_GET(ddq->d_bcount, ARCH_CONVERT) >=
 				INT_GET(ddq->d_blk_softlimit, ARCH_CONVERT)) {
-			if (INT_ISZERO(ddq->d_btimer, ARCH_CONVERT) &&
-			    !INT_ISZERO(ddq->d_id, ARCH_CONVERT)) {
+			if (INT_ISZERO(ddq->d_btimer, ARCH_CONVERT)) {
 				if (flags & XFS_QMOPT_DOWARN)
 					cmn_err(CE_ALERT,
 					"%s : Dquot ID 0x%x (0x%p) "
@@ -2065,8 +2064,7 @@
 		if (INT_GET(ddq->d_ino_softlimit, ARCH_CONVERT) &&
 		    INT_GET(ddq->d_icount, ARCH_CONVERT) >=
 				INT_GET(ddq->d_ino_softlimit, ARCH_CONVERT)) {
-			if (INT_ISZERO(ddq->d_itimer, ARCH_CONVERT) &&
-			    !INT_ISZERO(ddq->d_id, ARCH_CONVERT)) {
+			if (INT_ISZERO(ddq->d_itimer, ARCH_CONVERT)) {
 				if (flags & XFS_QMOPT_DOWARN)
 					cmn_err(CE_ALERT,
 					"%s : Dquot ID 0x%x (0x%p) "
@@ -2076,6 +2074,19 @@
 				errs++;
 			}
 		}
+		if (INT_GET(ddq->d_rtb_softlimit, ARCH_CONVERT) &&
+		    INT_GET(ddq->d_rtbcount, ARCH_CONVERT) >=
+				INT_GET(ddq->d_rtb_softlimit, ARCH_CONVERT)) {
+			if (INT_ISZERO(ddq->d_rtbtimer, ARCH_CONVERT)) {
+				if (flags & XFS_QMOPT_DOWARN)
+					cmn_err(CE_ALERT,
+					"%s : Dquot ID 0x%x (0x%p) "
+					"RTBLK TIMER NOT STARTED",
+					str, (int)
+					INT_GET(ddq->d_id, ARCH_CONVERT), ddq);
+				errs++;
+			}
+		}
 	}
 
 	if (!errs || !(flags & XFS_QMOPT_DQREPAIR))
@@ -2319,7 +2330,7 @@
 		 * invalidate the buffer when we write it out below.
 		 */
 		imap.im_blkno = 0;
-		xfs_imap(log->l_mp, 0, ino, &imap, 0);
+		xfs_imap(log->l_mp, NULL, ino, &imap, 0);
 	}
 
 	/*
diff -Nru a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
--- a/fs/xfs/xfs_mount.c	2004-05-20 21:16:53 -07:00
+++ b/fs/xfs/xfs_mount.c	2004-08-26 23:27:59 -07:00
@@ -140,9 +140,6 @@
 	 */
 	xfs_trans_ail_init(mp);
 
-	/* Init freeze sync structures */
-	spinlock_init(&mp->m_freeze_lock, "xfs_freeze");
-	init_sv(&mp->m_wait_unfreeze, SV_DEFAULT, "xfs_freeze", 0);
 	atomic_set(&mp->m_active_trans, 0);
 
 	return mp;
@@ -192,8 +189,6 @@
 		VFS_REMOVEBHV(vfsp, &mp->m_bhv);
 	}
 
-	spinlock_destroy(&mp->m_freeze_lock);
-	sv_destroy(&mp->m_wait_unfreeze);
 	kmem_free(mp, sizeof(xfs_mount_t));
 }
 
@@ -283,17 +278,22 @@
 		return XFS_ERROR(EFSCORRUPTED);
 	}
 
-#if !XFS_BIG_BLKNOS
+	ASSERT(PAGE_SHIFT >= sbp->sb_blocklog);
+	ASSERT(sbp->sb_blocklog >= BBSHIFT);
+
+#if XFS_BIG_BLKNOS     /* Limited by ULONG_MAX of page cache index */
+	if (unlikely(
+	    (sbp->sb_dblocks >> (PAGE_SHIFT - sbp->sb_blocklog)) > ULONG_MAX ||
+	    (sbp->sb_rblocks >> (PAGE_SHIFT - sbp->sb_blocklog)) > ULONG_MAX)) {
+#else                  /* Limited by UINT_MAX of sectors */
 	if (unlikely(
-	    (sbp->sb_dblocks << (__uint64_t)(sbp->sb_blocklog - BBSHIFT))
-		> UINT_MAX ||
-	    (sbp->sb_rblocks << (__uint64_t)(sbp->sb_blocklog - BBSHIFT))
-		> UINT_MAX)) {
+	    (sbp->sb_dblocks << (sbp->sb_blocklog - BBSHIFT)) > UINT_MAX ||
+	    (sbp->sb_rblocks << (sbp->sb_blocklog - BBSHIFT)) > UINT_MAX)) {
+#endif
 		cmn_err(CE_WARN,
 	"XFS: File system is too large to be mounted on this system.");
 		return XFS_ERROR(E2BIG);
 	}
-#endif
 
 	if (unlikely(sbp->sb_inprogress)) {
 		cmn_err(CE_WARN, "XFS: file system busy");
@@ -318,10 +318,10 @@
 	return 0;
 }
 
-void
-xfs_initialize_perag(xfs_mount_t *mp, int agcount)
+xfs_agnumber_t
+xfs_initialize_perag(xfs_mount_t *mp, xfs_agnumber_t agcount)
 {
-	int		index, max_metadata;
+	xfs_agnumber_t	index, max_metadata;
 	xfs_perag_t	*pag;
 	xfs_agino_t	agino;
 	xfs_ino_t	ino;
@@ -377,7 +377,7 @@
 			pag->pagi_inodeok = 1;
 		}
 	}
-	mp->m_maxagi = index;
+	return index;
 }
 
 /*
@@ -638,9 +638,8 @@
 	xfs_buf_t	*bp;
 	xfs_sb_t	*sbp = &(mp->m_sb);
 	xfs_inode_t	*rip;
-	vnode_t		*rvp = 0;
+	vnode_t		*rvp = NULL;
 	int		readio_log, writeio_log;
-	vmap_t		vmap;
 	xfs_daddr_t	d;
 	__uint64_t	ret64;
 	__int64_t	update_flags;
@@ -952,7 +951,7 @@
 	mp->m_perag =
 		kmem_zalloc(sbp->sb_agcount * sizeof(xfs_perag_t), KM_SLEEP);
 
-	xfs_initialize_perag(mp, sbp->sb_agcount);
+	mp->m_maxagi = xfs_initialize_perag(mp, sbp->sb_agcount);
 
 	/*
 	 * log's mount-time initialization. Perform 1st part recovery if needed
@@ -984,7 +983,6 @@
 
 	ASSERT(rip != NULL);
 	rvp = XFS_ITOV(rip);
-	VMAP(rvp, vmap);
 
 	if (unlikely((rip->i_d.di_mode & S_IFMT) != S_IFDIR)) {
 		cmn_err(CE_WARN, "XFS: corrupted root inode");
@@ -1048,7 +1046,6 @@
 	 * Free up the root inode.
 	 */
 	VN_RELE(rvp);
-	vn_purge(rvp, &vmap);
  error3:
 	xfs_log_unmount_dealloc(mp);
  error2:
@@ -1581,60 +1578,4 @@
 	}
 	xfs_mod_sb(tp, fields);
 	xfs_trans_commit(tp, 0, NULL);
-}
-
-/* Functions to lock access out of the filesystem for forced
- * shutdown or snapshot.
- */
-
-void
-xfs_start_freeze(
-	xfs_mount_t	*mp,
-	int		level)
-{
-	unsigned long	s = mutex_spinlock(&mp->m_freeze_lock);
-
-	mp->m_frozen = level;
-	mutex_spinunlock(&mp->m_freeze_lock, s);
-
-	if (level == XFS_FREEZE_TRANS) {
-		while (atomic_read(&mp->m_active_trans) > 0)
-			delay(100);
-	}
-}
-
-void
-xfs_finish_freeze(
-	xfs_mount_t	*mp)
-{
-	unsigned long	s = mutex_spinlock(&mp->m_freeze_lock);
-
-	if (mp->m_frozen) {
-		mp->m_frozen = 0;
-		sv_broadcast(&mp->m_wait_unfreeze);
-	}
-
-	mutex_spinunlock(&mp->m_freeze_lock, s);
-}
-
-void
-xfs_check_frozen(
-	xfs_mount_t	*mp,
-	bhv_desc_t	*bdp,
-	int		level)
-{
-	unsigned long	s;
-
-	if (mp->m_frozen) {
-		s = mutex_spinlock(&mp->m_freeze_lock);
-
-		if (mp->m_frozen < level) {
-			mutex_spinunlock(&mp->m_freeze_lock, s);
-		} else {
-			sv_wait(&mp->m_wait_unfreeze, 0, &mp->m_freeze_lock, s);
-		}
-	}
-
-	if (level == XFS_FREEZE_TRANS)
-		atomic_inc(&mp->m_active_trans);
 }
diff -Nru a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
--- a/fs/xfs/xfs_mount.h	2004-06-17 18:43:42 -07:00
+++ b/fs/xfs/xfs_mount.h	2004-08-26 23:27:59 -07:00
@@ -382,10 +382,6 @@
 	struct xfs_dmops	m_dm_ops;	/* vector of DMI ops */
 	struct xfs_qmops	m_qm_ops;	/* vector of XQM ops */
 	struct xfs_ioops	m_io_ops;	/* vector of I/O ops */
-	lock_t			m_freeze_lock;	/* Lock for m_frozen */
-	uint			m_frozen;	/* FS frozen for shutdown or
-						 * snapshot */
-	sv_t			m_wait_unfreeze;/* waiting to unfreeze */
 	atomic_t		m_active_trans;	/* number trans frozen */
 } xfs_mount_t;
 
@@ -555,19 +551,9 @@
 extern void	xfs_freesb(xfs_mount_t *);
 extern void	xfs_do_force_shutdown(bhv_desc_t *, int, char *, int);
 extern int	xfs_syncsub(xfs_mount_t *, int, int, int *);
-extern void	xfs_initialize_perag(xfs_mount_t *, int);
+extern xfs_agnumber_t	xfs_initialize_perag(xfs_mount_t *, xfs_agnumber_t);
 extern void	xfs_xlatesb(void *, struct xfs_sb *, int, xfs_arch_t,
 			__int64_t);
-
-/*
- * Flags for freeze operations.
- */
-#define XFS_FREEZE_WRITE	1
-#define XFS_FREEZE_TRANS	2
-
-extern void	xfs_start_freeze(xfs_mount_t *, int);
-extern void	xfs_finish_freeze(xfs_mount_t *);
-extern void	xfs_check_frozen(xfs_mount_t *, bhv_desc_t *, int);
 
 extern struct vfsops xfs_vfsops;
 extern struct vnodeops xfs_vnodeops;
diff -Nru a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h
--- a/fs/xfs/xfs_quota.h	2004-03-02 20:53:02 -08:00
+++ b/fs/xfs/xfs_quota.h	2004-08-18 20:37:21 -07:00
@@ -309,23 +309,27 @@
 } xfs_dqtrxops_t;
 
 #define XFS_DQTRXOP(mp, tp, op, args...) \
-			((mp)->m_qm_ops.xfs_dqtrxops ? \
-			((mp)->m_qm_ops.xfs_dqtrxops->op)(tp, ## args) : 0)
+		((mp)->m_qm_ops.xfs_dqtrxops ? \
+		((mp)->m_qm_ops.xfs_dqtrxops->op)(tp, ## args) : 0)
+
+#define XFS_DQTRXOP_VOID(mp, tp, op, args...) \
+		((mp)->m_qm_ops.xfs_dqtrxops ? \
+		((mp)->m_qm_ops.xfs_dqtrxops->op)(tp, ## args) : (void)0)
 
 #define XFS_TRANS_DUP_DQINFO(mp, otp, ntp) \
-	XFS_DQTRXOP(mp, otp, qo_dup_dqinfo, ntp)
+	XFS_DQTRXOP_VOID(mp, otp, qo_dup_dqinfo, ntp)
 #define XFS_TRANS_FREE_DQINFO(mp, tp) \
-	XFS_DQTRXOP(mp, tp, qo_free_dqinfo)
+	XFS_DQTRXOP_VOID(mp, tp, qo_free_dqinfo)
 #define XFS_TRANS_MOD_DQUOT_BYINO(mp, tp, ip, field, delta) \
-	XFS_DQTRXOP(mp, tp, qo_mod_dquot_byino, ip, field, delta)
+	XFS_DQTRXOP_VOID(mp, tp, qo_mod_dquot_byino, ip, field, delta)
 #define XFS_TRANS_APPLY_DQUOT_DELTAS(mp, tp) \
-	XFS_DQTRXOP(mp, tp, qo_apply_dquot_deltas)
+	XFS_DQTRXOP_VOID(mp, tp, qo_apply_dquot_deltas)
 #define XFS_TRANS_RESERVE_QUOTA_NBLKS(mp, tp, ip, nblks, ninos, fl) \
 	XFS_DQTRXOP(mp, tp, qo_reserve_quota_nblks, mp, ip, nblks, ninos, fl)
 #define XFS_TRANS_RESERVE_QUOTA_BYDQUOTS(mp, tp, ud, gd, nb, ni, fl) \
 	XFS_DQTRXOP(mp, tp, qo_reserve_quota_bydquots, mp, ud, gd, nb, ni, fl)
 #define XFS_TRANS_UNRESERVE_AND_MOD_DQUOTS(mp, tp) \
-	XFS_DQTRXOP(mp, tp, qo_unreserve_and_mod_dquots)
+	XFS_DQTRXOP_VOID(mp, tp, qo_unreserve_and_mod_dquots)
 
 #define XFS_TRANS_RESERVE_BLKQUOTA(mp, tp, ip, nblks) \
 	XFS_TRANS_RESERVE_QUOTA_NBLKS(mp, tp, ip, nblks, 0, \
diff -Nru a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c
--- a/fs/xfs/xfs_rtalloc.c	2004-06-17 17:09:15 -07:00
+++ b/fs/xfs/xfs_rtalloc.c	2004-08-18 17:19:16 -07:00
@@ -2353,13 +2353,7 @@
 	ASSERT(sbp->sb_rsumino != NULLFSINO);
 	error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, &mp->m_rsumip, 0);
 	if (error) {
-		vnode_t		*rbmvp;		/* vnode for bitmap file */
-		vmap_t		vmap;		/* vmap to delete vnode */
-
-		rbmvp = XFS_ITOV(mp->m_rbmip);
-		VMAP(rbmvp, vmap);
-		VN_RELE(rbmvp);
-		vn_purge(rbmvp, &vmap);
+		VN_RELE(XFS_ITOV(mp->m_rbmip));
 		return error;
 	}
 	ASSERT(mp->m_rsumip != NULL);
diff -Nru a/fs/xfs/xfs_sb.h b/fs/xfs/xfs_sb.h
--- a/fs/xfs/xfs_sb.h	2004-04-21 19:29:53 -07:00
+++ b/fs/xfs/xfs_sb.h	2004-08-18 23:02:22 -07:00
@@ -250,9 +250,9 @@
 	((((sbp)->sb_versionnum >= XFS_SB_VERSION_1) && \
 	  ((sbp)->sb_versionnum <= XFS_SB_VERSION_3)) || \
 	   ((XFS_SB_VERSION_NUM(sbp) == XFS_SB_VERSION_4) && \
-	    !(((sbp)->sb_versionnum & ~XFS_SB_VERSION_OKREALBITS) && \
-	      ((sbp)->sb_versionnum & XFS_SB_VERSION_MOREBITSBIT) && \
-	      ((sbp)->sb_features2 & ~XFS_SB_VERSION2_OKREALBITS))
+	    !(((sbp)->sb_versionnum & ~XFS_SB_VERSION_OKREALBITS) || \
+	      (((sbp)->sb_versionnum & XFS_SB_VERSION_MOREBITSBIT) && \
+	       ((sbp)->sb_features2 & ~XFS_SB_VERSION2_OKREALBITS)))
 
 #ifdef __KERNEL__
 #define	XFS_SB_GOOD_VERSION(sbp)	\
diff -Nru a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c
--- a/fs/xfs/xfs_trans.c	2004-05-21 00:08:21 -07:00
+++ b/fs/xfs/xfs_trans.c	2004-08-18 16:16:24 -07:00
@@ -131,7 +131,9 @@
 	xfs_mount_t	*mp,
 	uint		type)
 {
-	xfs_check_frozen(mp, NULL, XFS_FREEZE_TRANS);
+	fs_check_frozen(XFS_MTOVFS(mp), SB_FREEZE_TRANS);
+	atomic_inc(&mp->m_active_trans);
+
 	return (_xfs_trans_alloc(mp, type));
 
 }
@@ -488,6 +490,9 @@
 
 	if (tp->t_frextents_delta != 0) {
 		INT_MOD(sbp->sb_frextents, ARCH_CONVERT, tp->t_frextents_delta);
+	}
+	if (tp->t_res_frextents_delta != 0) {
+		INT_MOD(sbp->sb_frextents, ARCH_CONVERT, tp->t_res_frextents_delta);
 	}
 	if (tp->t_dblocks_delta != 0) {
 		INT_MOD(sbp->sb_dblocks, ARCH_CONVERT, tp->t_dblocks_delta);
diff -Nru a/fs/xfs/xfs_trans_item.c b/fs/xfs/xfs_trans_item.c
--- a/fs/xfs/xfs_trans_item.c	2004-05-21 00:08:21 -07:00
+++ b/fs/xfs/xfs_trans_item.c	2004-08-18 18:13:42 -07:00
@@ -290,7 +290,7 @@
 	}
 	ASSERT(0);
 	/* NOTREACHED */
-	return 0; /* keep gcc quite */
+	return NULL; /* keep gcc quite */
 }
 
 /*
diff -Nru a/fs/xfs/xfs_types.h b/fs/xfs/xfs_types.h
--- a/fs/xfs/xfs_types.h	2003-12-16 20:58:26 -08:00
+++ b/fs/xfs/xfs_types.h	2004-08-18 17:51:09 -07:00
@@ -58,7 +58,7 @@
 typedef __int64_t		prid_t;		/* project ID */
 typedef __uint32_t		inst_t;		/* an instruction */
 
-typedef __u64			xfs_off_t;
+typedef __s64			xfs_off_t;	/* <file offset> type */
 typedef __u64			xfs_ino_t;	/* <inode> type */
 typedef __s64			xfs_daddr_t;	/* <disk address> type */
 typedef char *			xfs_caddr_t;	/* <core address> type */
diff -Nru a/fs/xfs/xfs_vfsops.c b/fs/xfs/xfs_vfsops.c
--- a/fs/xfs/xfs_vfsops.c	2004-06-17 17:52:33 -07:00
+++ b/fs/xfs/xfs_vfsops.c	2004-08-18 16:24:20 -07:00
@@ -431,6 +431,16 @@
 	logdev = rtdev = NULL;
 
 	/*
+	 * Setup xfs_mount function vectors from available behaviors
+	 */
+	p = vfs_bhv_lookup(vfsp, VFS_POSITION_DM);
+	mp->m_dm_ops = p ? *(xfs_dmops_t *) vfs_bhv_custom(p) : xfs_dmcore_stub;
+	p = vfs_bhv_lookup(vfsp, VFS_POSITION_QM);
+	mp->m_qm_ops = p ? *(xfs_qmops_t *) vfs_bhv_custom(p) : xfs_qmcore_stub;
+	p = vfs_bhv_lookup(vfsp, VFS_POSITION_IO);
+	mp->m_io_ops = p ? *(xfs_ioops_t *) vfs_bhv_custom(p) : xfs_iocore_xfs;
+
+	/*
 	 * Open real time and log devices - order is important.
 	 */
 	if (args->logname[0]) {
@@ -455,68 +465,73 @@
 	}
 
 	/*
-	 * Setup xfs_mount function vectors from available behaviors
-	 */
-	p = vfs_bhv_lookup(vfsp, VFS_POSITION_DM);
-	mp->m_dm_ops = p ? *(xfs_dmops_t *) vfs_bhv_custom(p) : xfs_dmcore_stub;
-	p = vfs_bhv_lookup(vfsp, VFS_POSITION_QM);
-	mp->m_qm_ops = p ? *(xfs_qmops_t *) vfs_bhv_custom(p) : xfs_qmcore_stub;
-	p = vfs_bhv_lookup(vfsp, VFS_POSITION_IO);
-	mp->m_io_ops = p ? *(xfs_ioops_t *) vfs_bhv_custom(p) : xfs_iocore_xfs;
-
-	/*
 	 * Setup xfs_mount buffer target pointers
 	 */
+	error = ENOMEM;
 	mp->m_ddev_targp = xfs_alloc_buftarg(ddev);
-	if (rtdev)
+	if (!mp->m_ddev_targp) {
+		xfs_blkdev_put(logdev);
+		xfs_blkdev_put(rtdev);
+		return error;
+	}
+	if (rtdev) {
 		mp->m_rtdev_targp = xfs_alloc_buftarg(rtdev);
+		if (!mp->m_rtdev_targp)
+			goto error0;
+	}
 	mp->m_logdev_targp = (logdev && logdev != ddev) ?
 				xfs_alloc_buftarg(logdev) : mp->m_ddev_targp;
+	if (!mp->m_logdev_targp)
+		goto error0;
 
 	/*
 	 * Setup flags based on mount(2) options and then the superblock
 	 */
 	error = xfs_start_flags(vfsp, args, mp);
 	if (error)
-		goto error;
+		goto error1;
 	error = xfs_readsb(mp);
 	if (error)
-		goto error;
+		goto error1;
 	error = xfs_finish_flags(vfsp, args, mp);
-	if (error) {
-		xfs_freesb(mp);
-		goto error;
-	}
+	if (error)
+		goto error2;
 
 	/*
 	 * Setup xfs_mount buffer target pointers based on superblock
 	 */
-	xfs_setsize_buftarg(mp->m_ddev_targp, mp->m_sb.sb_blocksize,
-			    mp->m_sb.sb_sectsize);
-	if (logdev && logdev != ddev) {
+	error = xfs_setsize_buftarg(mp->m_ddev_targp, mp->m_sb.sb_blocksize,
+				    mp->m_sb.sb_sectsize);
+	if (!error && logdev && logdev != ddev) {
 		unsigned int	log_sector_size = BBSIZE;
 
 		if (XFS_SB_VERSION_HASSECTOR(&mp->m_sb))
 			log_sector_size = mp->m_sb.sb_logsectsize;
-		xfs_setsize_buftarg(mp->m_logdev_targp, mp->m_sb.sb_blocksize,
-				    log_sector_size);
-	}
-	if (rtdev)
-		xfs_setsize_buftarg(mp->m_rtdev_targp, mp->m_sb.sb_blocksize,
-				    mp->m_sb.sb_blocksize);
+		error = xfs_setsize_buftarg(mp->m_logdev_targp,
+					    mp->m_sb.sb_blocksize,
+					    log_sector_size);
+	}
+	if (!error && rtdev)
+		error = xfs_setsize_buftarg(mp->m_rtdev_targp,
+					    mp->m_sb.sb_blocksize,
+					    mp->m_sb.sb_sectsize);
+	if (error)
+		goto error2;
 
-	if (!(error = XFS_IOINIT(vfsp, args, flags)))
+	error = XFS_IOINIT(vfsp, args, flags);
+	if (!error)
 		return 0;
-
- error:
+error2:
+	if (mp->m_sb_bp)
+		xfs_freesb(mp);
+error1:
 	xfs_binval(mp->m_ddev_targp);
-	if (logdev != NULL && logdev != ddev) {
+	if (logdev && logdev != ddev)
 		xfs_binval(mp->m_logdev_targp);
-	}
-	if (rtdev != NULL) {
+	if (rtdev)
 		xfs_binval(mp->m_rtdev_targp);
-	}
-	xfs_unmountfs_close(mp, NULL);
+error0:
+	xfs_unmountfs_close(mp, credp);
 	return error;
 }
 
@@ -1508,7 +1523,10 @@
 	 * eventually kicked out of the cache.
 	 */
 	if (flags & SYNC_REFCACHE) {
-		xfs_refcache_purge_some(mp);
+		if (flags & SYNC_WAIT)
+			xfs_refcache_purge_mp(mp);
+		else
+			xfs_refcache_purge_some(mp);
 	}
 
 	/*
@@ -1866,6 +1884,20 @@
 	return 0;
 }
 
+STATIC void
+xfs_freeze(
+	bhv_desc_t	*bdp)
+{
+	xfs_mount_t	*mp = XFS_BHVTOM(bdp);
+
+	while (atomic_read(&mp->m_active_trans) > 0)
+		delay(100);
+
+	/* Push the superblock and write an unmount record */
+	xfs_log_unmount_write(mp);
+	xfs_unmountfs_writesb(mp);
+}
+
 
 vfsops_t xfs_vfsops = {
 	BHV_IDENTITY_INIT(VFS_BHV_XFS,VFS_POSITION_XFS),
@@ -1883,4 +1915,5 @@
 	.vfs_get_inode		= xfs_get_inode,
 	.vfs_init_vnode		= xfs_initialize_vnode,
 	.vfs_force_shutdown	= xfs_do_force_shutdown,
+	.vfs_freeze		= xfs_freeze,
 };
diff -Nru a/fs/xfs/xfs_vnodeops.c b/fs/xfs/xfs_vnodeops.c
--- a/fs/xfs/xfs_vnodeops.c	2004-06-17 17:59:16 -07:00
+++ b/fs/xfs/xfs_vnodeops.c	2004-08-26 23:09:36 -07:00
@@ -283,7 +283,7 @@
 /*
  * xfs_setattr
  */
-STATIC int
+int
 xfs_setattr(
 	bhv_desc_t		*bdp,
 	vattr_t			*vap,
@@ -305,6 +305,7 @@
 	int			mandlock_before, mandlock_after;
 	struct xfs_dquot	*udqp, *gdqp, *olddquot1, *olddquot2;
 	int			file_owner;
+	int			need_iolock = (flags & ATTR_DMI) == 0;
 
 	vp = BHV_TO_VNODE(bdp);
 	vn_trace_entry(vp, __FUNCTION__, (inst_t *)__return_address);
@@ -406,7 +407,8 @@
 				goto error_return;
 			}
 		}
-		lock_flags |= XFS_IOLOCK_EXCL;
+		if (need_iolock)
+			lock_flags |= XFS_IOLOCK_EXCL;
 	}
 
 	xfs_ilock(ip, lock_flags);
@@ -678,7 +680,8 @@
 					     XFS_TRANS_PERM_LOG_RES,
 					     XFS_ITRUNCATE_LOG_COUNT))) {
 			xfs_trans_cancel(tp, 0);
-			xfs_iunlock(ip, XFS_IOLOCK_EXCL);
+			if (need_iolock)
+				xfs_iunlock(ip, XFS_IOLOCK_EXCL);
 			return code;
 		}
 		commit_flags = XFS_TRANS_RELEASE_LOG_RES;
@@ -823,10 +826,15 @@
 				mp->m_sb.sb_blocklog;
 		}
 		if (mask & XFS_AT_XFLAGS) {
-			ip->i_d.di_flags = 0;
-			if (vap->va_xflags & XFS_XFLAG_REALTIME) {
+			/* can't set PREALLOC this way, just preserve it */
+			ip->i_d.di_flags =
+				(ip->i_d.di_flags & XFS_DIFLAG_PREALLOC);
+			if (vap->va_xflags & XFS_XFLAG_REALTIME &&
+			    (ip->i_d.di_mode & S_IFMT) == S_IFREG) {
 				ip->i_d.di_flags |= XFS_DIFLAG_REALTIME;
 				ip->i_iocore.io_flags |= XFS_IOCORE_RT;
+			} else {
+				ip->i_iocore.io_flags &= ~XFS_IOCORE_RT;
 			}
 			if (vap->va_xflags & XFS_XFLAG_IMMUTABLE)
 				ip->i_d.di_flags |= XFS_DIFLAG_IMMUTABLE;
@@ -838,7 +846,9 @@
 				ip->i_d.di_flags |= XFS_DIFLAG_NOATIME;
 			if (vap->va_xflags & XFS_XFLAG_NODUMP)
 				ip->i_d.di_flags |= XFS_DIFLAG_NODUMP;
-			/* can't set PREALLOC this way, just ignore it */
+			if ((vap->va_xflags & XFS_XFLAG_RTINHERIT) &&
+			    (ip->i_d.di_mode & S_IFMT) == S_IFDIR)
+				ip->i_d.di_flags |= XFS_DIFLAG_RTINHERIT;
 		}
 		xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 		timeflags |= XFS_ICHGTIME_CHG;
@@ -1822,8 +1832,6 @@
 }
 
 
-#define	XFS_CREATE_NEW_MAXTRIES	10000
-
 /*
  * xfs_create (create a new file).
  */
@@ -4292,6 +4300,7 @@
 	int			rt;
 	xfs_fileoff_t		startoffset_fsb;
 	xfs_trans_t		*tp;
+	int			need_iolock = (attr_flags & ATTR_DMI) == 0;
 
 	vn_trace_entry(XFS_ITOV(ip), __FUNCTION__, (inst_t *)__return_address);
 	mp = ip->i_mount;
@@ -4319,7 +4328,8 @@
 			return(error);
 	}
 
-	xfs_ilock(ip, XFS_IOLOCK_EXCL);
+	if (need_iolock)
+		xfs_ilock(ip, XFS_IOLOCK_EXCL);
 	rounding = MAX((__uint8_t)(1 << mp->m_sb.sb_blocklog),
 			(__uint8_t)NBPP);
 	ilen = len + (offset & (rounding - 1));
@@ -4338,7 +4348,7 @@
 		error = xfs_bmapi(NULL, ip, startoffset_fsb, 1, 0, NULL, 0,
 			&imap, &nimap, NULL);
 		if (error)
-			return error;
+			goto out_unlock_iolock;
 		ASSERT(nimap == 0 || nimap == 1);
 		if (nimap && imap.br_startblock != HOLESTARTBLOCK) {
 			xfs_daddr_t	block;
@@ -4353,7 +4363,7 @@
 		error = xfs_bmapi(NULL, ip, endoffset_fsb - 1, 1, 0, NULL, 0,
 			&imap, &nimap, NULL);
 		if (error)
-			return error;
+			goto out_unlock_iolock;
 		ASSERT(nimap == 0 || nimap == 1);
 		if (nimap && imap.br_startblock != HOLESTARTBLOCK) {
 			ASSERT(imap.br_startblock != DELAYSTARTBLOCK);
@@ -4442,14 +4452,17 @@
 		xfs_iunlock(ip, XFS_ILOCK_EXCL);
 	}
 
-	xfs_iunlock(ip, XFS_IOLOCK_EXCL);
+ out_unlock_iolock:
+	if (need_iolock)
+		xfs_iunlock(ip, XFS_IOLOCK_EXCL);
 	return error;
 
  error0:
 	xfs_bmap_cancel(&free_list);
  error1:
 	xfs_trans_cancel(tp, XFS_TRANS_RELEASE_LOG_RES | XFS_TRANS_ABORT);
-	xfs_iunlock(ip, XFS_ILOCK_EXCL | XFS_IOLOCK_EXCL);
+	xfs_iunlock(ip, need_iolock ? (XFS_ILOCK_EXCL | XFS_IOLOCK_EXCL) :
+		    XFS_ILOCK_EXCL);
 	return error;
 }
 
@@ -4606,20 +4619,21 @@
 	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
 	xfs_trans_ihold(tp, ip);
 
-	ip->i_d.di_mode &= ~S_ISUID;
-
-	/*
-	 * Note that we don't have to worry about mandatory
-	 * file locking being disabled here because we only
-	 * clear the S_ISGID bit if the Group execute bit is
-	 * on, but if it was on then mandatory locking wouldn't
-	 * have been enabled.
-	 */
-	if (ip->i_d.di_mode & S_IXGRP)
-		ip->i_d.di_mode &= ~S_ISGID;
+	if ((attr_flags & ATTR_DMI) == 0) {
+		ip->i_d.di_mode &= ~S_ISUID;
 
-	xfs_ichgtime(ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
+		/*
+		 * Note that we don't have to worry about mandatory
+		 * file locking being disabled here because we only
+		 * clear the S_ISGID bit if the Group execute bit is
+		 * on, but if it was on then mandatory locking wouldn't
+		 * have been enabled.
+		 */
+		if (ip->i_d.di_mode & S_IXGRP)
+			ip->i_d.di_mode &= ~S_ISGID;
 
+		xfs_ichgtime(ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
+	}
 	if (setprealloc)
 		ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
 	else if (clrprealloc)
diff -Nru a/include/asm-i386/smpboot.h b/include/asm-i386/smpboot.h
--- a/include/asm-i386/smpboot.h	2004-04-20 16:00:02 -07:00
+++ b/include/asm-i386/smpboot.h	2004-08-26 10:48:47 -07:00
@@ -130,8 +130,8 @@
 			cpu = (cpu+1)%smp_num_cpus;
 			return cpu_to_physical_apicid(cpu);
 		default:
+			return cpu_online_map;
 	}
-	return cpu_online_map;
 }
 #else
 #define target_cpus() (cpu_online_map)
diff -Nru a/include/asm-sparc/sigcontext.h b/include/asm-sparc/sigcontext.h
--- a/include/asm-sparc/sigcontext.h	2002-02-05 09:39:47 -08:00
+++ b/include/asm-sparc/sigcontext.h	2004-08-24 17:28:32 -07:00
@@ -57,20 +57,6 @@
 	} si_fpqueue [16];
 } __siginfo_fpu_t;
 
-#ifdef __KERNEL__
-
-/* This magic should be in g_upper[0] for all upper parts
-   to be valid.
-   This is generated by sparc64 only, but for 32bit processes,
-   so we define it here as well.  */
-#define SIGINFO_EXTRA_V8PLUS_MAGIC      0x130e269
-typedef struct {
-	unsigned   int g_upper[8];
-	unsigned   int o_upper[8];
-} siginfo_extra_v8plus_t;
-
-#endif
-
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(__SPARC_SIGCONTEXT_H) */
diff -Nru a/include/asm-sparc64/pbm.h b/include/asm-sparc64/pbm.h
--- a/include/asm-sparc64/pbm.h	2003-08-02 17:45:01 -07:00
+++ b/include/asm-sparc64/pbm.h	2004-08-18 13:58:50 -07:00
@@ -70,6 +70,13 @@
 	 */
 	u32		lowest_consistent_map;
 
+	/* In order to deal with some buggy third-party PCI bridges that
+	 * do wrong prefetching, we never mark valid mappings as invalid.
+	 * Instead we point them at this dummy page.
+	 */
+	unsigned long	dummy_page;
+	unsigned long	dummy_page_pa;
+
 	/* If PBM_NCLUSTERS is ever decreased to 4 or lower,
 	 * or if largest supported page_table_sz * 8K goes above
 	 * 2GB, you must increase the size of the type of
@@ -92,6 +99,8 @@
 	 */
 	u32 dma_addr_mask;
 };
+
+extern void pci_iommu_table_init(struct pci_iommu *, int);
 
 /* This describes a PCI bus module's streaming buffer. */
 struct pci_strbuf {
diff -Nru a/include/asm-sparc64/sigcontext.h b/include/asm-sparc64/sigcontext.h
--- a/include/asm-sparc64/sigcontext.h	2002-02-05 09:39:50 -08:00
+++ b/include/asm-sparc64/sigcontext.h	2004-08-24 17:28:32 -07:00
@@ -83,18 +83,6 @@
 	unsigned long		sigc_mask;
 };
 
-#ifdef __KERNEL__
-
-/* This magic should be in g_upper[0] for all upper parts
-   to be valid.  */
-#define SIGINFO_EXTRA_V8PLUS_MAGIC	0x130e269
-typedef struct {
-	unsigned   int g_upper[8];
-	unsigned   int o_upper[8];
-} siginfo_extra_v8plus_t;
-
-#endif
-
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(__SPARC64_SIGCONTEXT_H) */
diff -Nru a/include/asm-sparc64/string.h b/include/asm-sparc64/string.h
--- a/include/asm-sparc64/string.h	2004-07-27 13:08:37 -07:00
+++ b/include/asm-sparc64/string.h	2004-08-26 14:25:45 -07:00
@@ -15,35 +15,25 @@
 
 #include <asm/asi.h>
 
-extern void __memmove(void *,const void *,__kernel_size_t);
 extern void *__memset(void *,int,__kernel_size_t);
-extern void *__builtin_memset(void *,int,__kernel_size_t);
 
 #ifndef EXPORT_SYMTAB_STROPS
 
 /* First the mem*() things. */
-#define __HAVE_ARCH_BCOPY
 #define __HAVE_ARCH_MEMMOVE
-
-#undef memmove
-#define memmove(_to, _from, _n) \
-({ \
-	void *_t = (_to); \
-	__memmove(_t, (_from), (_n)); \
-	_t; \
-})
+extern void *memmove(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMCPY
-
-extern void * memcpy(void *,const void *,__kernel_size_t);
+extern void *memcpy(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMSET
+extern void *__builtin_memset(void *,int,__kernel_size_t);
 
 static inline void *__constant_memset(void *s, int c, __kernel_size_t count)
 {
 	extern __kernel_size_t __bzero(void *, __kernel_size_t);
 
-	if(!c) {
+	if (!c) {
 		__bzero(s, count);
 		return s;
 	} else
@@ -61,19 +51,19 @@
 #define __HAVE_ARCH_MEMSCAN
 
 #undef memscan
-#define memscan(__arg0, __char, __arg2)						\
-({										\
-	extern void *__memscan_zero(void *, size_t);				\
-	extern void *__memscan_generic(void *, int, size_t);			\
-	void *__retval, *__addr = (__arg0);					\
-	size_t __size = (__arg2);						\
-										\
-	if(__builtin_constant_p(__char) && !(__char))				\
-		__retval = __memscan_zero(__addr, __size);			\
-	else									\
-		__retval = __memscan_generic(__addr, (__char), __size);		\
-										\
-	__retval;								\
+#define memscan(__arg0, __char, __arg2)					\
+({									\
+	extern void *__memscan_zero(void *, size_t);			\
+	extern void *__memscan_generic(void *, int, size_t);		\
+	void *__retval, *__addr = (__arg0);				\
+	size_t __size = (__arg2);					\
+									\
+	if(__builtin_constant_p(__char) && !(__char))			\
+		__retval = __memscan_zero(__addr, __size);		\
+	else								\
+		__retval = __memscan_generic(__addr, (__char), __size);	\
+									\
+	__retval;							\
 })
 
 #define __HAVE_ARCH_MEMCMP
diff -Nru a/include/asm-sparc64/uaccess.h b/include/asm-sparc64/uaccess.h
--- a/include/asm-sparc64/uaccess.h	2002-02-04 23:49:27 -08:00
+++ b/include/asm-sparc64/uaccess.h	2004-08-26 14:25:45 -07:00
@@ -249,26 +249,50 @@
 
 extern int __get_user_bad(void);
 
-extern __kernel_size_t __copy_from_user(void *to, const void *from,
-					__kernel_size_t size);
+extern unsigned long ___copy_from_user(void *to, const void *from,
+				       unsigned long size);
+extern unsigned long copy_from_user_fixup(void *to, const void *from,
+					  unsigned long size);
+static inline unsigned long copy_from_user(void *to, const void *from,
+					   unsigned long size)
+{
+	unsigned long ret = ___copy_from_user(to, from, size);
 
-extern __kernel_size_t __copy_to_user(void *to, const void *from,
-				      __kernel_size_t size);
+	if (ret)
+		ret = copy_from_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_from_user copy_from_user
 
-extern __kernel_size_t __copy_in_user(void *to, const void *from,
-				      __kernel_size_t size);
+extern unsigned long ___copy_to_user(void *to, const void *from,
+				     unsigned long size);
+extern unsigned long copy_to_user_fixup(void *to, const void *from,
+					unsigned long size);
+static inline unsigned long copy_to_user(void *to, const void *from,
+					 unsigned long size)
+{
+	unsigned long ret = ___copy_to_user(to, from, size);
 
-#define copy_from_user(to,from,n)		\
-	__copy_from_user((void *)(to),	\
-		    (void *)(from), (__kernel_size_t)(n))
+	if (ret)
+		ret = copy_to_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_to_user copy_to_user
 
-#define copy_to_user(to,from,n) \
-	__copy_to_user((void *)(to), \
-	(void *) (from), (__kernel_size_t)(n))
+extern unsigned long ___copy_in_user(void *to, const void *from,
+				     unsigned long size);
+extern unsigned long copy_in_user_fixup(void *to, void *from,
+					unsigned long size);
+static inline unsigned long copy_in_user(void *to, void *from,
+					 unsigned long size)
+{
+	unsigned long ret = ___copy_in_user(to, from, size);
 
-#define copy_in_user(to,from,n) \
-	__copy_in_user((void *)(to), \
-	(void *) (from), (__kernel_size_t)(n))
+	if (ret)
+		ret = copy_in_user_fixup(to, from, size);
+	return ret;
+}
+#define __copy_in_user copy_in_user
 
 extern __inline__ __kernel_size_t __clear_user(void *addr, __kernel_size_t size)
 {
diff -Nru a/include/linux/libata.h b/include/linux/libata.h
--- a/include/linux/libata.h	2004-08-14 12:38:51 -07:00
+++ b/include/linux/libata.h	2004-08-31 00:01:32 -07:00
@@ -157,6 +157,7 @@
 };
 
 /* forward declarations */
+struct scsi_device;
 struct ata_port_operations;
 struct ata_port;
 struct ata_queued_cmd;
diff -Nru a/include/linux/mm.h b/include/linux/mm.h
--- a/include/linux/mm.h	2004-08-18 09:08:54 -07:00
+++ b/include/linux/mm.h	2004-08-27 05:11:55 -07:00
@@ -309,7 +309,7 @@
 #define UnlockPage(page)	unlock_page(page)
 #define Page_Uptodate(page)	test_bit(PG_uptodate, &(page)->flags)
 #ifndef SetPageUptodate
-#define SetPageUptodate(page)	set_bit(PG_uptodate, &(page)->flags);
+#define SetPageUptodate(page)	set_bit(PG_uptodate, &(page)->flags)
 #endif
 #define ClearPageUptodate(page)	clear_bit(PG_uptodate, &(page)->flags)
 #define PageDirty(page)		test_bit(PG_dirty, &(page)->flags)
diff -Nru a/include/linux/netdevice.h b/include/linux/netdevice.h
--- a/include/linux/netdevice.h	2004-04-16 05:55:00 -07:00
+++ b/include/linux/netdevice.h	2004-08-18 12:58:04 -07:00
@@ -352,8 +352,8 @@
 
 	struct Qdisc		*qdisc;
 	struct Qdisc		*qdisc_sleeping;
-	struct Qdisc		*qdisc_list;
 	struct Qdisc		*qdisc_ingress;
+	struct list_head	qdisc_list;
 	unsigned long		tx_queue_len;	/* Max frames per queue allowed */
 
 	/* hard_start_xmit synchronizer */
diff -Nru a/include/linux/netfilter_ipv4/ip_conntrack.h b/include/linux/netfilter_ipv4/ip_conntrack.h
--- a/include/linux/netfilter_ipv4/ip_conntrack.h	2004-06-21 14:15:33 -07:00
+++ b/include/linux/netfilter_ipv4/ip_conntrack.h	2004-08-29 21:11:49 -07:00
@@ -249,6 +249,7 @@
 /* Call me when a conntrack is destroyed. */
 extern void (*ip_conntrack_destroyed)(struct ip_conntrack *conntrack);
 
+extern int ip_ct_no_defrag;
 /* Returns new sk_buff, or NULL */
 struct sk_buff *
 ip_ct_gather_frags(struct sk_buff *skb);
diff -Nru a/include/linux/netfilter_ipv4/ip_conntrack_ftp.h b/include/linux/netfilter_ipv4/ip_conntrack_ftp.h
--- a/include/linux/netfilter_ipv4/ip_conntrack_ftp.h	2002-08-08 07:49:17 -07:00
+++ b/include/linux/netfilter_ipv4/ip_conntrack_ftp.h	2004-09-03 00:10:28 -07:00
@@ -4,11 +4,6 @@
 
 #ifdef __KERNEL__
 
-#include <linux/netfilter_ipv4/lockhelp.h>
-
-/* Protects ftp part of conntracks */
-DECLARE_LOCK_EXTERN(ip_ftp_lock);
-
 #define FTP_PORT	21
 
 #endif /* __KERNEL__ */
diff -Nru a/include/linux/netfilter_ipv4/ip_conntrack_irc.h b/include/linux/netfilter_ipv4/ip_conntrack_irc.h
--- a/include/linux/netfilter_ipv4/ip_conntrack_irc.h	2002-08-08 07:49:17 -07:00
+++ b/include/linux/netfilter_ipv4/ip_conntrack_irc.h	2004-09-03 00:10:28 -07:00
@@ -33,17 +33,12 @@
 
 #ifdef __KERNEL__
 
-#include <linux/netfilter_ipv4/lockhelp.h>
-
 #define IRC_PORT	6667
 
 struct dccproto {
 	char* match;
 	int matchlen;
 };
-
-/* Protects irc part of conntracks */
-DECLARE_LOCK_EXTERN(ip_irc_lock);
 
 #endif /* __KERNEL__ */
 
diff -Nru a/include/linux/pci_ids.h b/include/linux/pci_ids.h
--- a/include/linux/pci_ids.h	2004-08-13 13:40:01 -07:00
+++ b/include/linux/pci_ids.h	2004-08-30 12:45:41 -07:00
@@ -981,13 +981,16 @@
 #define PCI_DEVICE_ID_NVIDIA_UVTNT2		0x002D
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_IDE	0x0035
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA	0x0036
+#define PCI_DEVICE_ID_NVIDIA_MCP04_AUDIO	0x003a
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2	0x003e
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_IDE	0x0053
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA	0x0054
 #define PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2	0x0055
+#define PCI_DEVICE_ID_NVIDIA_CK804_AUDIO	0x0059
 #define PCI_DEVICE_ID_NVIDIA_NFORCE2_IDE	0x0065
 #define PCI_DEVICE_ID_NVIDIA_MCP2_AUDIO		0x006a
 #define PCI_DEVICE_ID_NVIDIA_NFORCE2S_IDE	0x0085
+#define PCI_DEVICE_ID_NVIDIA_MCP2S_AUDIO	0x008a
 #define PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA	0x008e
 #define PCI_DEVICE_ID_NVIDIA_ITNT2		0x00A0
 #define PCI_DEVICE_ID_NVIDIA_NFORCE3		0x00d1
@@ -996,6 +999,7 @@
 #define PCI_DEVICE_ID_NVIDIA_NFORCE3S		0x00e1
 #define PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA	0x00e3
 #define PCI_DEVICE_ID_NVIDIA_NFORCE3S_IDE	0x00e5
+#define PCI_DEVICE_ID_NVIDIA_CK8S_AUDIO		0x00ea
 #define PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2	0x00ee
 #define PCI_DEVICE_ID_NVIDIA_GEFORCE_SDR	0x0100
 #define PCI_DEVICE_ID_NVIDIA_GEFORCE_DDR	0x0101
diff -Nru a/include/linux/pkt_sched.h b/include/linux/pkt_sched.h
--- a/include/linux/pkt_sched.h	2004-07-28 19:04:35 -07:00
+++ b/include/linux/pkt_sched.h	2004-08-30 16:40:56 -07:00
@@ -433,6 +433,16 @@
 #define TCA_ATM_MAX	TCA_ATM_STATE
 
 /* Network emulator */
+
+enum
+{
+	TCA_NETEM_UNSPEC,
+	TCA_NETEM_CORR,
+	TCA_NETEM_DELAY_DIST,
+};
+
+#define TCA_NETEM_MAX	TCA_NETEM_DELAY_DIST
+
 struct tc_netem_qopt
 {
 	__u32	latency;	/* added delay (us) */
@@ -442,5 +452,14 @@
 	__u32   duplicate;	/* random packet dup  (0=none ~0=100%) */
 	__u32	jitter;		/* random jitter in latency (us) */
 };
+
+struct tc_netem_corr
+{
+	__u32	delay_corr;	/* delay correlation */
+	__u32	loss_corr;	/* packet loss correlation */
+	__u32	dup_corr;	/* duplicate correlation  */
+};
+
+#define NETEM_DIST_SCALE	8192
 
 #endif
diff -Nru a/include/net/ip.h b/include/net/ip.h
--- a/include/net/ip.h	2004-08-15 19:28:55 -07:00
+++ b/include/net/ip.h	2004-08-29 21:11:49 -07:00
@@ -228,6 +228,7 @@
  */
  
 struct sk_buff *ip_defrag(struct sk_buff *skb);
+extern void ipfrag_flush(void);
 extern int ip_frag_nqueues;
 extern atomic_t ip_frag_mem;
 
diff -Nru a/include/net/pkt_sched.h b/include/net/pkt_sched.h
--- a/include/net/pkt_sched.h	2004-01-29 14:14:49 -08:00
+++ b/include/net/pkt_sched.h	2004-08-18 12:58:04 -07:00
@@ -80,11 +80,11 @@
 #define TCQ_F_THROTTLED	2
 #define TCQ_F_INGRES	4
 	struct Qdisc_ops	*ops;
-	struct Qdisc		*next;
 	u32			handle;
 	atomic_t		refcnt;
 	struct sk_buff_head	q;
 	struct net_device	*dev;
+	struct list_head	list;
 
 	struct tc_stats		stats;
 	int			(*reshape_fail)(struct sk_buff *skb, struct Qdisc *q);
diff -Nru a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
--- a/net/ipv4/ip_fragment.c	2003-05-28 00:49:28 -07:00
+++ b/net/ipv4/ip_fragment.c	2004-08-29 21:11:49 -07:00
@@ -168,14 +168,18 @@
 atomic_t ip_frag_mem = ATOMIC_INIT(0);	/* Memory used for fragments */
 
 /* Memory Tracking Functions. */
-static __inline__ void frag_kfree_skb(struct sk_buff *skb)
+static __inline__ void frag_kfree_skb(struct sk_buff *skb, int *work)
 {
+	if (work)
+		*work -= skb->truesize;
 	atomic_sub(skb->truesize, &ip_frag_mem);
 	kfree_skb(skb);
 }
 
-static __inline__ void frag_free_queue(struct ipq *qp)
+static __inline__ void frag_free_queue(struct ipq *qp, int *work)
 {
+	if (work)
+		*work -= sizeof(struct ipq);
 	atomic_sub(sizeof(struct ipq), &ip_frag_mem);
 	kfree(qp);
 }
@@ -194,7 +198,7 @@
 /* Destruction primitives. */
 
 /* Complete destruction of ipq. */
-static void ip_frag_destroy(struct ipq *qp)
+static void ip_frag_destroy(struct ipq *qp, int *work)
 {
 	struct sk_buff *fp;
 
@@ -206,18 +210,18 @@
 	while (fp) {
 		struct sk_buff *xp = fp->next;
 
-		frag_kfree_skb(fp);
+		frag_kfree_skb(fp, work);
 		fp = xp;
 	}
 
 	/* Finally, release the queue descriptor itself. */
-	frag_free_queue(qp);
+	frag_free_queue(qp, work);
 }
 
-static __inline__ void ipq_put(struct ipq *ipq)
+static __inline__ void ipq_put(struct ipq *ipq, int *work)
 {
 	if (atomic_dec_and_test(&ipq->refcnt))
-		ip_frag_destroy(ipq);
+		ip_frag_destroy(ipq, work);
 }
 
 /* Kill ipq entry. It is not destroyed immediately,
@@ -236,16 +240,19 @@
 }
 
 /* Memory limiting on fragments.  Evictor trashes the oldest 
- * fragment queue until we are back under the low threshold.
+ * fragment queue until we are back under the threshold.
  */
-static void ip_evictor(void)
+static void __ip_evictor(int threshold)
 {
 	struct ipq *qp;
 	struct list_head *tmp;
+	int work;
 
-	for(;;) {
-		if (atomic_read(&ip_frag_mem) <= sysctl_ipfrag_low_thresh)
-			return;
+	work = atomic_read(&ip_frag_mem) - threshold;
+	if (work <= 0)
+		return;
+
+	while (work > 0) {
 		read_lock(&ipfrag_lock);
 		if (list_empty(&ipq_lru_list)) {
 			read_unlock(&ipfrag_lock);
@@ -261,11 +268,16 @@
 			ipq_kill(qp);
 		spin_unlock(&qp->lock);
 
-		ipq_put(qp);
+		ipq_put(qp, &work);
 		IP_INC_STATS_BH(IpReasmFails);
 	}
 }
 
+static inline void ip_evictor(void)
+{
+	__ip_evictor(sysctl_ipfrag_low_thresh);
+}
+
 /*
  * Oops, a fragment queue timed out.  Kill it and send an ICMP reply.
  */
@@ -293,7 +305,7 @@
 	}
 out:
 	spin_unlock(&qp->lock);
-	ipq_put(qp);
+	ipq_put(qp, NULL);
 }
 
 /* Creation primitives. */
@@ -316,7 +328,7 @@
 			atomic_inc(&qp->refcnt);
 			write_unlock(&ipfrag_lock);
 			qp_in->last_in |= COMPLETE;
-			ipq_put(qp_in);
+			ipq_put(qp_in, NULL);
 			return qp;
 		}
 	}
@@ -505,7 +517,7 @@
 				qp->fragments = next;
 
 			qp->meat -= free_it->len;
-			frag_kfree_skb(free_it);
+			frag_kfree_skb(free_it, NULL);
 		}
 	}
 
@@ -656,7 +668,7 @@
 			ret = ip_frag_reasm(qp, dev);
 
 		spin_unlock(&qp->lock);
-		ipq_put(qp);
+		ipq_put(qp, NULL);
 		return ret;
 	}
 
@@ -674,4 +686,9 @@
 	ipfrag_secret_timer.function = ipfrag_secret_rebuild;
 	ipfrag_secret_timer.expires = jiffies + sysctl_ipfrag_secret_interval;
 	add_timer(&ipfrag_secret_timer);
+}
+
+void ipfrag_flush(void)
+{
+	__ip_evictor(0);
 }
diff -Nru a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
--- a/net/ipv4/netfilter/arp_tables.c	2004-06-10 22:41:46 -07:00
+++ b/net/ipv4/netfilter/arp_tables.c	2004-08-28 22:51:45 -07:00
@@ -101,7 +101,7 @@
 {
 	char *arpptr = (char *)(arphdr + 1);
 	char *src_devaddr, *tgt_devaddr;
-	u32 *src_ipaddr, *tgt_ipaddr;
+	u32 src_ipaddr, tgt_ipaddr;
 	int i, ret;
 
 #define FWINV(bool,invflg) ((bool) ^ !!(arpinfo->invflags & invflg))
@@ -140,11 +140,11 @@
 
 	src_devaddr = arpptr;
 	arpptr += dev->addr_len;
-	src_ipaddr = (u32 *) arpptr;
+	memcpy(&src_ipaddr, arpptr, sizeof(u32));
 	arpptr += sizeof(u32);
 	tgt_devaddr = arpptr;
 	arpptr += dev->addr_len;
-	tgt_ipaddr = (u32 *) arpptr;
+	memcpy(&tgt_ipaddr, arpptr, sizeof(u32));
 
 	if (FWINV(arp_devaddr_compare(&arpinfo->src_devaddr, src_devaddr, dev->addr_len),
 		  ARPT_INV_SRCDEVADDR) ||
@@ -155,19 +155,19 @@
 		return 0;
 	}
 
-	if (FWINV(((*src_ipaddr) & arpinfo->smsk.s_addr) != arpinfo->src.s_addr,
+	if (FWINV(((src_ipaddr) & arpinfo->smsk.s_addr) != arpinfo->src.s_addr,
 		  ARPT_INV_SRCIP) ||
-	    FWINV((((*tgt_ipaddr) & arpinfo->tmsk.s_addr) != arpinfo->tgt.s_addr),
+	    FWINV((((tgt_ipaddr) & arpinfo->tmsk.s_addr) != arpinfo->tgt.s_addr),
 		  ARPT_INV_TGTIP)) {
 		dprintf("Source or target IP address mismatch.\n");
 
 		dprintf("SRC: %u.%u.%u.%u. Mask: %u.%u.%u.%u. Target: %u.%u.%u.%u.%s\n",
-			NIPQUAD(*src_ipaddr),
+			NIPQUAD(src_ipaddr),
 			NIPQUAD(arpinfo->smsk.s_addr),
 			NIPQUAD(arpinfo->src.s_addr),
 			arpinfo->invflags & ARPT_INV_SRCIP ? " (INV)" : "");
 		dprintf("TGT: %u.%u.%u.%u Mask: %u.%u.%u.%u Target: %u.%u.%u.%u.%s\n",
-			NIPQUAD(*tgt_ipaddr),
+			NIPQUAD(tgt_ipaddr),
 			NIPQUAD(arpinfo->tmsk.s_addr),
 			NIPQUAD(arpinfo->tgt.s_addr),
 			arpinfo->invflags & ARPT_INV_TGTIP ? " (INV)" : "");
@@ -188,7 +188,10 @@
 	}
 
 	for (i = 0, ret = 0; i < IFNAMSIZ/sizeof(unsigned long); i++) {
-		ret |= (((const unsigned long *)outdev)[i]
+		unsigned long odev;
+		memcpy(&odev, outdev + i*sizeof(unsigned long),
+		       sizeof(unsigned long));
+		ret |= (odev
 			^ ((const unsigned long *)arpinfo->outiface)[i])
 			& ((const unsigned long *)arpinfo->outiface_mask)[i];
 	}
diff -Nru a/net/ipv4/netfilter/ip_conntrack_core.c b/net/ipv4/netfilter/ip_conntrack_core.c
--- a/net/ipv4/netfilter/ip_conntrack_core.c	2004-06-21 14:15:33 -07:00
+++ b/net/ipv4/netfilter/ip_conntrack_core.c	2004-08-29 21:11:49 -07:00
@@ -1183,6 +1183,8 @@
 	WRITE_UNLOCK(&ip_conntrack_lock);
 }
 
+int ip_ct_no_defrag;
+
 /* Returns new sk_buff, or NULL */
 struct sk_buff *
 ip_ct_gather_frags(struct sk_buff *skb)
@@ -1191,6 +1193,12 @@
 #ifdef CONFIG_NETFILTER_DEBUG
 	unsigned int olddebug = skb->nf_debug;
 #endif
+
+	if (unlikely(ip_ct_no_defrag)) {
+		kfree_skb(skb);
+		return NULL;
+	}
+
 	if (sk) {
 		sock_hold(sk);
 		skb_orphan(skb);
diff -Nru a/net/ipv4/netfilter/ip_conntrack_ftp.c b/net/ipv4/netfilter/ip_conntrack_ftp.c
--- a/net/ipv4/netfilter/ip_conntrack_ftp.c	2003-06-24 15:19:42 -07:00
+++ b/net/ipv4/netfilter/ip_conntrack_ftp.c	2004-09-03 00:10:28 -07:00
@@ -11,7 +11,7 @@
 #include <linux/netfilter_ipv4/ip_conntrack_helper.h>
 #include <linux/netfilter_ipv4/ip_conntrack_ftp.h>
 
-DECLARE_LOCK(ip_ftp_lock);
+static DECLARE_LOCK(ip_ftp_lock);
 struct module *ip_conntrack_ftp = THIS_MODULE;
 
 #define MAX_PORTS 8
@@ -338,7 +338,6 @@
 	memset(&expect, 0, sizeof(expect));
 
 	/* Update the ftp info */
-	LOCK_BH(&ip_ftp_lock);
 	if (htonl((array[0] << 24) | (array[1] << 16) | (array[2] << 8) | array[3])
 	    == ct->tuplehash[dir].tuple.src.ip) {
 		exp->seq = ntohl(tcph->seq) + matchoff;
@@ -358,7 +357,8 @@
 		   <lincoln@cesar.org.br> for reporting this potential
 		   problem (DMZ machines opening holes to internal
 		   networks, or the packet filter itself). */
-		if (!loose) goto out;
+		if (!loose)
+			return NF_ACCEPT;
 	}
 
 	exp->tuple = ((struct ip_conntrack_tuple)
@@ -376,9 +376,6 @@
 
 	/* Ignore failure; should only happen with NAT */
 	ip_conntrack_expect_related(ct, &expect);
- out:
-	UNLOCK_BH(&ip_ftp_lock);
-
 	return NF_ACCEPT;
 }
 
diff -Nru a/net/ipv4/netfilter/ip_conntrack_irc.c b/net/ipv4/netfilter/ip_conntrack_irc.c
--- a/net/ipv4/netfilter/ip_conntrack_irc.c	2003-10-02 23:22:51 -07:00
+++ b/net/ipv4/netfilter/ip_conntrack_irc.c	2004-09-03 00:10:28 -07:00
@@ -29,7 +29,6 @@
 #include <net/checksum.h>
 #include <net/tcp.h>
 
-#include <linux/netfilter_ipv4/lockhelp.h>
 #include <linux/netfilter_ipv4/ip_conntrack_helper.h>
 #include <linux/netfilter_ipv4/ip_conntrack_irc.h>
 
@@ -61,7 +60,6 @@
 };
 #define MINMATCHLEN	5
 
-DECLARE_LOCK(ip_irc_lock);
 struct module *ip_conntrack_irc = THIS_MODULE;
 
 #if 0
@@ -208,8 +206,6 @@
 			
 			memset(&expect, 0, sizeof(expect));
 
-			LOCK_BH(&ip_irc_lock);
-
 			/* save position of address in dcc string,
 			 * neccessary for NAT */
 			DEBUGP("tcph->seq = %u\n", tcph->seq);
@@ -236,8 +232,6 @@
 				ntohs(exp->tuple.dst.u.tcp.port));
 
 			ip_conntrack_expect_related(ct, &expect);
-			UNLOCK_BH(&ip_irc_lock);
-
 			return NF_ACCEPT;
 		} /* for .. NUM_DCCPROTO */
 	} /* while data < ... */
@@ -314,8 +308,6 @@
 		ip_conntrack_helper_unregister(&irc_helpers[i]);
 	}
 }
-
-EXPORT_SYMBOL(ip_irc_lock);
 
 module_init(init);
 module_exit(fini);
diff -Nru a/net/ipv4/netfilter/ip_conntrack_standalone.c b/net/ipv4/netfilter/ip_conntrack_standalone.c
--- a/net/ipv4/netfilter/ip_conntrack_standalone.c	2004-06-21 14:16:57 -07:00
+++ b/net/ipv4/netfilter/ip_conntrack_standalone.c	2004-08-29 21:11:49 -07:00
@@ -393,6 +393,13 @@
  cleanup_inandlocalops:
 	nf_unregister_hook(&ip_conntrack_local_out_ops);
  cleanup_inops:
+	/* Frag queues may hold fragments with skb->dst == NULL */
+	ip_ct_no_defrag = 1;
+	local_bh_disable();
+	br_write_lock(BR_NETPROTO_LOCK);
+	br_write_unlock(BR_NETPROTO_LOCK);
+	ipfrag_flush();
+	local_bh_enable();
 	nf_unregister_hook(&ip_conntrack_in_ops);
  cleanup_proc:
 	proc_net_remove("ip_conntrack");
diff -Nru a/net/ipv4/netfilter/ip_nat_ftp.c b/net/ipv4/netfilter/ip_nat_ftp.c
--- a/net/ipv4/netfilter/ip_nat_ftp.c	2003-06-24 15:19:42 -07:00
+++ b/net/ipv4/netfilter/ip_nat_ftp.c	2004-09-03 00:10:28 -07:00
@@ -24,8 +24,6 @@
 MODULE_PARM(ports, "1-" __MODULE_STRING(MAX_PORTS) "i");
 #endif
 
-DECLARE_LOCK_EXTERN(ip_ftp_lock);
-
 /* FIXME: Time out? --RR */
 
 static unsigned int
@@ -48,8 +46,6 @@
 	DEBUGP("nat_expected: We have a connection!\n");
 	exp_ftp_info = &ct->master->help.exp_ftp_info;
 
-	LOCK_BH(&ip_ftp_lock);
-
 	if (exp_ftp_info->ftptype == IP_CT_FTP_PORT
 	    || exp_ftp_info->ftptype == IP_CT_FTP_EPRT) {
 		/* PORT command: make connection go to the client. */
@@ -64,7 +60,6 @@
 		DEBUGP("nat_expected: PASV cmd. %u.%u.%u.%u->%u.%u.%u.%u\n",
 		       NIPQUAD(newsrcip), NIPQUAD(newdstip));
 	}
-	UNLOCK_BH(&ip_ftp_lock);
 
 	if (HOOK2MANIP(hooknum) == IP_NAT_MANIP_SRC)
 		newip = newsrcip;
@@ -100,8 +95,6 @@
 {
 	char buffer[sizeof("nnn,nnn,nnn,nnn,nnn,nnn")];
 
-	MUST_BE_LOCKED(&ip_ftp_lock);
-
 	sprintf(buffer, "%u,%u,%u,%u,%u,%u",
 		NIPQUAD(newip), port>>8, port&0xFF);
 
@@ -123,8 +116,6 @@
 {
 	char buffer[sizeof("|1|255.255.255.255|65535|")];
 
-	MUST_BE_LOCKED(&ip_ftp_lock);
-
 	sprintf(buffer, "|1|%u.%u.%u.%u|%u|", NIPQUAD(newip), port);
 
 	DEBUGP("calling ip_nat_mangle_tcp_packet\n");
@@ -145,8 +136,6 @@
 {
 	char buffer[sizeof("|||65535|")];
 
-	MUST_BE_LOCKED(&ip_ftp_lock);
-
 	sprintf(buffer, "|||%u|", port);
 
 	DEBUGP("calling ip_nat_mangle_tcp_packet\n");
@@ -166,7 +155,7 @@
     [IP_CT_FTP_EPSV] mangle_epsv_packet
 };
 
-static int ftp_data_fixup(const struct ip_ct_ftp_expect *ct_ftp_info,
+static int ftp_data_fixup(const struct ip_ct_ftp_expect *exp_ftp_info,
 			  struct ip_conntrack *ct,
 			  struct sk_buff **pskb,
 			  enum ip_conntrack_info ctinfo,
@@ -178,15 +167,14 @@
 	u_int16_t port;
 	struct ip_conntrack_tuple newtuple;
 
-	MUST_BE_LOCKED(&ip_ftp_lock);
 	DEBUGP("FTP_NAT: seq %u + %u in %u\n",
-	       expect->seq, ct_ftp_info->len,
+	       expect->seq, exp_ftp_info->len,
 	       ntohl(tcph->seq));
 
 	/* Change address inside packet to match way we're mapping
 	   this connection. */
-	if (ct_ftp_info->ftptype == IP_CT_FTP_PASV
-	    || ct_ftp_info->ftptype == IP_CT_FTP_EPSV) {
+	if (exp_ftp_info->ftptype == IP_CT_FTP_PASV
+	    || exp_ftp_info->ftptype == IP_CT_FTP_EPSV) {
 		/* PASV/EPSV response: must be where client thinks server
 		   is */
 		newip = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.ip;
@@ -208,7 +196,7 @@
 	newtuple.src.u.tcp.port = expect->tuple.src.u.tcp.port;
 
 	/* Try to get same port: if not, try to change it. */
-	for (port = ct_ftp_info->port; port != 0; port++) {
+	for (port = exp_ftp_info->port; port != 0; port++) {
 		newtuple.dst.u.tcp.port = htons(port);
 
 		if (ip_conntrack_change_expect(expect, &newtuple) == 0)
@@ -217,9 +205,9 @@
 	if (port == 0)
 		return 0;
 
-	if (!mangle[ct_ftp_info->ftptype](pskb, newip, port,
+	if (!mangle[exp_ftp_info->ftptype](pskb, newip, port,
 					  expect->seq - ntohl(tcph->seq),
-					  ct_ftp_info->len, ct, ctinfo))
+					  exp_ftp_info->len, ct, ctinfo))
 		return 0;
 
 	return 1;
@@ -236,12 +224,12 @@
 	struct tcphdr *tcph = (void *)iph + iph->ihl*4;
 	unsigned int datalen;
 	int dir;
-	struct ip_ct_ftp_expect *ct_ftp_info;
+	struct ip_ct_ftp_expect *exp_ftp_info;
 
 	if (!exp)
 		DEBUGP("ip_nat_ftp: no exp!!");
 
-	ct_ftp_info = &exp->help.exp_ftp_info;
+	exp_ftp_info = &exp->help.exp_ftp_info;
 
 	/* Only mangle things once: original direction in POST_ROUTING
 	   and reply direction on PRE_ROUTING. */
@@ -257,29 +245,23 @@
 	}
 
 	datalen = (*pskb)->len - iph->ihl * 4 - tcph->doff * 4;
-	LOCK_BH(&ip_ftp_lock);
 	/* If it's in the right range... */
-	if (between(exp->seq + ct_ftp_info->len,
+	if (between(exp->seq + exp_ftp_info->len,
 		    ntohl(tcph->seq),
 		    ntohl(tcph->seq) + datalen)) {
-		if (!ftp_data_fixup(ct_ftp_info, ct, pskb, ctinfo, exp)) {
-			UNLOCK_BH(&ip_ftp_lock);
+		if (!ftp_data_fixup(exp_ftp_info, ct, pskb, ctinfo, exp))
 			return NF_DROP;
-		}
 	} else {
 		/* Half a match?  This means a partial retransmisison.
 		   It's a cracker being funky. */
 		if (net_ratelimit()) {
 			printk("FTP_NAT: partial packet %u/%u in %u/%u\n",
-			       exp->seq, ct_ftp_info->len,
+			       exp->seq, exp_ftp_info->len,
 			       ntohl(tcph->seq),
 			       ntohl(tcph->seq) + datalen);
 		}
-		UNLOCK_BH(&ip_ftp_lock);
 		return NF_DROP;
 	}
-	UNLOCK_BH(&ip_ftp_lock);
-
 	return NF_ACCEPT;
 }
 
diff -Nru a/net/ipv4/netfilter/ip_nat_irc.c b/net/ipv4/netfilter/ip_nat_irc.c
--- a/net/ipv4/netfilter/ip_nat_irc.c	2003-09-12 21:05:49 -07:00
+++ b/net/ipv4/netfilter/ip_nat_irc.c	2004-09-03 00:10:28 -07:00
@@ -46,9 +46,6 @@
 MODULE_PARM_DESC(ports, "port numbers of IRC servers");
 #endif
 
-/* protects irc part of conntracks */
-DECLARE_LOCK_EXTERN(ip_irc_lock);
-
 /* FIXME: Time out? --RR */
 
 static unsigned int
@@ -89,7 +86,7 @@
 	return ip_nat_setup_info(ct, &mr, hooknum);
 }
 
-static int irc_data_fixup(const struct ip_ct_irc_expect *ct_irc_info,
+static int irc_data_fixup(const struct ip_ct_irc_expect *exp_irc_info,
 			  struct ip_conntrack *ct,
 			  struct sk_buff **pskb,
 			  enum ip_conntrack_info ctinfo,
@@ -104,23 +101,16 @@
 	/* "4294967296 65635 " */
 	char buffer[18];
 
-	MUST_BE_LOCKED(&ip_irc_lock);
-
 	DEBUGP("IRC_NAT: info (seq %u + %u) in %u\n",
-	       expect->seq, ct_irc_info->len,
+	       expect->seq, exp_irc_info->len,
 	       ntohl(tcph->seq));
 
 	newip = ct->tuplehash[IP_CT_DIR_REPLY].tuple.dst.ip;
 
 	/* Alter conntrack's expectations. */
-
-	/* We can read expect here without conntrack lock, since it's
-	   only set in ip_conntrack_irc, with ip_irc_lock held
-	   writable */
-
 	t = expect->tuple;
 	t.dst.ip = newip;
-	for (port = ct_irc_info->port; port != 0; port++) {
+	for (port = exp_irc_info->port; port != 0; port++) {
 		t.dst.u.tcp.port = htons(port);
 		if (ip_conntrack_change_expect(expect, &t) == 0) {
 			DEBUGP("using port %d", port);
@@ -150,7 +140,7 @@
 
 	return ip_nat_mangle_tcp_packet(pskb, ct, ctinfo, 
 					expect->seq - ntohl(tcph->seq),
-					ct_irc_info->len, buffer, 
+					exp_irc_info->len, buffer, 
 					strlen(buffer));
 }
 
@@ -165,12 +155,12 @@
 	struct tcphdr *tcph = (void *) iph + iph->ihl * 4;
 	unsigned int datalen;
 	int dir;
-	struct ip_ct_irc_expect *ct_irc_info;
+	struct ip_ct_irc_expect *exp_irc_info;
 
 	if (!exp)
 		DEBUGP("ip_nat_irc: no exp!!");
 		
-	ct_irc_info = &exp->help.exp_irc_info;
+	exp_irc_info = &exp->help.exp_irc_info;
 
 	/* Only mangle things once: original direction in POST_ROUTING
 	   and reply direction on PRE_ROUTING. */
@@ -187,30 +177,24 @@
 	DEBUGP("got beyond not touching\n");
 
 	datalen = (*pskb)->len - iph->ihl * 4 - tcph->doff * 4;
-	LOCK_BH(&ip_irc_lock);
 	/* Check wether the whole IP/address pattern is carried in the payload */
-	if (between(exp->seq + ct_irc_info->len,
+	if (between(exp->seq + exp_irc_info->len,
 		    ntohl(tcph->seq),
 		    ntohl(tcph->seq) + datalen)) {
-		if (!irc_data_fixup(ct_irc_info, ct, pskb, ctinfo, exp)) {
-			UNLOCK_BH(&ip_irc_lock);
+		if (!irc_data_fixup(exp_irc_info, ct, pskb, ctinfo, exp))
 			return NF_DROP;
-		}
 	} else { 
 		/* Half a match?  This means a partial retransmisison.
 		   It's a cracker being funky. */
 		if (net_ratelimit()) {
 			printk
 			    ("IRC_NAT: partial packet %u/%u in %u/%u\n",
-			     exp->seq, ct_irc_info->len,
+			     exp->seq, exp_irc_info->len,
 			     ntohl(tcph->seq),
 			     ntohl(tcph->seq) + datalen);
 		}
-		UNLOCK_BH(&ip_irc_lock);
 		return NF_DROP;
 	}
-	UNLOCK_BH(&ip_irc_lock);
-
 	return NF_ACCEPT;
 }
 
diff -Nru a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
--- a/net/ipv4/tcp_input.c	2004-08-02 18:46:17 -07:00
+++ b/net/ipv4/tcp_input.c	2004-08-18 17:57:42 -07:00
@@ -851,8 +851,10 @@
 	 * to low value, and then abruptly stops to do it and starts to delay
 	 * ACKs, wait for troubles.
 	 */
-	if (dst->rtt > tp->srtt)
+	if (dst->rtt > tp->srtt) {
 		tp->srtt = dst->rtt;
+		tp->rtt_seq = tp->snd_nxt;
+	}
 	if (dst->rttvar > tp->mdev) {
 		tp->mdev = dst->rttvar;
 		tp->mdev_max = tp->rttvar = max(tp->mdev, TCP_RTO_MIN);
diff -Nru a/net/ipv6/netfilter/ip6t_LOG.c b/net/ipv6/netfilter/ip6t_LOG.c
--- a/net/ipv6/netfilter/ip6t_LOG.c	2003-07-11 22:39:02 -07:00
+++ b/net/ipv6/netfilter/ip6t_LOG.c	2004-08-26 13:54:27 -07:00
@@ -56,7 +56,7 @@
 		repeatedly...with a large stick...no, an even LARGER
 		stick...no, you're still not thinking big enough */
 			nexthdr = **hdrptr;
-			hdrlen = *hdrptr[1] * 4 + 8;
+			hdrlen = (*hdrptr)[1] * 4 + 8;
 			*hdrptr = *hdrptr + hdrlen;
 			break;
 		/*stupid rfc2402 */
@@ -64,7 +64,7 @@
 		case IPPROTO_ROUTING:
 		case IPPROTO_HOPOPTS:
 			nexthdr = **hdrptr;
-			hdrlen = *hdrptr[1] * 8 + 8;
+			hdrlen = (*hdrptr)[1] * 8 + 8;
 			*hdrptr = *hdrptr + hdrlen;
 			break;
 		case IPPROTO_FRAGMENT:
diff -Nru a/net/ipv6/reassembly.c b/net/ipv6/reassembly.c
--- a/net/ipv6/reassembly.c	2004-07-15 13:33:03 -07:00
+++ b/net/ipv6/reassembly.c	2004-08-18 14:31:18 -07:00
@@ -191,14 +191,18 @@
 atomic_t ip6_frag_mem = ATOMIC_INIT(0);
 
 /* Memory Tracking Functions. */
-static inline void frag_kfree_skb(struct sk_buff *skb)
+static inline void frag_kfree_skb(struct sk_buff *skb, int *work)
 {
+	if (work)
+		*work -= skb->truesize;
 	atomic_sub(skb->truesize, &ip6_frag_mem);
 	kfree_skb(skb);
 }
 
-static inline void frag_free_queue(struct frag_queue *fq)
+static inline void frag_free_queue(struct frag_queue *fq, int *work)
 {
+	if (work)
+		*work -= sizeof(struct frag_queue);
 	atomic_sub(sizeof(struct frag_queue), &ip6_frag_mem);
 	kfree(fq);
 }
@@ -216,7 +220,7 @@
 /* Destruction primitives. */
 
 /* Complete destruction of fq. */
-static void ip6_frag_destroy(struct frag_queue *fq)
+static void ip6_frag_destroy(struct frag_queue *fq, int *work)
 {
 	struct sk_buff *fp;
 
@@ -228,17 +232,17 @@
 	while (fp) {
 		struct sk_buff *xp = fp->next;
 
-		frag_kfree_skb(fp);
+		frag_kfree_skb(fp, work);
 		fp = xp;
 	}
 
-	frag_free_queue(fq);
+	frag_free_queue(fq, work);
 }
 
-static __inline__ void fq_put(struct frag_queue *fq)
+static __inline__ void fq_put(struct frag_queue *fq, int *work)
 {
 	if (atomic_dec_and_test(&fq->refcnt))
-		ip6_frag_destroy(fq);
+		ip6_frag_destroy(fq, work);
 }
 
 /* Kill fq entry. It is not destroyed immediately,
@@ -260,10 +264,13 @@
 {
 	struct frag_queue *fq;
 	struct list_head *tmp;
+	int work;
 
-	for(;;) {
-		if (atomic_read(&ip6_frag_mem) <= sysctl_ip6frag_low_thresh)
-			return;
+	work = atomic_read(&ip6_frag_mem) - sysctl_ip6frag_low_thresh;
+	if (work <= 0)
+		return;
+
+	while(work > 0) {
 		read_lock(&ip6_frag_lock);
 		if (list_empty(&ip6_frag_lru_list)) {
 			read_unlock(&ip6_frag_lock);
@@ -279,7 +286,7 @@
 			fq_kill(fq);
 		spin_unlock(&fq->lock);
 
-		fq_put(fq);
+		fq_put(fq, &work);
 		IP6_INC_STATS_BH(Ip6ReasmFails);
 	}
 }
@@ -316,7 +323,7 @@
 	}
 out:
 	spin_unlock(&fq->lock);
-	fq_put(fq);
+	fq_put(fq, NULL);
 }
 
 /* Creation primitives. */
@@ -336,7 +343,7 @@
 			atomic_inc(&fq->refcnt);
 			write_unlock(&ip6_frag_lock);
 			fq_in->last_in |= COMPLETE;
-			fq_put(fq_in);
+			fq_put(fq_in, NULL);
 			return fq;
 		}
 	}
@@ -533,7 +540,7 @@
 				fq->fragments = next;
 
 			fq->meat -= free_it->len;
-			frag_kfree_skb(free_it);
+			frag_kfree_skb(free_it, NULL);
 		}
 	}
 
@@ -733,7 +740,7 @@
 			ret = ip6_frag_reasm(fq, skbp, dev);
 
 		spin_unlock(&fq->lock);
-		fq_put(fq);
+		fq_put(fq, NULL);
 		return ret;
 	}
 
diff -Nru a/net/irda/ircomm/ircomm_param.c b/net/irda/ircomm/ircomm_param.c
--- a/net/irda/ircomm/ircomm_param.c	2003-10-11 12:19:28 -07:00
+++ b/net/irda/ircomm/ircomm_param.c	2004-08-26 10:29:46 -07:00
@@ -198,7 +198,7 @@
 		IRDA_DEBUG(2, "%s(), No common service type to use!\n", __FUNCTION__);
 		return -1;
 	}
-	IRDA_DEBUG(0, __FUNCTION__ "%s(), services in common=%02x\n", __FUNCTION__,
+	IRDA_DEBUG(0, "%s(), services in common=%02x\n", __FUNCTION__ ,
 		   service_type);
 
 	/*
diff -Nru a/net/irda/irlmp.c b/net/irda/irlmp.c
--- a/net/irda/irlmp.c	2002-12-18 09:56:13 -08:00
+++ b/net/irda/irlmp.c	2004-08-26 10:38:48 -07:00
@@ -1241,7 +1241,7 @@
 	/* Get the number of lsap. That's the only safe way to know
 	 * that we have looped around... - Jean II */
 	lsap_todo = HASHBIN_GET_SIZE(self->lsaps);
-	IRDA_DEBUG(4, __FUNCTION__ "() : %d lsaps to scan\n", lsap_todo);
+	IRDA_DEBUG(4, "%s() : %d lsaps to scan\n", __FUNCTION__ , lsap_todo);
 
 	/* Poll lsap in order until the queue is full or until we
 	 * tried them all.
@@ -1255,7 +1255,7 @@
 			/* Note that if there is only one LSAP on the LAP
 			 * (most common case), self->flow_next is always NULL,
 			 * so we always avoid this loop. - Jean II */
-			IRDA_DEBUG(4, __FUNCTION__ "() : searching my LSAP\n");
+			IRDA_DEBUG(4, "%s() : searching my LSAP\n", __FUNCTION__ );
 
 			/* We look again in hashbins, because the lsap
 			 * might have gone away... - Jean II */
@@ -1274,14 +1274,14 @@
 
 		/* Next time, we will get the next one (or the first one) */
 		self->flow_next = (struct lsap_cb *) hashbin_get_next(self->lsaps);
-		IRDA_DEBUG(4, __FUNCTION__ "() : curr is %p, next was %p and is now %p, still %d to go - queue len = %d\n", curr, next, self->flow_next, lsap_todo, IRLAP_GET_TX_QUEUE_LEN(self->irlap));
+		IRDA_DEBUG(4, "%s() : curr is %p, next was %p and is now %p, still %d to go - queue len = %d\n", __FUNCTION__ , curr, next, self->flow_next, lsap_todo, IRLAP_GET_TX_QUEUE_LEN(self->irlap));
 
 		/* Inform lsap user that it can send one more packet. */
 		if (curr->notify.flow_indication != NULL)
 			curr->notify.flow_indication(curr->notify.instance, 
 						     curr, flow);
 		else
-			IRDA_DEBUG(1, __FUNCTION__ "(), no handler\n");
+			IRDA_DEBUG(1, "%s(), no handler\n", __FUNCTION__ );
 	}
 }
 
diff -Nru a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
--- a/net/netlink/af_netlink.c	2003-11-04 11:49:23 -08:00
+++ b/net/netlink/af_netlink.c	2004-08-30 17:04:34 -07:00
@@ -42,6 +42,7 @@
 #include <linux/notifier.h>
 #include <net/sock.h>
 #include <net/scm.h>
+#include <linux/tqueue.h>
 
 #define Nprintk(a...)
 
@@ -63,6 +64,15 @@
 	void			(*data_ready)(struct sock *sk, int bytes);
 };
 
+struct netlink_work
+{
+	struct sock 		*sk;
+	int 			len;
+	struct tq_struct	work;
+};
+
+static DECLARE_TASK_QUEUE(tq_netlink);
+static DECLARE_WAIT_QUEUE_HEAD(netlink_thread_wait);
 static struct sock *nl_table[MAX_LINKS];
 static DECLARE_WAIT_QUEUE_HEAD(nl_table_wait);
 static unsigned nl_nonroot[MAX_LINKS];
@@ -81,6 +91,15 @@
 
 static struct notifier_block *netlink_chain;
 
+void netlink_tq_handler(void *data)
+{
+	struct netlink_work *work = data;
+	
+	work->sk->data_ready(work->sk, work->len);
+	sock_put(work->sk);
+	kfree(work);
+}
+
 static void netlink_sock_destruct(struct sock *sk)
 {
 	skb_queue_purge(&sk->receive_queue);
@@ -437,6 +456,8 @@
 
 	if (atomic_read(&sk->rmem_alloc) > sk->rcvbuf ||
 	    test_bit(0, &sk->protinfo.af_netlink->state)) {
+		struct task_struct *client;
+		
 		if (!timeo) {
 			if (ssk->protinfo.af_netlink->pid == 0)
 				netlink_overrun(sk);
@@ -445,6 +466,19 @@
 			return -EAGAIN;
 		}
 
+		if (!sk->protinfo.af_netlink->pid) {
+			/* Kernel is sending information to user space
+			 * and socket buffer is full: Wake up user */
+			
+			client = find_task_by_pid(sk->protinfo.af_netlink->pid);
+			if (!client) {
+				sock_put(sk);
+				kfree_skb(skb);
+				return -EAGAIN;
+			}
+			wake_up_process(client);
+		}
+
 		__set_current_state(TASK_INTERRUPTIBLE);
 		add_wait_queue(&sk->protinfo.af_netlink->wait, &wait);
 
@@ -467,8 +501,26 @@
 	skb_orphan(skb);
 	skb_set_owner_r(skb, sk);
 	skb_queue_tail(&sk->receive_queue, skb);
-	sk->data_ready(sk, len);
-	sock_put(sk);
+
+	if (!sk->protinfo.af_netlink->pid) {
+		struct netlink_work *nlwork = 
+			kmalloc(sizeof(struct netlink_work), GFP_KERNEL);
+		
+		if  (!nlwork) {
+			sock_put(sk);
+			return -EAGAIN;
+		}
+		
+		INIT_TQUEUE(&nlwork->work, netlink_tq_handler, nlwork);
+		nlwork->sk = sk;
+		nlwork->len = len;
+		queue_task(&nlwork->work, &tq_netlink);
+		wake_up(&netlink_thread_wait);
+	} else {
+		sk->data_ready(sk, len);
+		sock_put(sk);
+	}
+	
 	return len;
 
 no_dst:
@@ -490,7 +542,22 @@
                 skb_orphan(skb);
 		skb_set_owner_r(skb, sk);
 		skb_queue_tail(&sk->receive_queue, skb);
-		sk->data_ready(sk, skb->len);
+
+		if (!sk->protinfo.af_netlink->pid) {
+			struct netlink_work *nlwork = 
+				kmalloc(sizeof(struct netlink_work), GFP_KERNEL);
+		
+			if  (!nlwork) 
+				return -EAGAIN;
+		
+			INIT_TQUEUE(&nlwork->work, netlink_tq_handler, nlwork);
+			nlwork->sk = sk;
+			nlwork->len = skb->len;
+			queue_task(&nlwork->work, &tq_netlink);
+			wake_up(&netlink_thread_wait);
+		} else 
+			sk->data_ready(sk, skb->len);
+
 		return 0;
 	}
 	return -1;
@@ -534,11 +601,12 @@
 			netlink_overrun(sk);
 			/* Clone failed. Notify ALL listeners. */
 			failure = 1;
+			sock_put(sk);
 		} else if (netlink_broadcast_deliver(sk, skb2)) {
 			netlink_overrun(sk);
+			sock_put(sk);
 		} else
 			skb2 = NULL;
-		sock_put(sk);
 	}
 
 	netlink_unlock_table();
@@ -868,6 +936,26 @@
 	netlink_unicast(in_skb->sk, skb, NETLINK_CB(in_skb).pid, MSG_DONTWAIT);
 }
 
+static int netlink_thread(void *unused)
+{
+	struct task_struct *tsk = current;
+	DECLARE_WAITQUEUE(wait, tsk);
+
+	daemonize();
+	strcpy(tsk->comm, "knetlinkd");
+	sigfillset(&tsk->blocked);
+	mb();
+
+	for (;;) {
+		run_task_queue(&tq_netlink);
+		
+		__set_current_state(TASK_INTERRUPTIBLE);
+		add_wait_queue(&netlink_thread_wait, &wait);
+		schedule();
+		__set_current_state(TASK_RUNNING);
+		remove_wait_queue(&netlink_thread_wait, &wait);
+	}
+}
 
 #ifdef NL_EMULATE_DEV
 
@@ -1027,6 +1115,8 @@
 #ifdef CONFIG_PROC_FS
 	create_proc_read_entry("net/netlink", 0, 0, netlink_read_proc, NULL);
 #endif
+	kernel_thread(netlink_thread, NULL, CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
+	
 	return 0;
 }
 
diff -Nru a/net/netsyms.c b/net/netsyms.c
--- a/net/netsyms.c	2004-06-08 11:27:18 -07:00
+++ b/net/netsyms.c	2004-08-29 21:11:49 -07:00
@@ -283,6 +283,7 @@
 EXPORT_SYMBOL(inetdev_by_index);
 EXPORT_SYMBOL(in_dev_finish_destroy);
 EXPORT_SYMBOL(ip_defrag);
+EXPORT_SYMBOL(ipfrag_flush);
 
 /* Route manipulation */
 EXPORT_SYMBOL(ip_rt_ioctl);
diff -Nru a/net/sched/sch_api.c b/net/sched/sch_api.c
--- a/net/sched/sch_api.c	2002-07-18 23:30:42 -07:00
+++ b/net/sched/sch_api.c	2004-08-18 12:58:04 -07:00
@@ -32,6 +32,7 @@
 #include <linux/init.h>
 #include <linux/proc_fs.h>
 #include <linux/kmod.h>
+#include <linux/list.h>
 
 #include <net/sock.h>
 #include <net/pkt_sched.h>
@@ -193,7 +194,7 @@
 {
 	struct Qdisc *q;
 
-	for (q = dev->qdisc_list; q; q = q->next) {
+	list_for_each_entry(q, &dev->qdisc_list, list) {
 		if (q->handle == handle)
 			return q;
 	}
@@ -424,6 +425,7 @@
 
 	memset(sch, 0, size);
 
+	INIT_LIST_HEAD(&sch->list);
 	skb_queue_head_init(&sch->q);
 
 	if (handle == TC_H_INGRESS)
@@ -449,8 +451,7 @@
 
 	if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS-1])) == 0) {
 		write_lock(&qdisc_tree_lock);
-		sch->next = dev->qdisc_list;
-		dev->qdisc_list = sch;
+		list_add_tail(&sch->list, &dev->qdisc_list);
 		write_unlock(&qdisc_tree_lock);
 #ifdef CONFIG_NET_ESTIMATOR
 		if (tca[TCA_RATE-1])
@@ -805,15 +806,18 @@
 		if (idx > s_idx)
 			s_q_idx = 0;
 		read_lock(&qdisc_tree_lock);
-		for (q = dev->qdisc_list, q_idx = 0; q;
-		     q = q->next, q_idx++) {
-			if (q_idx < s_q_idx)
+		q_idx = 0;
+		list_for_each_entry(q, &dev->qdisc_list, list) {
+			if (q_idx < s_q_idx) {
+				q_idx++;
 				continue;
+			}
 			if (tc_fill_qdisc(skb, q, 0, NETLINK_CB(cb->skb).pid,
 					  cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) {
 				read_unlock(&qdisc_tree_lock);
 				goto done;
 			}
+			q_idx++;
 		}
 		read_unlock(&qdisc_tree_lock);
 	}
@@ -1024,13 +1028,16 @@
 		return 0;
 
 	s_t = cb->args[0];
+	t = 0;
 
 	read_lock(&qdisc_tree_lock);
-	for (q=dev->qdisc_list, t=0; q; q = q->next, t++) {
-		if (t < s_t) continue;
-		if (!q->ops->cl_ops) continue;
-		if (tcm->tcm_parent && TC_H_MAJ(tcm->tcm_parent) != q->handle)
+	list_for_each_entry(q, &dev->qdisc_list, list) {
+		if (t < s_t || !q->ops->cl_ops ||
+		    (tcm->tcm_parent &&
+		     TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
+			t++;
 			continue;
+		}
 		if (t > s_t)
 			memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
 		arg.w.fn = qdisc_class_dump;
@@ -1043,6 +1050,7 @@
 		cb->args[1] = arg.w.count;
 		if (arg.w.stop)
 			break;
+		t++;
 	}
 	read_unlock(&qdisc_tree_lock);
 
diff -Nru a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
--- a/net/sched/sch_cbq.c	2004-06-18 13:53:45 -07:00
+++ b/net/sched/sch_cbq.c	2004-08-22 21:31:25 -07:00
@@ -1712,15 +1712,18 @@
 	}
 }
 
-static void cbq_destroy_class(struct cbq_class *cl)
+static void cbq_destroy_class(struct Qdisc *sch, struct cbq_class *cl)
 {
+	struct cbq_sched_data *q = (struct cbq_sched_data *)sch->data;
+
 	cbq_destroy_filters(cl);
 	qdisc_destroy(cl->q);
 	qdisc_put_rtab(cl->R_tab);
 #ifdef CONFIG_NET_ESTIMATOR
 	qdisc_kill_estimator(&cl->stats);
 #endif
-	kfree(cl);
+	if (cl != &q->link)
+		kfree(cl);
 }
 
 static void
@@ -1743,8 +1746,7 @@
 
 		for (cl = q->classes[h]; cl; cl = next) {
 			next = cl->next;
-			if (cl != &q->link)
-				cbq_destroy_class(cl);
+			cbq_destroy_class(sch, cl);
 		}
 	}
 
@@ -1766,7 +1768,7 @@
 		spin_unlock_bh(&sch->dev->queue_lock);
 #endif
 
-		cbq_destroy_class(cl);
+		cbq_destroy_class(sch, cl);
 	}
 }
 
@@ -2000,7 +2002,7 @@
 	sch_tree_unlock(sch);
 
 	if (--cl->refcnt == 0)
-		cbq_destroy_class(cl);
+		cbq_destroy_class(sch, cl);
 
 	return 0;
 }
diff -Nru a/net/sched/sch_generic.c b/net/sched/sch_generic.c
--- a/net/sched/sch_generic.c	2003-11-18 17:39:23 -08:00
+++ b/net/sched/sch_generic.c	2004-08-18 12:58:04 -07:00
@@ -29,6 +29,7 @@
 #include <linux/skbuff.h>
 #include <linux/rtnetlink.h>
 #include <linux/init.h>
+#include <linux/list.h>
 #include <net/sock.h>
 #include <net/pkt_sched.h>
 
@@ -391,6 +392,7 @@
 		return NULL;
 	memset(sch, 0, size);
 
+	INIT_LIST_HEAD(&sch->list);
 	skb_queue_head_init(&sch->q);
 	sch->ops = ops;
 	sch->enqueue = ops->enqueue;
@@ -420,22 +422,10 @@
 void qdisc_destroy(struct Qdisc *qdisc)
 {
 	struct Qdisc_ops *ops = qdisc->ops;
-	struct net_device *dev;
 
 	if (!atomic_dec_and_test(&qdisc->refcnt))
 		return;
-
-	dev = qdisc->dev;
-
-	if (dev) {
-		struct Qdisc *q, **qp;
-		for (qp = &qdisc->dev->qdisc_list; (q=*qp) != NULL; qp = &q->next) {
-			if (q == qdisc) {
-				*qp = q->next;
-				break;
-			}
-		}
-	}
+	list_del(&qdisc->list);
 #ifdef CONFIG_NET_ESTIMATOR
 	qdisc_kill_estimator(&qdisc->stats);
 #endif
@@ -464,10 +454,8 @@
 				printk(KERN_INFO "%s: activation failed\n", dev->name);
 				return;
 			}
-
 			write_lock(&qdisc_tree_lock);
-			qdisc->next = dev->qdisc_list;
-			dev->qdisc_list = qdisc;
+			list_add_tail(&qdisc->list, &dev->qdisc_list);
 			write_unlock(&qdisc_tree_lock);
 
 		} else {
@@ -513,7 +501,7 @@
 	dev->qdisc = &noop_qdisc;
 	spin_unlock_bh(&dev->queue_lock);
 	dev->qdisc_sleeping = &noop_qdisc;
-	dev->qdisc_list = NULL;
+	INIT_LIST_HEAD(&dev->qdisc_list);
 	write_unlock(&qdisc_tree_lock);
 
 	dev_watchdog_init(dev);
@@ -535,9 +523,8 @@
 		qdisc_destroy(qdisc);
         }
 #endif
-	BUG_TRAP(dev->qdisc_list == NULL);
+	BUG_TRAP(list_empty(&dev->qdisc_list));
 	BUG_TRAP(!timer_pending(&dev->watchdog_timer));
-	dev->qdisc_list = NULL;
 	spin_unlock_bh(&dev->queue_lock);
 	write_unlock(&qdisc_tree_lock);
 }
diff -Nru a/net/sched/sch_netem.c b/net/sched/sch_netem.c
--- a/net/sched/sch_netem.c	2004-08-03 14:11:55 -07:00
+++ b/net/sched/sch_netem.c	2004-08-30 16:40:56 -07:00
@@ -6,13 +6,15 @@
  * 		as published by the Free Software Foundation; either version
  * 		2 of the License, or (at your option) any later version.
  *
+ *  		Many of the algorithms and ideas for this came from
+ *		NIST Net which is not copyrighted. 
+ *
  * Authors:	Stephen Hemminger <shemminger@osdl.org>
  *		Catalin(ux aka Dino) BOIE <catab at umbrella dot ro>
  */
 
 #include <linux/config.h>
 #include <linux/module.h>
-#include <linux/init.h>
 #include <asm/bitops.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
@@ -23,11 +25,33 @@
 
 #include <net/pkt_sched.h>
 
-/*	Network emulator
- *
- *	This scheduler can alters spacing and order
- *	Similar to NISTnet and BSD Dummynet.
- */
+#define qdisc_priv(q)	((void *)(q->data))
+
+/*	Network Emulation Queuing algorithm.
+	====================================
+
+	Sources: [1] Mark Carson, Darrin Santay, "NIST Net - A Linux-based
+		 Network Emulation Tool
+		 [2] Luigi Rizzo, DummyNet for FreeBSD
+
+	 ----------------------------------------------------------------
+
+	 This started out as a simple way to delay outgoing packets to
+	 test TCP but has grown to include most of the functionality
+	 of a full blown network emulator like NISTnet. It can delay
+	 packets and add random jitter (and correlation). The random
+	 distribution can be loaded from a table as well to provide
+	 normal, Pareto, or experimental curves. Packet loss,
+	 duplication, and reordering can also be emulated.
+
+	 This qdisc does not do classification that can be handled in
+	 layering other disciplines.  It does not need to do bandwidth
+	 control either since that can be handled by using token
+	 bucket or other rate control.
+
+	 The simulator is limited by the Linux timer resolution
+	 and will create packet bursts on the HZ boundary (1ms).
+*/
 
 struct netem_sched_data {
 	struct Qdisc	*qdisc;
@@ -40,6 +64,17 @@
 	u32 counter;
 	u32 gap;
 	u32 jitter;
+	u32 duplicate;
+
+	struct crndstate {
+		unsigned long last;
+		unsigned long rho;
+	} delay_cor, loss_cor, dup_cor;
+
+	struct disttable {
+		u32  size;
+		s16 table[0];
+	} *delay_dist;
 };
 
 /* Time stamp put into socket buffer control block */
@@ -47,576 +82,110 @@
 	psched_time_t	time_to_send;
 };
 
-/* This is the distribution table for the normal distribution produced
- * with NISTnet tools.
- * The entries represent a scaled inverse of the cumulative distribution
- * function.
+/* init_crandom - initialize correlated random number generator
+ * Use entropy source for initial seed.
  */
-#define TABLESIZE	2048
-#define TABLEFACTOR	8192
+static void init_crandom(struct crndstate *state, unsigned long rho)
+{
+	state->rho = rho;
+	state->last = net_random();
+}
 
-static const short disttable[TABLESIZE] = {
-	-31473,		-26739,		-25226,		-24269,
-	-23560,		-22993,		-22518,		-22109,
-	-21749,		-21426,		-21133,		-20865,
-	-20618,		-20389,		-20174,		-19972,
-	-19782,		-19601,		-19430,		-19267,
-	-19112,		-18962,		-18819,		-18681,
-	-18549,		-18421,		-18298,		-18178,
-	-18062,		-17950,		-17841,		-17735,
-	-17632,		-17532,		-17434,		-17339,
-	-17245,		-17155,		-17066,		-16979,
-	-16894,		-16811,		-16729,		-16649,
-	-16571,		-16494,		-16419,		-16345,
-	-16272,		-16201,		-16130,		-16061,
-	-15993,		-15926,		-15861,		-15796,
-	-15732,		-15669,		-15607,		-15546,
-	-15486,		-15426,		-15368,		-15310,
-	-15253,		-15196,		-15140,		-15086,
-	-15031,		-14977,		-14925,		-14872,
-	-14821,		-14769,		-14719,		-14669,
-	-14619,		-14570,		-14522,		-14473,
-	-14426,		-14379,		-14332,		-14286,
-	-14241,		-14196,		-14150,		-14106,
-	-14062,		-14019,		-13976,		-13933,
-	-13890,		-13848,		-13807,		-13765,
-	-13724,		-13684,		-13643,		-13604,
-	-13564,		-13525,		-13486,		-13447,
-	-13408,		-13370,		-13332,		-13295,
-	-13258,		-13221,		-13184,		-13147,
-	-13111,		-13075,		-13040,		-13004,
-	-12969,		-12934,		-12899,		-12865,
-	-12830,		-12796,		-12762,		-12729,
-	-12695,		-12662,		-12629,		-12596,
-	-12564,		-12531,		-12499,		-12467,
-	-12435,		-12404,		-12372,		-12341,
-	-12310,		-12279,		-12248,		-12218,
-	-12187,		-12157,		-12127,		-12097,
-	-12067,		-12038,		-12008,		-11979,
-	-11950,		-11921,		-11892,		-11863,
-	-11835,		-11806,		-11778,		-11750,
-	-11722,		-11694,		-11666,		-11639,
-	-11611,		-11584,		-11557,		-11530,
-	-11503,		-11476,		-11450,		-11423,
-	-11396,		-11370,		-11344,		-11318,
-	-11292,		-11266,		-11240,		-11214,
-	-11189,		-11164,		-11138,		-11113,
-	-11088,		-11063,		-11038,		-11013,
-	-10988,		-10964,		-10939,		-10915,
-	-10891,		-10866,		-10843,		-10818,
-	-10794,		-10770,		-10747,		-10723,
-	-10700,		-10676,		-10652,		-10630,
-	-10606,		-10583,		-10560,		-10537,
-	-10514,		-10491,		-10469,		-10446,
-	-10424,		-10401,		-10378,		-10356,
-	-10334,		-10312,		-10290,		-10267,
-	-10246,		-10224,		-10202,		-10180,
-	-10158,		-10137,		-10115,		-10094,
-	-10072,		-10051,		-10030,		-10009,
-	-9988,		-9967,		-9945,		-9925,
-	-9904,		-9883,		-9862,		-9842,
-	-9821,		-9800,		-9780,		-9760,
-	-9739,		-9719,		-9699,		-9678,
-	-9658,		-9638,		-9618,		-9599,
-	-9578,		-9559,		-9539,		-9519,
-	-9499,		-9480,		-9461,		-9441,
-	-9422,		-9402,		-9383,		-9363,
-	-9344,		-9325,		-9306,		-9287,
-	-9268,		-9249,		-9230,		-9211,
-	-9192,		-9173,		-9155,		-9136,
-	-9117,		-9098,		-9080,		-9062,
-	-9043,		-9025,		-9006,		-8988,
-	-8970,		-8951,		-8933,		-8915,
-	-8897,		-8879,		-8861,		-8843,
-	-8825,		-8807,		-8789,		-8772,
-	-8754,		-8736,		-8718,		-8701,
-	-8683,		-8665,		-8648,		-8630,
-	-8613,		-8595,		-8578,		-8561,
-	-8543,		-8526,		-8509,		-8492,
-	-8475,		-8458,		-8441,		-8423,
-	-8407,		-8390,		-8373,		-8356,
-	-8339,		-8322,		-8305,		-8289,
-	-8272,		-8255,		-8239,		-8222,
-	-8206,		-8189,		-8172,		-8156,
-	-8140,		-8123,		-8107,		-8090,
-	-8074,		-8058,		-8042,		-8025,
-	-8009,		-7993,		-7977,		-7961,
-	-7945,		-7929,		-7913,		-7897,
-	-7881,		-7865,		-7849,		-7833,
-	-7817,		-7802,		-7786,		-7770,
-	-7754,		-7739,		-7723,		-7707,
-	-7692,		-7676,		-7661,		-7645,
-	-7630,		-7614,		-7599,		-7583,
-	-7568,		-7553,		-7537,		-7522,
-	-7507,		-7492,		-7476,		-7461,
-	-7446,		-7431,		-7416,		-7401,
-	-7385,		-7370,		-7356,		-7340,
-	-7325,		-7311,		-7296,		-7281,
-	-7266,		-7251,		-7236,		-7221,
-	-7207,		-7192,		-7177,		-7162,
-	-7148,		-7133,		-7118,		-7104,
-	-7089,		-7075,		-7060,		-7046,
-	-7031,		-7016,		-7002,		-6988,
-	-6973,		-6959,		-6944,		-6930,
-	-6916,		-6901,		-6887,		-6873,
-	-6859,		-6844,		-6830,		-6816,
-	-6802,		-6788,		-6774,		-6760,
-	-6746,		-6731,		-6717,		-6704,
-	-6690,		-6675,		-6661,		-6647,
-	-6633,		-6620,		-6606,		-6592,
-	-6578,		-6564,		-6550,		-6537,
-	-6523,		-6509,		-6495,		-6482,
-	-6468,		-6454,		-6441,		-6427,
-	-6413,		-6400,		-6386,		-6373,
-	-6359,		-6346,		-6332,		-6318,
-	-6305,		-6291,		-6278,		-6264,
-	-6251,		-6238,		-6224,		-6211,
-	-6198,		-6184,		-6171,		-6158,
-	-6144,		-6131,		-6118,		-6105,
-	-6091,		-6078,		-6065,		-6052,
-	-6039,		-6025,		-6012,		-5999,
-	-5986,		-5973,		-5960,		-5947,
-	-5934,		-5921,		-5908,		-5895,
-	-5882,		-5869,		-5856,		-5843,
-	-5830,		-5817,		-5804,		-5791,
-	-5779,		-5766,		-5753,		-5740,
-	-5727,		-5714,		-5702,		-5689,
-	-5676,		-5663,		-5650,		-5638,
-	-5625,		-5612,		-5600,		-5587,
-	-5575,		-5562,		-5549,		-5537,
-	-5524,		-5512,		-5499,		-5486,
-	-5474,		-5461,		-5449,		-5436,
-	-5424,		-5411,		-5399,		-5386,
-	-5374,		-5362,		-5349,		-5337,
-	-5324,		-5312,		-5299,		-5287,
-	-5275,		-5263,		-5250,		-5238,
-	-5226,		-5213,		-5201,		-5189,
-	-5177,		-5164,		-5152,		-5140,
-	-5128,		-5115,		-5103,		-5091,
-	-5079,		-5067,		-5055,		-5043,
-	-5030,		-5018,		-5006,		-4994,
-	-4982,		-4970,		-4958,		-4946,
-	-4934,		-4922,		-4910,		-4898,
-	-4886,		-4874,		-4862,		-4850,
-	-4838,		-4826,		-4814,		-4803,
-	-4791,		-4778,		-4767,		-4755,
-	-4743,		-4731,		-4719,		-4708,
-	-4696,		-4684,		-4672,		-4660,
-	-4649,		-4637,		-4625,		-4613,
-	-4601,		-4590,		-4578,		-4566,
-	-4554,		-4543,		-4531,		-4520,
-	-4508,		-4496,		-4484,		-4473,
-	-4461,		-4449,		-4438,		-4427,
-	-4415,		-4403,		-4392,		-4380,
-	-4368,		-4357,		-4345,		-4334,
-	-4322,		-4311,		-4299,		-4288,
-	-4276,		-4265,		-4253,		-4242,
-	-4230,		-4219,		-4207,		-4196,
-	-4184,		-4173,		-4162,		-4150,
-	-4139,		-4128,		-4116,		-4105,
-	-4094,		-4082,		-4071,		-4060,
-	-4048,		-4037,		-4026,		-4014,
-	-4003,		-3992,		-3980,		-3969,
-	-3958,		-3946,		-3935,		-3924,
-	-3913,		-3901,		-3890,		-3879,
-	-3868,		-3857,		-3845,		-3834,
-	-3823,		-3812,		-3801,		-3790,
-	-3779,		-3767,		-3756,		-3745,
-	-3734,		-3723,		-3712,		-3700,
-	-3689,		-3678,		-3667,		-3656,
-	-3645,		-3634,		-3623,		-3612,
-	-3601,		-3590,		-3579,		-3568,
-	-3557,		-3545,		-3535,		-3524,
-	-3513,		-3502,		-3491,		-3480,
-	-3469,		-3458,		-3447,		-3436,
-	-3425,		-3414,		-3403,		-3392,
-	-3381,		-3370,		-3360,		-3348,
-	-3337,		-3327,		-3316,		-3305,
-	-3294,		-3283,		-3272,		-3262,
-	-3251,		-3240,		-3229,		-3218,
-	-3207,		-3197,		-3185,		-3175,
-	-3164,		-3153,		-3142,		-3132,
-	-3121,		-3110,		-3099,		-3088,
-	-3078,		-3067,		-3056,		-3045,
-	-3035,		-3024,		-3013,		-3003,
-	-2992,		-2981,		-2970,		-2960,
-	-2949,		-2938,		-2928,		-2917,
-	-2906,		-2895,		-2885,		-2874,
-	-2864,		-2853,		-2842,		-2832,
-	-2821,		-2810,		-2800,		-2789,
-	-2778,		-2768,		-2757,		-2747,
-	-2736,		-2725,		-2715,		-2704,
-	-2694,		-2683,		-2673,		-2662,
-	-2651,		-2641,		-2630,		-2620,
-	-2609,		-2599,		-2588,		-2578,
-	-2567,		-2556,		-2546,		-2535,
-	-2525,		-2515,		-2504,		-2493,
-	-2483,		-2472,		-2462,		-2451,
-	-2441,		-2431,		-2420,		-2410,
-	-2399,		-2389,		-2378,		-2367,
-	-2357,		-2347,		-2336,		-2326,
-	-2315,		-2305,		-2295,		-2284,
-	-2274,		-2263,		-2253,		-2243,
-	-2232,		-2222,		-2211,		-2201,
-	-2191,		-2180,		-2170,		-2159,
-	-2149,		-2139,		-2128,		-2118,
-	-2107,		-2097,		-2087,		-2076,
-	-2066,		-2056,		-2046,		-2035,
-	-2025,		-2014,		-2004,		-1994,
-	-1983,		-1973,		-1963,		-1953,
-	-1942,		-1932,		-1921,		-1911,
-	-1901,		-1891,		-1880,		-1870,
-	-1860,		-1849,		-1839,		-1829,
-	-1819,		-1808,		-1798,		-1788,
-	-1778,		-1767,		-1757,		-1747,
-	-1736,		-1726,		-1716,		-1706,
-	-1695,		-1685,		-1675,		-1665,
-	-1654,		-1644,		-1634,		-1624,
-	-1613,		-1603,		-1593,		-1583,
-	-1573,		-1563,		-1552,		-1542,
-	-1532,		-1522,		-1511,		-1501,
-	-1491,		-1481,		-1471,		-1461,
-	-1450,		-1440,		-1430,		-1420,
-	-1409,		-1400,		-1389,		-1379,
-	-1369,		-1359,		-1348,		-1339,
-	-1328,		-1318,		-1308,		-1298,
-	-1288,		-1278,		-1267,		-1257,
-	-1247,		-1237,		-1227,		-1217,
-	-1207,		-1196,		-1186,		-1176,
-	-1166,		-1156,		-1146,		-1135,
-	-1126,		-1115,		-1105,		-1095,
-	-1085,		-1075,		-1065,		-1055,
-	-1044,		-1034,		-1024,		-1014,
-	-1004,		-994,		-984,		-974,
-	-964,		-954,		-944,		-933,
-	-923,		-913,		-903,		-893,
-	-883,		-873,		-863,		-853,
-	-843,		-833,		-822,		-812,
-	-802,		-792,		-782,		-772,
-	-762,		-752,		-742,		-732,
-	-722,		-712,		-702,		-691,
-	-682,		-671,		-662,		-651,
-	-641,		-631,		-621,		-611,
-	-601,		-591,		-581,		-571,
-	-561,		-551,		-541,		-531,
-	-521,		-511,		-501,		-491,
-	-480,		-471,		-460,		-451,
-	-440,		-430,		-420,		-410,
-	-400,		-390,		-380,		-370,
-	-360,		-350,		-340,		-330,
-	-320,		-310,		-300,		-290,
-	-280,		-270,		-260,		-250,
-	-240,		-230,		-220,		-210,
-	-199,		-190,		-179,		-170,
-	-159,		-150,		-139,		-129,
-	-119,		-109,		-99,		-89,
-	-79,		-69,		-59,		-49,
-	-39,		-29,		-19,		-9,
-	1,		11,		21,		31,
-	41,		51,		61,		71,
-	81,		91,		101,		111,
-	121,		131,		141,		152,
-	161,		172,		181,		192,
-	202,		212,		222,		232,
-	242,		252,		262,		272,
-	282,		292,		302,		312,
-	322,		332,		342,		352,
-	362,		372,		382,		392,
-	402,		412,		422,		433,
-	442,		453,		462,		473,
-	483,		493,		503,		513,
-	523,		533,		543,		553,
-	563,		573,		583,		593,
-	603,		613,		623,		633,
-	643,		653,		664,		673,
-	684,		694,		704,		714,
-	724,		734,		744,		754,
-	764,		774,		784,		794,
-	804,		815,		825,		835,
-	845,		855,		865,		875,
-	885,		895,		905,		915,
-	925,		936,		946,		956,
-	966,		976,		986,		996,
-	1006,		1016,		1026,		1037,
-	1047,		1057,		1067,		1077,
-	1087,		1097,		1107,		1117,
-	1128,		1138,		1148,		1158,
-	1168,		1178,		1188,		1198,
-	1209,		1219,		1229,		1239,
-	1249,		1259,		1269,		1280,
-	1290,		1300,		1310,		1320,
-	1330,		1341,		1351,		1361,
-	1371,		1381,		1391,		1402,
-	1412,		1422,		1432,		1442,
-	1452,		1463,		1473,		1483,
-	1493,		1503,		1513,		1524,
-	1534,		1544,		1554,		1565,
-	1575,		1585,		1595,		1606,
-	1616,		1626,		1636,		1647,
-	1656,		1667,		1677,		1687,
-	1697,		1708,		1718,		1729,
-	1739,		1749,		1759,		1769,
-	1780,		1790,		1800,		1810,
-	1821,		1831,		1841,		1851,
-	1862,		1872,		1883,		1893,
-	1903,		1913,		1923,		1934,
-	1944,		1955,		1965,		1975,
-	1985,		1996,		2006,		2016,
-	2027,		2037,		2048,		2058,
-	2068,		2079,		2089,		2099,
-	2110,		2120,		2130,		2141,
-	2151,		2161,		2172,		2182,
-	2193,		2203,		2213,		2224,
-	2234,		2245,		2255,		2265,
-	2276,		2286,		2297,		2307,
-	2318,		2328,		2338,		2349,
-	2359,		2370,		2380,		2391,
-	2401,		2412,		2422,		2433,
-	2443,		2454,		2464,		2475,
-	2485,		2496,		2506,		2517,
-	2527,		2537,		2548,		2559,
-	2569,		2580,		2590,		2601,
-	2612,		2622,		2632,		2643,
-	2654,		2664,		2675,		2685,
-	2696,		2707,		2717,		2728,
-	2738,		2749,		2759,		2770,
-	2781,		2791,		2802,		2813,
-	2823,		2834,		2845,		2855,
-	2866,		2877,		2887,		2898,
-	2909,		2919,		2930,		2941,
-	2951,		2962,		2973,		2984,
-	2994,		3005,		3015,		3027,
-	3037,		3048,		3058,		3069,
-	3080,		3091,		3101,		3113,
-	3123,		3134,		3145,		3156,
-	3166,		3177,		3188,		3199,
-	3210,		3220,		3231,		3242,
-	3253,		3264,		3275,		3285,
-	3296,		3307,		3318,		3329,
-	3340,		3351,		3362,		3373,
-	3384,		3394,		3405,		3416,
-	3427,		3438,		3449,		3460,
-	3471,		3482,		3493,		3504,
-	3515,		3526,		3537,		3548,
-	3559,		3570,		3581,		3592,
-	3603,		3614,		3625,		3636,
-	3647,		3659,		3670,		3681,
-	3692,		3703,		3714,		3725,
-	3736,		3747,		3758,		3770,
-	3781,		3792,		3803,		3814,
-	3825,		3837,		3848,		3859,
-	3870,		3881,		3893,		3904,
-	3915,		3926,		3937,		3949,
-	3960,		3971,		3983,		3994,
-	4005,		4017,		4028,		4039,
-	4051,		4062,		4073,		4085,
-	4096,		4107,		4119,		4130,
-	4141,		4153,		4164,		4175,
-	4187,		4198,		4210,		4221,
-	4233,		4244,		4256,		4267,
-	4279,		4290,		4302,		4313,
-	4325,		4336,		4348,		4359,
-	4371,		4382,		4394,		4406,
-	4417,		4429,		4440,		4452,
-	4464,		4475,		4487,		4499,
-	4510,		4522,		4533,		4545,
-	4557,		4569,		4581,		4592,
-	4604,		4616,		4627,		4639,
-	4651,		4663,		4674,		4686,
-	4698,		4710,		4722,		4734,
-	4746,		4758,		4769,		4781,
-	4793,		4805,		4817,		4829,
-	4841,		4853,		4865,		4877,
-	4889,		4900,		4913,		4925,
-	4936,		4949,		4961,		4973,
-	4985,		4997,		5009,		5021,
-	5033,		5045,		5057,		5070,
-	5081,		5094,		5106,		5118,
-	5130,		5143,		5155,		5167,
-	5179,		5191,		5204,		5216,
-	5228,		5240,		5253,		5265,
-	5278,		5290,		5302,		5315,
-	5327,		5340,		5352,		5364,
-	5377,		5389,		5401,		5414,
-	5426,		5439,		5451,		5464,
-	5476,		5489,		5502,		5514,
-	5527,		5539,		5552,		5564,
-	5577,		5590,		5603,		5615,
-	5628,		5641,		5653,		5666,
-	5679,		5691,		5704,		5717,
-	5730,		5743,		5756,		5768,
-	5781,		5794,		5807,		5820,
-	5833,		5846,		5859,		5872,
-	5885,		5897,		5911,		5924,
-	5937,		5950,		5963,		5976,
-	5989,		6002,		6015,		6028,
-	6042,		6055,		6068,		6081,
-	6094,		6108,		6121,		6134,
-	6147,		6160,		6174,		6187,
-	6201,		6214,		6227,		6241,
-	6254,		6267,		6281,		6294,
-	6308,		6321,		6335,		6348,
-	6362,		6375,		6389,		6403,
-	6416,		6430,		6443,		6457,
-	6471,		6485,		6498,		6512,
-	6526,		6540,		6554,		6567,
-	6581,		6595,		6609,		6623,
-	6637,		6651,		6665,		6679,
-	6692,		6706,		6721,		6735,
-	6749,		6763,		6777,		6791,
-	6805,		6819,		6833,		6848,
-	6862,		6876,		6890,		6905,
-	6919,		6933,		6948,		6962,
-	6976,		6991,		7005,		7020,
-	7034,		7049,		7064,		7078,
-	7093,		7107,		7122,		7136,
-	7151,		7166,		7180,		7195,
-	7210,		7225,		7240,		7254,
-	7269,		7284,		7299,		7314,
-	7329,		7344,		7359,		7374,
-	7389,		7404,		7419,		7434,
-	7449,		7465,		7480,		7495,
-	7510,		7526,		7541,		7556,
-	7571,		7587,		7602,		7618,
-	7633,		7648,		7664,		7680,
-	7695,		7711,		7726,		7742,
-	7758,		7773,		7789,		7805,
-	7821,		7836,		7852,		7868,
-	7884,		7900,		7916,		7932,
-	7948,		7964,		7981,		7997,
-	8013,		8029,		8045,		8061,
-	8078,		8094,		8110,		8127,
-	8143,		8160,		8176,		8193,
-	8209,		8226,		8242,		8259,
-	8276,		8292,		8309,		8326,
-	8343,		8360,		8377,		8394,
-	8410,		8428,		8444,		8462,
-	8479,		8496,		8513,		8530,
-	8548,		8565,		8582,		8600,
-	8617,		8634,		8652,		8670,
-	8687,		8704,		8722,		8740,
-	8758,		8775,		8793,		8811,
-	8829,		8847,		8865,		8883,
-	8901,		8919,		8937,		8955,
-	8974,		8992,		9010,		9029,
-	9047,		9066,		9084,		9103,
-	9121,		9140,		9159,		9177,
-	9196,		9215,		9234,		9253,
-	9272,		9291,		9310,		9329,
-	9349,		9368,		9387,		9406,
-	9426,		9445,		9465,		9484,
-	9504,		9524,		9544,		9563,
-	9583,		9603,		9623,		9643,
-	9663,		9683,		9703,		9723,
-	9744,		9764,		9785,		9805,
-	9826,		9846,		9867,		9888,
-	9909,		9930,		9950,		9971,
-	9993,		10013,		10035,		10056,
-	10077,		10099,		10120,		10142,
-	10163,		10185,		10207,		10229,
-	10251,		10273,		10294,		10317,
-	10339,		10361,		10384,		10406,
-	10428,		10451,		10474,		10496,
-	10519,		10542,		10565,		10588,
-	10612,		10635,		10658,		10682,
-	10705,		10729,		10752,		10776,
-	10800,		10824,		10848,		10872,
-	10896,		10921,		10945,		10969,
-	10994,		11019,		11044,		11069,
-	11094,		11119,		11144,		11169,
-	11195,		11221,		11246,		11272,
-	11298,		11324,		11350,		11376,
-	11402,		11429,		11456,		11482,
-	11509,		11536,		11563,		11590,
-	11618,		11645,		11673,		11701,
-	11728,		11756,		11785,		11813,
-	11842,		11870,		11899,		11928,
-	11957,		11986,		12015,		12045,
-	12074,		12104,		12134,		12164,
-	12194,		12225,		12255,		12286,
-	12317,		12348,		12380,		12411,
-	12443,		12475,		12507,		12539,
-	12571,		12604,		12637,		12670,
-	12703,		12737,		12771,		12804,
-	12839,		12873,		12907,		12942,
-	12977,		13013,		13048,		13084,
-	13120,		13156,		13192,		13229,
-	13267,		13304,		13341,		13379,
-	13418,		13456,		13495,		13534,
-	13573,		13613,		13653,		13693,
-	13734,		13775,		13817,		13858,
-	13901,		13943,		13986,		14029,
-	14073,		14117,		14162,		14206,
-	14252,		14297,		14343,		14390,
-	14437,		14485,		14533,		14582,
-	14631,		14680,		14731,		14782,
-	14833,		14885,		14937,		14991,
-	15044,		15099,		15154,		15210,
-	15266,		15324,		15382,		15441,
-	15500,		15561,		15622,		15684,
-	15747,		15811,		15877,		15943,
-	16010,		16078,		16148,		16218,
-	16290,		16363,		16437,		16513,
-	16590,		16669,		16749,		16831,
-	16915,		17000,		17088,		17177,
-	17268,		17362,		17458,		17556,
-	17657,		17761,		17868,		17977,
-	18090,		18207,		18328,		18452,
-	18581,		18715,		18854,		18998,
-	19149,		19307,		19472,		19645,
-	19828,		20021,		20226,		20444,
-	20678,		20930,		21204,		21503,
-	21835,		22206,		22630,		23124,
-	23721,		24478,		25529,		27316,
-};
+/* get_crandom - correlated random number generator
+ * Next number depends on last value.
+ * rho is scaled to avoid floating point.
+ */
+static unsigned long get_crandom(struct crndstate *state)
+{
+	u64 value, rho;
+	unsigned long answer;
+
+	if (state->rho == 0)	/* no correllation */
+		return net_random();
+
+	value = net_random();
+	rho = (u64)state->rho + 1;
+	answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32;
+	state->last = answer;
+	return answer;
+}
 
 /* tabledist - return a pseudo-randomly distributed value with mean mu and
  * std deviation sigma.  Uses table lookup to approximate the desired
  * distribution, and a uniformly-distributed pseudo-random source.
  */
-static inline int tabledist(int mu, int sigma)
+static long tabledist(unsigned long mu, long sigma, 
+		      struct crndstate *state, const struct disttable *dist)
 {
-	int x;
-	int index;
-	int sigmamod, sigmadiv;
+	long t, x;
+	unsigned long rnd;
 
 	if (sigma == 0)
 		return mu;
 
-	index = (net_random() & (TABLESIZE-1));
-	sigmamod = sigma%TABLEFACTOR;
-	sigmadiv = sigma/TABLEFACTOR;
-	x = sigmamod*disttable[index];
+	rnd = get_crandom(state);
+
+	/* default uniform distribution */
+	if (dist == NULL) 
+		return (rnd % (2*sigma)) - sigma + mu;
 
+	t = dist->table[rnd % dist->size];
+	x = (sigma % NETEM_DIST_SCALE) * t;
 	if (x >= 0)
-		x += TABLEFACTOR/2;
+		x += NETEM_DIST_SCALE/2;
 	else
-		x -= TABLEFACTOR/2;
+		x -= NETEM_DIST_SCALE/2;
 
-	x /= TABLEFACTOR;
-	x += sigmadiv*disttable[index];
-	x += mu;
-	return x;
+	return  x / NETEM_DIST_SCALE + (sigma / NETEM_DIST_SCALE) * t + mu;
 }
 
-/* Enqueue packets with underlying discipline (fifo)
- * but mark them with current time first.
- */
-static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+/* Put skb in the private delayed queue. */
+static int delay_skb(struct Qdisc *sch, struct sk_buff *skb)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 	struct netem_skb_cb *cb = (struct netem_skb_cb *)skb->cb;
 	psched_time_t now;
-	long delay;
+	
+	PSCHED_GET_TIME(now);
+	PSCHED_TADD2(now, tabledist(q->latency, q->jitter, 
+				    &q->delay_cor, q->delay_dist),
+		     cb->time_to_send);
+	
+	/* Always queue at tail to keep packets in order */
+	if (likely(q->delayed.qlen < q->limit)) {
+		__skb_queue_tail(&q->delayed, skb);
+		sch->q.qlen++;
+		sch->stats.bytes += skb->len;
+		sch->stats.packets++;
+		return NET_XMIT_SUCCESS;
+	}
+
+	sch->stats.drops++;
+	kfree_skb(skb);
+	return NET_XMIT_DROP;
+}
+
+static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+{
+	struct netem_sched_data *q = qdisc_priv(sch);
 
 	pr_debug("netem_enqueue skb=%p @%lu\n", skb, jiffies);
 
 	/* Random packet drop 0 => none, ~0 => all */
-	if (q->loss && q->loss >= net_random()) {
+	if (q->loss && q->loss >= get_crandom(&q->loss_cor)) {
+		pr_debug("netem_enqueue: random loss\n");
 		sch->stats.drops++;
 		return 0;	/* lie about loss so TCP doesn't know */
 	}
 
+	/* Random duplication */
+	if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor)) {
+		struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
+
+		pr_debug("netem_enqueue: dup %p\n", skb2);
+		if (skb2)
+			delay_skb(sch, skb2);
+	}
 
 	/* If doing simple delay then gap == 0 so all packets
 	 * go into the delayed holding queue
@@ -634,33 +203,14 @@
 	}
 	
 	q->counter = 0;
-	
-	PSCHED_GET_TIME(now);
-	if (q->jitter) 
-		delay = tabledist(q->latency, q->jitter);
-	else
-		delay = q->latency;
 
-	PSCHED_TADD2(now, delay, cb->time_to_send);
-	
-	/* Always queue at tail to keep packets in order */
-	if (likely(q->delayed.qlen < q->limit)) {
-		__skb_queue_tail(&q->delayed, skb);
-		sch->q.qlen++;
-		sch->stats.bytes += skb->len;
-		sch->stats.packets++;
-		return 0;
-	}
-
-	sch->stats.drops++;
-	kfree_skb(skb);
-	return NET_XMIT_DROP;
+	return delay_skb(sch, skb);
 }
 
 /* Requeue packets but don't change time stamp */
 static int netem_requeue(struct sk_buff *skb, struct Qdisc *sch)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 	int ret;
 
 	if ((ret = q->qdisc->ops->requeue(skb, q->qdisc)) == 0)
@@ -671,7 +221,7 @@
 
 static unsigned int netem_drop(struct Qdisc* sch)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 	unsigned int len;
 
 	if ((len = q->qdisc->ops->drop(q->qdisc)) != 0) {
@@ -687,7 +237,7 @@
  */
 static struct sk_buff *netem_dequeue(struct Qdisc *sch)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 	struct sk_buff *skb;
 	psched_time_t now;
 
@@ -727,7 +277,7 @@
 
 static void netem_reset(struct Qdisc *sch)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 
 	qdisc_reset(q->qdisc);
 	skb_queue_purge(&q->delayed);
@@ -753,92 +303,163 @@
 	return ret;
 }
 
-static int netem_change(struct Qdisc *sch, struct rtattr *opt)
+/*
+ * Distribution data is a variable size payload containing
+ * signed 16 bit values.
+ */
+static int get_dist_table(struct Qdisc *sch, const struct rtattr *attr)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
-	struct tc_netem_qopt *qopt = RTA_DATA(opt);
-	struct Qdisc *child;
-	int ret;
+	struct netem_sched_data *q = qdisc_priv(sch);
+	unsigned long n = RTA_PAYLOAD(attr)/sizeof(__s16);
+	const __s16 *data = RTA_DATA(attr);
+	struct disttable *d;
+	int i;
+
+	if (n > 65536)
+		return -EINVAL;
 
-	if (opt->rta_len < RTA_LENGTH(sizeof(*qopt)))
+	d = kmalloc(sizeof(*d) + n*sizeof(d->table[0]), GFP_KERNEL);
+	if (!d)
+		return -ENOMEM;
+
+	d->size = n;
+	for (i = 0; i < n; i++)
+		d->table[i] = data[i];
+	
+	spin_lock_bh(&sch->dev->queue_lock);
+	d = xchg(&q->delay_dist, d);
+	spin_unlock_bh(&sch->dev->queue_lock);
+
+	kfree(d);
+	return 0;
+}
+
+static int get_correlation(struct Qdisc *sch, const struct rtattr *attr)
+{
+	struct netem_sched_data *q = qdisc_priv(sch);
+	const struct tc_netem_corr *c = RTA_DATA(attr);
+
+	if (RTA_PAYLOAD(attr) != sizeof(*c))
 		return -EINVAL;
 
-	child = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops);
-	if (!child)
+	init_crandom(&q->delay_cor, c->delay_corr);
+	init_crandom(&q->loss_cor, c->loss_corr);
+	init_crandom(&q->dup_cor, c->dup_corr);
+	return 0;
+}
+
+static int netem_change(struct Qdisc *sch, struct rtattr *opt)
+{
+	struct netem_sched_data *q = qdisc_priv(sch);
+	struct tc_netem_qopt *qopt;
+	int ret;
+	
+	if (opt == NULL || RTA_PAYLOAD(opt) < sizeof(*qopt))
 		return -EINVAL;
 
-	ret = set_fifo_limit(child, qopt->limit);
+	qopt = RTA_DATA(opt);
+	ret = set_fifo_limit(q->qdisc, qopt->limit);
 	if (ret) {
-		qdisc_destroy(child);
+		pr_debug("netem: can't set fifo limit\n");
 		return ret;
 	}
-
-	sch_tree_lock(sch);
-	if (child) {
-		child = xchg(&q->qdisc, child);
-		if (child != &noop_qdisc)
-			qdisc_destroy(child);
 	
-		q->latency = qopt->latency;
-		q->jitter = qopt->jitter;
-		q->limit = qopt->limit;
-		q->gap = qopt->gap;
-		q->loss = qopt->loss;
+	q->latency = qopt->latency;
+	q->jitter = qopt->jitter;
+	q->limit = qopt->limit;
+	q->gap = qopt->gap;
+	q->loss = qopt->loss;
+	q->duplicate = qopt->duplicate;
+
+	/* Handle nested options after initial queue options.
+	 * Should have put all options in nested format but too late now.
+	 */ 
+	if (RTA_PAYLOAD(opt) > sizeof(*qopt)) {
+		struct rtattr *tb[TCA_NETEM_MAX];
+		if (rtattr_parse(tb, TCA_NETEM_MAX, 
+				 RTA_DATA(opt) + sizeof(*qopt),
+				 RTA_PAYLOAD(opt) - sizeof(*qopt)))
+			return -EINVAL;
+
+		if (tb[TCA_NETEM_CORR-1]) {
+			ret = get_correlation(sch, tb[TCA_NETEM_CORR-1]);
+			if (ret)
+				return ret;
+		}
+
+		if (tb[TCA_NETEM_DELAY_DIST-1]) {
+			ret = get_dist_table(sch, tb[TCA_NETEM_DELAY_DIST-1]);
+			if (ret)
+				return ret;
+		}
 	}
-	sch_tree_unlock(sch);
+
 
 	return 0;
 }
 
 static int netem_init(struct Qdisc *sch, struct rtattr *opt)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
-	int err;
+	struct netem_sched_data *q = qdisc_priv(sch);
+	int ret;
 
 	if (!opt)
 		return -EINVAL;
 
 	MOD_INC_USE_COUNT;
 	skb_queue_head_init(&q->delayed);
-	q->qdisc = &noop_qdisc;
-
 	init_timer(&q->timer);
 	q->timer.function = netem_watchdog;
 	q->timer.data = (unsigned long) sch;
 	q->counter = 0;
 
-	err = netem_change(sch, opt);
-	if (err != 0)
+	q->qdisc = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops);
+	if (!q->qdisc) {
+		pr_debug("netem: qdisc create failed\n");
+		return -ENOMEM;
+	}
+
+	ret = netem_change(sch, opt);
+	if (ret) {
+		pr_debug("netem: change failed\n");
+		qdisc_destroy(q->qdisc);
 		MOD_DEC_USE_COUNT;
-	return err;
+	}
+	return ret;
 }
 
 static void netem_destroy(struct Qdisc *sch)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
-
-	del_timer(&q->timer);
+	struct netem_sched_data *q = qdisc_priv(sch);
 
+	del_timer_sync(&q->timer);
 	qdisc_destroy(q->qdisc);
-	q->qdisc = &noop_qdisc;
-
+	kfree(q->delay_dist);
 	MOD_DEC_USE_COUNT;
 }
 
 static int netem_dump(struct Qdisc *sch, struct sk_buff *skb)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	const struct netem_sched_data *q = qdisc_priv(sch);
 	unsigned char	 *b = skb->tail;
+	struct rtattr *rta = (struct rtattr *) b;
 	struct tc_netem_qopt qopt;
+	struct tc_netem_corr cor;
 
 	qopt.latency = q->latency;
 	qopt.jitter = q->jitter;
 	qopt.limit = q->limit;
 	qopt.loss = q->loss;
 	qopt.gap = q->gap;
-
+	qopt.duplicate = q->duplicate;
 	RTA_PUT(skb, TCA_OPTIONS, sizeof(qopt), &qopt);
 
+	cor.delay_corr = q->delay_cor.rho;
+	cor.loss_corr = q->loss_cor.rho;
+	cor.dup_corr = q->dup_cor.rho;
+	RTA_PUT(skb, TCA_NETEM_CORR, sizeof(cor), &cor);
+	rta->rta_len = skb->tail - b;
+
 	return skb->len;
 
 rtattr_failure:
@@ -849,7 +470,7 @@
 static int netem_dump_class(struct Qdisc *sch, unsigned long cl,
 			  struct sk_buff *skb, struct tcmsg *tcm)
 {
-	struct netem_sched_data *q = (struct netem_sched_data*)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 
 	if (cl != 1) 	/* only one class */
 		return -ENOENT;
@@ -863,7 +484,7 @@
 static int netem_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
 		     struct Qdisc **old)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 
 	if (new == NULL)
 		new = &noop_qdisc;
@@ -879,7 +500,7 @@
 
 static struct Qdisc *netem_leaf(struct Qdisc *sch, unsigned long arg)
 {
-	struct netem_sched_data *q = (struct netem_sched_data *)sch->data;
+	struct netem_sched_data *q = qdisc_priv(sch);
 	return q->qdisc;
 }
 
