summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/ia64/Kconfig4
-rw-r--r--arch/ia64/lib/Makefile2
-rw-r--r--arch/x86_64/kernel/Makefile2
-rw-r--r--drivers/block/aoe/aoecmd.c13
-rw-r--r--drivers/block/genhd.c29
-rw-r--r--drivers/block/ll_rw_blk.c47
-rw-r--r--drivers/md/linear.c10
-rw-r--r--drivers/md/md.c4
-rw-r--r--drivers/md/multipath.c10
-rw-r--r--drivers/md/raid0.c10
-rw-r--r--drivers/md/raid1.c12
-rw-r--r--drivers/md/raid10.c12
-rw-r--r--drivers/md/raid5.c10
-rw-r--r--drivers/md/raid6main.c12
-rw-r--r--fs/partitions/check.c7
-rw-r--r--include/asm-x86_64/dma-mapping.h31
-rw-r--r--include/asm-x86_64/swiotlb.h8
-rw-r--r--include/linux/genhd.h10
-rw-r--r--lib/Makefile2
-rw-r--r--lib/swiotlb.c (renamed from arch/ia64/lib/swiotlb.c)142
20 files changed, 209 insertions, 168 deletions
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 1642375fb14e2..3b4248cff9a7d 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -26,6 +26,10 @@ config MMU
bool
default y
+config SWIOTLB
+ bool
+ default y
+
config RWSEM_XCHGADD_ALGORITHM
bool
default y
diff --git a/arch/ia64/lib/Makefile b/arch/ia64/lib/Makefile
index cb1af597370bc..ac64664a1807d 100644
--- a/arch/ia64/lib/Makefile
+++ b/arch/ia64/lib/Makefile
@@ -9,7 +9,7 @@ lib-y := __divsi3.o __udivsi3.o __modsi3.o __umodsi3.o \
bitop.o checksum.o clear_page.o csum_partial_copy.o \
clear_user.o strncpy_from_user.o strlen_user.o strnlen_user.o \
flush.o ip_fast_csum.o do_csum.o \
- memset.o strlen.o swiotlb.o
+ memset.o strlen.o
lib-$(CONFIG_ITANIUM) += copy_page.o copy_user.o memcpy.o
lib-$(CONFIG_MCKINLEY) += copy_page_mck.o memcpy_mck.o
diff --git a/arch/x86_64/kernel/Makefile b/arch/x86_64/kernel/Makefile
index bcdd0a805fe70..14328cab5d3ac 100644
--- a/arch/x86_64/kernel/Makefile
+++ b/arch/x86_64/kernel/Makefile
@@ -27,7 +27,6 @@ obj-$(CONFIG_CPU_FREQ) += cpufreq/
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
obj-$(CONFIG_GART_IOMMU) += pci-gart.o aperture.o
obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o pci-dma.o
-obj-$(CONFIG_SWIOTLB) += swiotlb.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_X86_PM_TIMER) += pmtimer.o
@@ -41,7 +40,6 @@ CFLAGS_vsyscall.o := $(PROFILING) -g0
bootflag-y += ../../i386/kernel/bootflag.o
cpuid-$(subst m,y,$(CONFIG_X86_CPUID)) += ../../i386/kernel/cpuid.o
topology-y += ../../i386/mach-default/topology.o
-swiotlb-$(CONFIG_SWIOTLB) += ../../ia64/lib/swiotlb.o
microcode-$(subst m,y,$(CONFIG_MICROCODE)) += ../../i386/kernel/microcode.o
intel_cacheinfo-y += ../../i386/kernel/cpu/intel_cacheinfo.o
quirks-y += ../../i386/kernel/quirks.o
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 5c9c7c1a3d4cf..326ca3876b685 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -468,16 +468,11 @@ aoecmd_ata_rsp(struct sk_buff *skb)
unsigned long duration = jiffies - buf->start_time;
unsigned long n_sect = buf->bio->bi_size >> 9;
struct gendisk *disk = d->gd;
+ const int rw = bio_data_dir(buf->bio);
- if (bio_data_dir(buf->bio) == WRITE) {
- disk_stat_inc(disk, writes);
- disk_stat_add(disk, write_ticks, duration);
- disk_stat_add(disk, write_sectors, n_sect);
- } else {
- disk_stat_inc(disk, reads);
- disk_stat_add(disk, read_ticks, duration);
- disk_stat_add(disk, read_sectors, n_sect);
- }
+ disk_stat_inc(disk, ios[rw]);
+ disk_stat_add(disk, ticks[rw], duration);
+ disk_stat_add(disk, sectors[rw], n_sect);
disk_stat_add(disk, io_ticks, duration);
n = (buf->flags & BUFFL_FAIL) ? -EIO : 0;
bio_endio(buf->bio, buf->bio->bi_size, n);
diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 486ce1fdeb8c8..54aec4a1ae13a 100644
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -391,13 +391,12 @@ static ssize_t disk_stats_read(struct gendisk * disk, char *page)
"%8u %8u %8llu %8u "
"%8u %8u %8u"
"\n",
- disk_stat_read(disk, reads), disk_stat_read(disk, read_merges),
- (unsigned long long)disk_stat_read(disk, read_sectors),
- jiffies_to_msecs(disk_stat_read(disk, read_ticks)),
- disk_stat_read(disk, writes),
- disk_stat_read(disk, write_merges),
- (unsigned long long)disk_stat_read(disk, write_sectors),
- jiffies_to_msecs(disk_stat_read(disk, write_ticks)),
+ disk_stat_read(disk, ios[0]), disk_stat_read(disk, merges[0]),
+ (unsigned long long)disk_stat_read(disk, sectors[0]),
+ jiffies_to_msecs(disk_stat_read(disk, ticks[0])),
+ disk_stat_read(disk, ios[1]), disk_stat_read(disk, merges[1]),
+ (unsigned long long)disk_stat_read(disk, sectors[1]),
+ jiffies_to_msecs(disk_stat_read(disk, ticks[1])),
disk->in_flight,
jiffies_to_msecs(disk_stat_read(disk, io_ticks)),
jiffies_to_msecs(disk_stat_read(disk, time_in_queue)));
@@ -583,12 +582,12 @@ static int diskstats_show(struct seq_file *s, void *v)
preempt_enable();
seq_printf(s, "%4d %4d %s %u %u %llu %u %u %u %llu %u %u %u %u\n",
gp->major, n + gp->first_minor, disk_name(gp, n, buf),
- disk_stat_read(gp, reads), disk_stat_read(gp, read_merges),
- (unsigned long long)disk_stat_read(gp, read_sectors),
- jiffies_to_msecs(disk_stat_read(gp, read_ticks)),
- disk_stat_read(gp, writes), disk_stat_read(gp, write_merges),
- (unsigned long long)disk_stat_read(gp, write_sectors),
- jiffies_to_msecs(disk_stat_read(gp, write_ticks)),
+ disk_stat_read(gp, ios[0]), disk_stat_read(gp, merges[0]),
+ (unsigned long long)disk_stat_read(gp, sectors[0]),
+ jiffies_to_msecs(disk_stat_read(gp, ticks[0])),
+ disk_stat_read(gp, ios[1]), disk_stat_read(gp, merges[1]),
+ (unsigned long long)disk_stat_read(gp, sectors[1]),
+ jiffies_to_msecs(disk_stat_read(gp, ticks[1])),
gp->in_flight,
jiffies_to_msecs(disk_stat_read(gp, io_ticks)),
jiffies_to_msecs(disk_stat_read(gp, time_in_queue)));
@@ -601,8 +600,8 @@ static int diskstats_show(struct seq_file *s, void *v)
seq_printf(s, "%4d %4d %s %u %u %u %u\n",
gp->major, n + gp->first_minor + 1,
disk_name(gp, n + 1, buf),
- hd->reads, hd->read_sectors,
- hd->writes, hd->write_sectors);
+ hd->ios[0], hd->sectors[0],
+ hd->ios[1], hd->sectors[1]);
}
return 0;
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index 0af73512b9a8c..2747741677fb0 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -2387,16 +2387,9 @@ static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io)
if (!blk_fs_request(rq) || !rq->rq_disk)
return;
- if (rw == READ) {
- __disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
- if (!new_io)
- __disk_stat_inc(rq->rq_disk, read_merges);
- } else if (rw == WRITE) {
- __disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
- if (!new_io)
- __disk_stat_inc(rq->rq_disk, write_merges);
- }
- if (new_io) {
+ if (!new_io) {
+ __disk_stat_inc(rq->rq_disk, merges[rw]);
+ } else {
disk_round_stats(rq->rq_disk);
rq->rq_disk->in_flight++;
}
@@ -2791,17 +2784,11 @@ static inline void blk_partition_remap(struct bio *bio)
if (bdev != bdev->bd_contains) {
struct hd_struct *p = bdev->bd_part;
+ const int rw = bio_data_dir(bio);
+
+ p->sectors[rw] += bio_sectors(bio);
+ p->ios[rw]++;
- switch (bio_data_dir(bio)) {
- case READ:
- p->read_sectors += bio_sectors(bio);
- p->reads++;
- break;
- case WRITE:
- p->write_sectors += bio_sectors(bio);
- p->writes++;
- break;
- }
bio->bi_sector += p->start_sect;
bio->bi_bdev = bdev->bd_contains;
}
@@ -3048,6 +3035,12 @@ static int __end_that_request_first(struct request *req, int uptodate,
(unsigned long long)req->sector);
}
+ if (blk_fs_request(req) && req->rq_disk) {
+ const int rw = rq_data_dir(req);
+
+ __disk_stat_add(req->rq_disk, sectors[rw], nr_bytes >> 9);
+ }
+
total_bytes = bio_nbytes = 0;
while ((bio = req->bio) != NULL) {
int nbytes;
@@ -3176,16 +3169,10 @@ void end_that_request_last(struct request *req)
if (disk && blk_fs_request(req)) {
unsigned long duration = jiffies - req->start_time;
- switch (rq_data_dir(req)) {
- case WRITE:
- __disk_stat_inc(disk, writes);
- __disk_stat_add(disk, write_ticks, duration);
- break;
- case READ:
- __disk_stat_inc(disk, reads);
- __disk_stat_add(disk, read_ticks, duration);
- break;
- }
+ const int rw = rq_data_dir(req);
+
+ __disk_stat_inc(disk, ios[rw]);
+ __disk_stat_add(disk, ticks[rw], duration);
disk_round_stats(disk);
disk->in_flight--;
}
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index bb279fad2fd27..946efef3a8f58 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -271,6 +271,7 @@ static int linear_stop (mddev_t *mddev)
static int linear_make_request (request_queue_t *q, struct bio *bio)
{
+ const int rw = bio_data_dir(bio);
mddev_t *mddev = q->queuedata;
dev_info_t *tmp_dev;
sector_t block;
@@ -280,13 +281,8 @@ static int linear_make_request (request_queue_t *q, struct bio *bio)
return 0;
}
- if (bio_data_dir(bio)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
tmp_dev = which_dev(mddev, bio->bi_sector);
block = bio->bi_sector >> 1;
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 2a8a5696bf8ae..9ecf51ee596fb 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -3466,8 +3466,8 @@ static int is_mddev_idle(mddev_t *mddev)
idle = 1;
ITERATE_RDEV(mddev,rdev,tmp) {
struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
- curr_events = disk_stat_read(disk, read_sectors) +
- disk_stat_read(disk, write_sectors) -
+ curr_events = disk_stat_read(disk, sectors[0]) +
+ disk_stat_read(disk, sectors[1]) -
atomic_read(&disk->sync_io);
/* Allow some slack between valud of curr_events and last_events,
* as there are some uninteresting races.
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index 1151c3ed30067..c06f4474192b7 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -168,6 +168,7 @@ static int multipath_make_request (request_queue_t *q, struct bio * bio)
multipath_conf_t *conf = mddev_to_conf(mddev);
struct multipath_bh * mp_bh;
struct multipath_info *multipath;
+ const int rw = bio_data_dir(bio);
if (unlikely(bio_barrier(bio))) {
bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
@@ -179,13 +180,8 @@ static int multipath_make_request (request_queue_t *q, struct bio * bio)
mp_bh->master_bio = bio;
mp_bh->mddev = mddev;
- if (bio_data_dir(bio)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
mp_bh->path = multipath_map(conf);
if (mp_bh->path < 0) {
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index f6757259ce7fe..fece3277c2a5e 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -403,19 +403,15 @@ static int raid0_make_request (request_queue_t *q, struct bio *bio)
mdk_rdev_t *tmp_dev;
unsigned long chunk;
sector_t block, rsect;
+ const int rw = bio_data_dir(bio);
if (unlikely(bio_barrier(bio))) {
bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
return 0;
}
- if (bio_data_dir(bio)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
chunk_size = mddev->chunk_size >> 10;
chunk_sects = mddev->chunk_size >> 9;
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 0e1f148dd41de..e16f473bcf46b 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -647,6 +647,7 @@ static int make_request(request_queue_t *q, struct bio * bio)
unsigned long flags;
struct bio_list bl;
struct page **behind_pages = NULL;
+ const int rw = bio_data_dir(bio);
if (unlikely(bio_barrier(bio))) {
bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
@@ -665,13 +666,8 @@ static int make_request(request_queue_t *q, struct bio * bio)
conf->nr_pending++;
spin_unlock_irq(&conf->resync_lock);
- if (bio_data_dir(bio)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
/*
* make_request() can abort the operation when READA is being
@@ -686,7 +682,7 @@ static int make_request(request_queue_t *q, struct bio * bio)
r1_bio->mddev = mddev;
r1_bio->sector = bio->bi_sector;
- if (bio_data_dir(bio) == READ) {
+ if (rw == READ) {
/*
* read balancing logic:
*/
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 28dd028415e49..bbe40e9cf9236 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -668,6 +668,7 @@ static int make_request(request_queue_t *q, struct bio * bio)
struct bio *read_bio;
int i;
int chunk_sects = conf->chunk_mask + 1;
+ const int rw = bio_data_dir(bio);
if (unlikely(bio_barrier(bio))) {
bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
@@ -718,13 +719,8 @@ static int make_request(request_queue_t *q, struct bio * bio)
conf->nr_pending++;
spin_unlock_irq(&conf->resync_lock);
- if (bio_data_dir(bio)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
@@ -734,7 +730,7 @@ static int make_request(request_queue_t *q, struct bio * bio)
r10_bio->mddev = mddev;
r10_bio->sector = bio->bi_sector;
- if (bio_data_dir(bio) == READ) {
+ if (rw == READ) {
/*
* read balancing logic:
*/
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 4683ca24c0469..6497295ebfb93 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -1462,6 +1462,7 @@ static int make_request (request_queue_t *q, struct bio * bi)
sector_t new_sector;
sector_t logical_sector, last_sector;
struct stripe_head *sh;
+ const int rw = bio_data_dir(bi);
if (unlikely(bio_barrier(bi))) {
bio_endio(bi, bi->bi_size, -EOPNOTSUPP);
@@ -1470,13 +1471,8 @@ static int make_request (request_queue_t *q, struct bio * bi)
md_write_start(mddev, bi);
- if (bio_data_dir(bi)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bi));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bi));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bi));
logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
last_sector = bi->bi_sector + (bi->bi_size>>9);
diff --git a/drivers/md/raid6main.c b/drivers/md/raid6main.c
index 267eb1430c834..6437a95ffc1cd 100644
--- a/drivers/md/raid6main.c
+++ b/drivers/md/raid6main.c
@@ -1621,6 +1621,7 @@ static int make_request (request_queue_t *q, struct bio * bi)
sector_t new_sector;
sector_t logical_sector, last_sector;
struct stripe_head *sh;
+ const int rw = bio_data_dir(bi);
if (unlikely(bio_barrier(bi))) {
bio_endio(bi, bi->bi_size, -EOPNOTSUPP);
@@ -1629,13 +1630,8 @@ static int make_request (request_queue_t *q, struct bio * bi)
md_write_start(mddev, bi);
- if (bio_data_dir(bi)==WRITE) {
- disk_stat_inc(mddev->gendisk, writes);
- disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bi));
- } else {
- disk_stat_inc(mddev->gendisk, reads);
- disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bi));
- }
+ disk_stat_inc(mddev->gendisk, ios[rw]);
+ disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bi));
logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
last_sector = bi->bi_sector + (bi->bi_size>>9);
@@ -1682,7 +1678,7 @@ static int make_request (request_queue_t *q, struct bio * bi)
if (--bi->bi_phys_segments == 0) {
int bytes = bi->bi_size;
- if ( bio_data_dir(bi) == WRITE )
+ if (rw == WRITE )
md_write_end(mddev);
bi->bi_size = 0;
bi->bi_end_io(bi, bytes, 0);
diff --git a/fs/partitions/check.c b/fs/partitions/check.c
index 9c06c5434ec47..8dc1822a70226 100644
--- a/fs/partitions/check.c
+++ b/fs/partitions/check.c
@@ -246,8 +246,8 @@ static ssize_t part_size_read(struct hd_struct * p, char *page)
static ssize_t part_stat_read(struct hd_struct * p, char *page)
{
return sprintf(page, "%8u %8llu %8u %8llu\n",
- p->reads, (unsigned long long)p->read_sectors,
- p->writes, (unsigned long long)p->write_sectors);
+ p->ios[0], (unsigned long long)p->sectors[0],
+ p->ios[1], (unsigned long long)p->sectors[1]);
}
static struct part_attribute part_attr_uevent = {
.attr = {.name = "uevent", .mode = S_IWUSR },
@@ -303,7 +303,8 @@ void delete_partition(struct gendisk *disk, int part)
disk->part[part-1] = NULL;
p->start_sect = 0;
p->nr_sects = 0;
- p->reads = p->writes = p->read_sectors = p->write_sectors = 0;
+ p->ios[0] = p->ios[1] = 0;
+ p->sectors[0] = p->sectors[1] = 0;
devfs_remove("%s/part%d", disk->devfs_name, part);
kobject_unregister(&p->kobj);
}
diff --git a/include/asm-x86_64/dma-mapping.h b/include/asm-x86_64/dma-mapping.h
index 54a380efed413..36d16dfbac88d 100644
--- a/include/asm-x86_64/dma-mapping.h
+++ b/include/asm-x86_64/dma-mapping.h
@@ -85,10 +85,33 @@ static inline void dma_sync_single_for_device(struct device *hwdev,
flush_write_buffers();
}
-#define dma_sync_single_range_for_cpu(dev, dma_handle, offset, size, dir) \
- dma_sync_single_for_cpu(dev, dma_handle, size, dir)
-#define dma_sync_single_range_for_device(dev, dma_handle, offset, size, dir) \
- dma_sync_single_for_device(dev, dma_handle, size, dir)
+static inline void dma_sync_single_range_for_cpu(struct device *hwdev,
+ dma_addr_t dma_handle,
+ unsigned long offset,
+ size_t size, int direction)
+{
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (swiotlb)
+ return swiotlb_sync_single_range_for_cpu(hwdev,dma_handle,offset,size,direction);
+
+ flush_write_buffers();
+}
+
+static inline void dma_sync_single_range_for_device(struct device *hwdev,
+ dma_addr_t dma_handle,
+ unsigned long offset,
+ size_t size, int direction)
+{
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (swiotlb)
+ return swiotlb_sync_single_range_for_device(hwdev,dma_handle,offset,size,direction);
+
+ flush_write_buffers();
+}
static inline void dma_sync_sg_for_cpu(struct device *hwdev,
struct scatterlist *sg,
diff --git a/include/asm-x86_64/swiotlb.h b/include/asm-x86_64/swiotlb.h
index 7cbfd10ecc3c1..dddf1b218681a 100644
--- a/include/asm-x86_64/swiotlb.h
+++ b/include/asm-x86_64/swiotlb.h
@@ -15,6 +15,14 @@ extern void swiotlb_sync_single_for_cpu(struct device *hwdev,
extern void swiotlb_sync_single_for_device(struct device *hwdev,
dma_addr_t dev_addr,
size_t size, int dir);
+extern void swiotlb_sync_single_range_for_cpu(struct device *hwdev,
+ dma_addr_t dev_addr,
+ unsigned long offset,
+ size_t size, int dir);
+extern void swiotlb_sync_single_range_for_device(struct device *hwdev,
+ dma_addr_t dev_addr,
+ unsigned long offset,
+ size_t size, int dir);
extern void swiotlb_sync_sg_for_cpu(struct device *hwdev,
struct scatterlist *sg, int nelems,
int dir);
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index eabdb5cce357f..8eeaa53a68c9f 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -78,7 +78,7 @@ struct hd_struct {
sector_t start_sect;
sector_t nr_sects;
struct kobject kobj;
- unsigned reads, read_sectors, writes, write_sectors;
+ unsigned ios[2], sectors[2];
int policy, partno;
};
@@ -89,10 +89,10 @@ struct hd_struct {
#define GENHD_FL_SUPPRESS_PARTITION_INFO 32
struct disk_stats {
- unsigned read_sectors, write_sectors;
- unsigned reads, writes;
- unsigned read_merges, write_merges;
- unsigned read_ticks, write_ticks;
+ unsigned sectors[2];
+ unsigned ios[2];
+ unsigned merges[2];
+ unsigned ticks[2];
unsigned io_ticks;
unsigned time_in_queue;
};
diff --git a/lib/Makefile b/lib/Makefile
index 44a46750690ae..8535f4d7d1c38 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -44,6 +44,8 @@ obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
+obj-$(CONFIG_SWIOTLB) += swiotlb.o
+
hostprogs-y := gen_crc32table
clean-files := crc32table.h
diff --git a/arch/ia64/lib/swiotlb.c b/lib/swiotlb.c
index 96edcc0fdcd9b..57216f3544cad 100644
--- a/arch/ia64/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -1,7 +1,7 @@
/*
* Dynamic DMA mapping support.
*
- * This implementation is for IA-64 platforms that do not support
+ * This implementation is for IA-64 and EM64T platforms that do not support
* I/O TLBs (aka DMA address translation hardware).
* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
@@ -11,21 +11,23 @@
* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
* unnecessary i-cache flushing.
- * 04/07/.. ak Better overflow handling. Assorted fixes.
+ * 04/07/.. ak Better overflow handling. Assorted fixes.
+ * 05/09/10 linville Add support for syncing ranges, support syncing for
+ * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
*/
#include <linux/cache.h>
+#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/module.h>
-#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ctype.h>
#include <asm/io.h>
-#include <asm/pci.h>
#include <asm/dma.h>
+#include <asm/scatterlist.h>
#include <linux/init.h>
#include <linux/bootmem.h>
@@ -58,6 +60,14 @@
*/
#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
+/*
+ * Enumeration for sync targets
+ */
+enum dma_sync_target {
+ SYNC_FOR_CPU = 0,
+ SYNC_FOR_DEVICE = 1,
+};
+
int swiotlb_force;
/*
@@ -117,7 +127,7 @@ __setup("swiotlb=", setup_io_tlb_npages);
/*
* Statically reserve bounce buffer space and initialize bounce buffer data
- * structures for the software IO TLB used to implement the PCI DMA API.
+ * structures for the software IO TLB used to implement the DMA API.
*/
void
swiotlb_init_with_default_size (size_t default_size)
@@ -397,21 +407,28 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
}
static void
-sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
+sync_single(struct device *hwdev, char *dma_addr, size_t size,
+ int dir, int target)
{
int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
char *buffer = io_tlb_orig_addr[index];
- /*
- * bounce... copy the data back into/from the original buffer
- * XXX How do you handle DMA_BIDIRECTIONAL here ?
- */
- if (dir == DMA_FROM_DEVICE)
- memcpy(buffer, dma_addr, size);
- else if (dir == DMA_TO_DEVICE)
- memcpy(dma_addr, buffer, size);
- else
+ switch (target) {
+ case SYNC_FOR_CPU:
+ if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
+ memcpy(buffer, dma_addr, size);
+ else if (dir != DMA_TO_DEVICE)
+ BUG();
+ break;
+ case SYNC_FOR_DEVICE:
+ if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
+ memcpy(dma_addr, buffer, size);
+ else if (dir != DMA_FROM_DEVICE)
+ BUG();
+ break;
+ default:
BUG();
+ }
}
void *
@@ -485,24 +502,24 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
/*
* Ran out of IOMMU space for this operation. This is very bad.
* Unfortunately the drivers cannot handle this operation properly.
- * unless they check for pci_dma_mapping_error (most don't)
+ * unless they check for dma_mapping_error (most don't)
* When the mapping is small enough return a static buffer to limit
* the damage, or panic when the transfer is too big.
*/
- printk(KERN_ERR "PCI-DMA: Out of SW-IOMMU space for %lu bytes at "
+ printk(KERN_ERR "DMA: Out of SW-IOMMU space for %lu bytes at "
"device %s\n", size, dev ? dev->bus_id : "?");
if (size > io_tlb_overflow && do_panic) {
- if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
- panic("PCI-DMA: Memory would be corrupted\n");
- if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
- panic("PCI-DMA: Random memory would be DMAed\n");
+ if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
+ panic("DMA: Memory would be corrupted\n");
+ if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
+ panic("DMA: Random memory would be DMAed\n");
}
}
/*
* Map a single buffer of the indicated size for DMA in streaming mode. The
- * PCI address to use is returned.
+ * physical address to use is returned.
*
* Once the device is given the dma address, the device owns this memory until
* either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
@@ -589,39 +606,73 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
* after a transfer.
*
* If you perform a swiotlb_map_single() but wish to interrogate the buffer
- * using the cpu, yet do not wish to teardown the PCI dma mapping, you must
- * call this function before doing so. At the next point you give the PCI dma
+ * using the cpu, yet do not wish to teardown the dma mapping, you must
+ * call this function before doing so. At the next point you give the dma
* address back to the card, you must first perform a
* swiotlb_dma_sync_for_device, and then the device again owns the buffer
*/
-void
-swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
- size_t size, int dir)
+static inline void
+swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, int dir, int target)
{
char *dma_addr = phys_to_virt(dev_addr);
if (dir == DMA_NONE)
BUG();
if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
- sync_single(hwdev, dma_addr, size, dir);
+ sync_single(hwdev, dma_addr, size, dir, target);
else if (dir == DMA_FROM_DEVICE)
mark_clean(dma_addr, size);
}
void
+swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, int dir)
+{
+ swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
+}
+
+void
swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
size_t size, int dir)
{
- char *dma_addr = phys_to_virt(dev_addr);
+ swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
+}
+
+/*
+ * Same as above, but for a sub-range of the mapping.
+ */
+static inline void
+swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
+ unsigned long offset, size_t size,
+ int dir, int target)
+{
+ char *dma_addr = phys_to_virt(dev_addr) + offset;
if (dir == DMA_NONE)
BUG();
if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
- sync_single(hwdev, dma_addr, size, dir);
+ sync_single(hwdev, dma_addr, size, dir, target);
else if (dir == DMA_FROM_DEVICE)
mark_clean(dma_addr, size);
}
+void
+swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
+ unsigned long offset, size_t size, int dir)
+{
+ swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
+ SYNC_FOR_CPU);
+}
+
+void
+swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
+ unsigned long offset, size_t size, int dir)
+{
+ swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
+ SYNC_FOR_DEVICE);
+}
+
/*
* Map a set of buffers described by scatterlist in streaming mode for DMA.
* This is the scatter-gather version of the above swiotlb_map_single
@@ -696,9 +747,9 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
* The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
* and usage.
*/
-void
-swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
- int nelems, int dir)
+static inline void
+swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int dir, int target)
{
int i;
@@ -708,22 +759,21 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
for (i = 0; i < nelems; i++, sg++)
if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
sync_single(hwdev, (void *) sg->dma_address,
- sg->dma_length, dir);
+ sg->dma_length, dir, target);
+}
+
+void
+swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int dir)
+{
+ swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
}
void
swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
int nelems, int dir)
{
- int i;
-
- if (dir == DMA_NONE)
- BUG();
-
- for (i = 0; i < nelems; i++, sg++)
- if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
- sync_single(hwdev, (void *) sg->dma_address,
- sg->dma_length, dir);
+ swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
}
int
@@ -733,9 +783,9 @@ swiotlb_dma_mapping_error(dma_addr_t dma_addr)
}
/*
- * Return whether the given PCI device DMA address mask can be supported
+ * Return whether the given device DMA address mask can be supported
* properly. For example, if your device can only drive the low 24-bits
- * during PCI bus mastering, then you would pass 0x00ffffff as the mask to
+ * during bus mastering, then you would pass 0x00ffffff as the mask to
* this function.
*/
int
@@ -751,6 +801,8 @@ EXPORT_SYMBOL(swiotlb_map_sg);
EXPORT_SYMBOL(swiotlb_unmap_sg);
EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
EXPORT_SYMBOL(swiotlb_sync_single_for_device);
+EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
+EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
EXPORT_SYMBOL(swiotlb_dma_mapping_error);
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-25 08:31:27 +0000