Commit 6d36dcd4 authored by Chris Mason's avatar Chris Mason
Browse files

Btrfs: Avoid memcpy where possible in extent_buffers 



Signed-off-by: default avatarChris Mason <chris.mason@oracle.com>
parent 479965d6

fs/btrfs/ctree.h

+40 −5
Original line number Diff line number Diff line
@@ -483,15 +483,50 @@ static inline void btrfs_set_##name(struct extent_buffer *eb, \ 
#define BTRFS_SETGET_HEADER_FUNCS(name, type, member, bits)		\

static inline u##bits btrfs_##name(struct extent_buffer *eb)		\

{									\

	int err;							\

	char *map_token;						\

	char *kaddr;							\

	unsigned long map_start;					\

	unsigned long map_len;						\

	unsigned long offset = offsetof(type, member);			\

	err = map_extent_buffer(eb, offset,				\

			        sizeof(((type *)0)->member),		\

				&map_token, &kaddr,			\

				&map_start, &map_len, KM_USER0);	\

	if (!err) {							\

		__le##bits *tmp = (__le##bits *)(kaddr + offset -	\

					       map_start);		\

		u##bits res = le##bits##_to_cpu(*tmp);			\

		unmap_extent_buffer(eb, map_token, KM_USER0);		\

		return res;						\

	} else {							\

		__le##bits res;						\

		read_eb_member(eb, NULL, type, member, &res);		\

		return le##bits##_to_cpu(res);				\

	}								\

}									\

static inline void btrfs_set_##name(struct extent_buffer *eb,		\

				    u##bits val)			\

{									\

	int err;							\

	char *map_token;						\

	char *kaddr;							\

	unsigned long map_start;					\

	unsigned long map_len;						\

	unsigned long offset = offsetof(type, member);			\

	err = map_extent_buffer(eb, offset,				\

			        sizeof(((type *)0)->member),		\

				&map_token, &kaddr,			\

				&map_start, &map_len, KM_USER0);	\

	if (!err) {							\

		__le##bits *tmp = (__le##bits *)(kaddr + offset -	\

					       map_start);		\

		*tmp = cpu_to_le##bits(val);				\

		unmap_extent_buffer(eb, map_token, KM_USER0);		\

	} else {							\

		val = cpu_to_le##bits(val);				\

		write_eb_member(eb, NULL, type, member, &val);		\

	}								\

}



#define BTRFS_SETGET_STACK_FUNCS(name, type, member, bits)		\

fs/btrfs/disk-io.c

+3 −3
Original line number Diff line number Diff line
@@ -609,20 +609,20 @@ int close_ctree(struct btrfs_root *root) 


int btrfs_buffer_uptodate(struct extent_buffer *buf)

{

	struct inode *btree_inode = buf->pages[0]->mapping->host;

	struct inode *btree_inode = buf->first_page->mapping->host;

	return extent_buffer_uptodate(&BTRFS_I(btree_inode)->extent_tree, buf);

}



int btrfs_set_buffer_uptodate(struct extent_buffer *buf)

{

	struct inode *btree_inode = buf->pages[0]->mapping->host;

	struct inode *btree_inode = buf->first_page->mapping->host;

	return set_extent_buffer_uptodate(&BTRFS_I(btree_inode)->extent_tree,

					  buf);

}



void btrfs_mark_buffer_dirty(struct extent_buffer *buf)

{

	struct btrfs_root *root = BTRFS_I(buf->pages[0]->mapping->host)->root;

	struct btrfs_root *root = BTRFS_I(buf->first_page->mapping->host)->root;

	u64 transid = btrfs_header_generation(buf);

	struct inode *btree_inode = root->fs_info->btree_inode;

fs/btrfs/extent_map.c

+102 −30
Original line number Diff line number Diff line
@@ -18,6 +18,11 @@ struct kmem_cache *btrfs_cache_create(const char *name, size_t size, 


static struct kmem_cache *extent_map_cache;

static struct kmem_cache *extent_state_cache;

static struct kmem_cache *extent_buffer_cache;

static LIST_HEAD(extent_buffers);

static spinlock_t extent_buffers_lock;

static int nr_extent_buffers;

#define MAX_EXTENT_BUFFER_CACHE 128



struct tree_entry {

	u64 start;
@@ -29,21 +34,33 @@ struct tree_entry { 
void __init extent_map_init(void)

{

	extent_map_cache = btrfs_cache_create("extent_map",

					    sizeof(struct extent_map),

					    SLAB_DESTROY_BY_RCU,

					    sizeof(struct extent_map), 0,

					    NULL);

	extent_state_cache = btrfs_cache_create("extent_state",

					    sizeof(struct extent_state),

					    SLAB_DESTROY_BY_RCU,

					    sizeof(struct extent_state), 0,

					    NULL);

	extent_buffer_cache = btrfs_cache_create("extent_buffers",

					    sizeof(struct extent_buffer), 0,

					    NULL);

	spin_lock_init(&extent_buffers_lock);

}



void __exit extent_map_exit(void)

{

	struct extent_buffer *eb;



	while (!list_empty(&extent_buffers)) {

		eb = list_entry(extent_buffers.next,

				struct extent_buffer, list);

		list_del(&eb->list);

		kmem_cache_free(extent_buffer_cache, eb);

	}

	if (extent_map_cache)

		kmem_cache_destroy(extent_map_cache);

	if (extent_state_cache)

		kmem_cache_destroy(extent_state_cache);

	if (extent_buffer_cache)

		kmem_cache_destroy(extent_buffer_cache);

}



void extent_map_tree_init(struct extent_map_tree *tree,
@@ -1858,6 +1875,48 @@ sector_t extent_bmap(struct address_space *mapping, sector_t iblock, 
	return (em->block_start + start - em->start) >> inode->i_blkbits;

}



static struct extent_buffer *__alloc_extent_buffer(gfp_t mask)

{

	struct extent_buffer *eb = NULL;

	spin_lock(&extent_buffers_lock);

	if (!list_empty(&extent_buffers)) {

		eb = list_entry(extent_buffers.next, struct extent_buffer,

				list);

		list_del(&eb->list);

		WARN_ON(nr_extent_buffers == 0);

		nr_extent_buffers--;

	}

	spin_unlock(&extent_buffers_lock);

	if (eb) {

		memset(eb, 0, sizeof(*eb));

		return eb;

	}

	return kmem_cache_zalloc(extent_buffer_cache, mask);

}



static void __free_extent_buffer(struct extent_buffer *eb)

{

	if (nr_extent_buffers >= MAX_EXTENT_BUFFER_CACHE) {

		kmem_cache_free(extent_buffer_cache, eb);

	} else {

		spin_lock(&extent_buffers_lock);

		list_add(&eb->list, &extent_buffers);

		nr_extent_buffers++;

		spin_unlock(&extent_buffers_lock);

	}

}



static inline struct page *extent_buffer_page(struct extent_buffer *eb, int i)

{

	struct page *p;

	if (i == 0)

		return eb->first_page;

	i += eb->start >> PAGE_CACHE_SHIFT;

	p = find_get_page(eb->first_page->mapping, i);

	page_cache_release(p);

	return p;

}



struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,

					  u64 start, unsigned long len,

					  gfp_t mask)
@@ -1871,7 +1930,7 @@ struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree, 
	struct address_space *mapping = tree->mapping;

	int uptodate = 0;



	eb = kzalloc(EXTENT_BUFFER_SIZE(num_pages), mask);

	eb = __alloc_extent_buffer(mask);

	if (!eb || IS_ERR(eb))

		return NULL;
@@ -1881,9 +1940,16 @@ struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree, 


	for (i = 0; i < num_pages; i++, index++) {

		p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);

		if (!p)

		if (!p) {

			/* make sure the free only frees the pages we've

			 * grabbed a reference on

			 */

			eb->len = i << PAGE_CACHE_SHIFT;

			eb->start &= ~((u64)PAGE_CACHE_SIZE - 1);

			goto fail;

		eb->pages[i] = p;

		}

		if (i == 0)

			eb->first_page = p;

		if (!PageUptodate(p))

			uptodate = 0;

		unlock_page(p);
@@ -1909,7 +1975,7 @@ struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree, 
	struct page *p;

	struct address_space *mapping = tree->mapping;



	eb = kzalloc(EXTENT_BUFFER_SIZE(num_pages), mask);

	eb = __alloc_extent_buffer(mask);

	if (!eb || IS_ERR(eb))

		return NULL;
@@ -1919,9 +1985,16 @@ struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree, 


	for (i = 0; i < num_pages; i++, index++) {

		p = find_get_page(mapping, index);

		if (!p)

		if (!p) {

			/* make sure the free only frees the pages we've

			 * grabbed a reference on

			 */

			eb->len = i << PAGE_CACHE_SHIFT;

			eb->start &= ~((u64)PAGE_CACHE_SIZE - 1);

			goto fail;

		eb->pages[i] = p;

		}

		if (i == 0)

			eb->first_page = p;

	}

	return eb;

fail:
@@ -1944,11 +2017,12 @@ void free_extent_buffer(struct extent_buffer *eb) 
	num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) -

		(eb->start >> PAGE_CACHE_SHIFT) + 1;



	for (i = 0; i < num_pages; i++) {

		if (eb->pages[i])

			page_cache_release(eb->pages[i]);

	if (eb->first_page)

		page_cache_release(eb->first_page);

	for (i = 1; i < num_pages; i++) {

		page_cache_release(extent_buffer_page(eb, i));

	}

	kfree(eb);

	__free_extent_buffer(eb);

}

EXPORT_SYMBOL(free_extent_buffer);
@@ -1968,7 +2042,7 @@ int clear_extent_buffer_dirty(struct extent_map_tree *tree, 
		(eb->start >> PAGE_CACHE_SHIFT) + 1;



	for (i = 0; i < num_pages; i++) {

		page = eb->pages[i];

		page = extent_buffer_page(eb, i);

		lock_page(page);

		/*

		 * if we're on the last page or the first page and the
@@ -2021,7 +2095,7 @@ int set_extent_buffer_uptodate(struct extent_map_tree *tree, 
	set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,

			    GFP_NOFS);

	for (i = 0; i < num_pages; i++) {

		page = eb->pages[i];

		page = extent_buffer_page(eb, i);

		if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||

		    ((i == num_pages - 1) &&

		     ((eb->start + eb->len - 1) & (PAGE_CACHE_SIZE - 1)))) {
@@ -2064,7 +2138,7 @@ int read_extent_buffer_pages(struct extent_map_tree *tree, 
	num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) -

		(eb->start >> PAGE_CACHE_SHIFT) + 1;

	for (i = 0; i < num_pages; i++) {

		page = eb->pages[i];

		page = extent_buffer_page(eb, i);

		if (PageUptodate(page)) {

			continue;

		}
@@ -2090,7 +2164,7 @@ int read_extent_buffer_pages(struct extent_map_tree *tree, 
	}



	for (i = 0; i < num_pages; i++) {

		page = eb->pages[i];

		page = extent_buffer_page(eb, i);

		wait_on_page_locked(page);

		if (!PageUptodate(page)) {

			ret = -EIO;
@@ -2116,12 +2190,12 @@ void read_extent_buffer(struct extent_buffer *eb, void *dstv, 
	WARN_ON(start > eb->len);

	WARN_ON(start + len > eb->start + eb->len);



	page = eb->pages[i];

	offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);

	if (i == 0)

		offset += start_offset;



	while(len > 0) {

		page = extent_buffer_page(eb, i);

		WARN_ON(!PageUptodate(page));



		cur = min(len, (PAGE_CACHE_SIZE - offset));
@@ -2134,7 +2208,6 @@ void read_extent_buffer(struct extent_buffer *eb, void *dstv, 
		len -= cur;

		offset = 0;

		i++;

		page = eb->pages[i];

	}

}

EXPORT_SYMBOL(read_extent_buffer);
@@ -2165,7 +2238,7 @@ int map_extent_buffer(struct extent_buffer *eb, unsigned long start, 
	}



	// kaddr = kmap_atomic(eb->pages[i], km);

	kaddr = page_address(eb->pages[i]);

	kaddr = page_address(extent_buffer_page(eb, i));

	*token = kaddr;

	*map = kaddr + offset;

	*map_len = PAGE_CACHE_SIZE - offset;
@@ -2195,12 +2268,12 @@ int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv, 
	WARN_ON(start > eb->len);

	WARN_ON(start + len > eb->start + eb->len);



	page = eb->pages[i];

	offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);

	if (i == 0)

		offset += start_offset;



	while(len > 0) {

		page = extent_buffer_page(eb, i);

		WARN_ON(!PageUptodate(page));



		cur = min(len, (PAGE_CACHE_SIZE - offset));
@@ -2216,7 +2289,6 @@ int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv, 
		len -= cur;

		offset = 0;

		i++;

		page = eb->pages[i];

	}

	return ret;

}
@@ -2236,12 +2308,12 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv, 
	WARN_ON(start > eb->len);

	WARN_ON(start + len > eb->start + eb->len);



	page = eb->pages[i];

	offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);

	if (i == 0)

		offset += start_offset;



	while(len > 0) {

		page = extent_buffer_page(eb, i);

		WARN_ON(!PageUptodate(page));



		cur = min(len, PAGE_CACHE_SIZE - offset);
@@ -2254,7 +2326,6 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv, 
		len -= cur;

		offset = 0;

		i++;

		page = eb->pages[i];

	}

}

EXPORT_SYMBOL(write_extent_buffer);
@@ -2272,12 +2343,12 @@ void memset_extent_buffer(struct extent_buffer *eb, char c, 
	WARN_ON(start > eb->len);

	WARN_ON(start + len > eb->start + eb->len);



	page = eb->pages[i];

	offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);

	if (i == 0)

		offset += start_offset;



	while(len > 0) {

		page = extent_buffer_page(eb, i);

		WARN_ON(!PageUptodate(page));



		cur = min(len, PAGE_CACHE_SIZE - offset);
@@ -2289,7 +2360,6 @@ void memset_extent_buffer(struct extent_buffer *eb, char c, 
		len -= cur;

		offset = 0;

		i++;

		page = eb->pages[i];

	}

}

EXPORT_SYMBOL(memset_extent_buffer);
@@ -2313,7 +2383,7 @@ void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, 
		offset += start_offset;



	while(len > 0) {

		page = dst->pages[i];

		page = extent_buffer_page(dst, i);

		WARN_ON(!PageUptodate(page));



		cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
@@ -2414,7 +2484,8 @@ void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 
		cur = min(cur, (unsigned long)(PAGE_CACHE_SIZE -

					       dst_off_in_page));



		copy_pages(dst->pages[dst_i], dst->pages[src_i],

		copy_pages(extent_buffer_page(dst, dst_i),

			   extent_buffer_page(dst, src_i),

			   dst_off_in_page, src_off_in_page, cur);



		src_offset += cur;
@@ -2467,7 +2538,8 @@ void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 
		cur = min(len, src_off_in_page + 1);

		cur = min(cur, dst_off_in_page + 1);

// printk("move pages orig dst %lu src %lu len %lu, this %lu %lu %lu\n", dst_offset, src_offset, len, dst_off_in_page - cur + 1, src_off_in_page - cur + 1, cur);

		move_pages(dst->pages[dst_i], dst->pages[src_i],

		move_pages(extent_buffer_page(dst, dst_i),

			   extent_buffer_page(dst, src_i),

			   dst_off_in_page - cur + 1,

			   src_off_in_page - cur + 1, cur);

fs/btrfs/extent_map.h

+2 −3
Original line number Diff line number Diff line
@@ -62,14 +62,13 @@ struct extent_state { 
	struct list_head list;

};



#define EXTENT_BUFFER_SIZE(nr) (sizeof(struct extent_buffer) + \

			       (nr - 1) * sizeof(struct page *))

struct extent_buffer {

	u64 start;

	unsigned long len;

	atomic_t refs;

	int flags;

	struct page *pages[];

	struct list_head list;

	struct page *first_page;

};



typedef struct extent_map *(get_extent_t)(struct inode *inode,