summaryrefslogtreecommitdiff
path: root/net
diff options
context:
space:
mode:
authorAlexander Duyck <alexander.h.duyck@redhat.com>2015-05-06 21:11:51 -0700
committerDavid S. Miller <davem@davemloft.net>2015-05-12 10:39:26 -0400
commit0e39250845c0f91acc64264709b25f7f9b85c2c3 (patch)
treed9b2a71f0af7180f5f5fcb7f6a17bf93bf67a3bc /net
parent2ee52ad4962b32797bac33fa29ec8159e64c4ee3 (diff)
downloadlinux-0e39250845c0f91acc64264709b25f7f9b85c2c3.tar.gz
net: Store virtual address instead of page in netdev_alloc_cache
This change makes it so that we store the virtual address of the page in the netdev_alloc_cache instead of the page pointer. The idea behind this is to avoid multiple calls to page_address since the virtual address is required for every access, but the page pointer is only needed at allocation or reset of the page. While I was at it I also reordered the netdev_alloc_cache structure a bit so that the size is always 16 bytes by dropping size in the case where PAGE_SIZE is greater than or equal to 32KB. Signed-off-by: Alexander Duyck <alexander.h.duyck@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net')
-rw-r--r--net/core/skbuff.c55
1 files changed, 32 insertions, 23 deletions
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index d6851ca32598..a3062ec341c3 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -348,7 +348,13 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size)
EXPORT_SYMBOL(build_skb);
struct netdev_alloc_cache {
- struct page_frag frag;
+ void * va;
+#if (PAGE_SIZE < NETDEV_FRAG_PAGE_MAX_SIZE)
+ __u16 offset;
+ __u16 size;
+#else
+ __u32 offset;
+#endif
/* we maintain a pagecount bias, so that we dont dirty cache line
* containing page->_count every time we allocate a fragment.
*/
@@ -361,21 +367,20 @@ static DEFINE_PER_CPU(struct netdev_alloc_cache, napi_alloc_cache);
static struct page *__page_frag_refill(struct netdev_alloc_cache *nc,
gfp_t gfp_mask)
{
- const unsigned int order = NETDEV_FRAG_PAGE_MAX_ORDER;
struct page *page = NULL;
gfp_t gfp = gfp_mask;
- if (order) {
- gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
- __GFP_NOMEMALLOC;
- page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
- nc->frag.size = PAGE_SIZE << (page ? order : 0);
- }
-
+#if (PAGE_SIZE < NETDEV_FRAG_PAGE_MAX_SIZE)
+ gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
+ __GFP_NOMEMALLOC;
+ page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
+ NETDEV_FRAG_PAGE_MAX_ORDER);
+ nc->size = page ? NETDEV_FRAG_PAGE_MAX_SIZE : PAGE_SIZE;
+#endif
if (unlikely(!page))
page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
- nc->frag.page = page;
+ nc->va = page ? page_address(page) : NULL;
return page;
}
@@ -383,19 +388,20 @@ static struct page *__page_frag_refill(struct netdev_alloc_cache *nc,
static void *__alloc_page_frag(struct netdev_alloc_cache *nc,
unsigned int fragsz, gfp_t gfp_mask)
{
- struct page *page = nc->frag.page;
- unsigned int size;
+ unsigned int size = PAGE_SIZE;
+ struct page *page;
int offset;
- if (unlikely(!page)) {
+ if (unlikely(!nc->va)) {
refill:
page = __page_frag_refill(nc, gfp_mask);
if (!page)
return NULL;
- /* if size can vary use frag.size else just use PAGE_SIZE */
- size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
-
+#if (PAGE_SIZE < NETDEV_FRAG_PAGE_MAX_SIZE)
+ /* if size can vary use size else just use PAGE_SIZE */
+ size = nc->size;
+#endif
/* Even if we own the page, we do not use atomic_set().
* This would break get_page_unless_zero() users.
*/
@@ -404,17 +410,20 @@ refill:
/* reset page count bias and offset to start of new frag */
nc->pfmemalloc = page->pfmemalloc;
nc->pagecnt_bias = size;
- nc->frag.offset = size;
+ nc->offset = size;
}
- offset = nc->frag.offset - fragsz;
+ offset = nc->offset - fragsz;
if (unlikely(offset < 0)) {
+ page = virt_to_page(nc->va);
+
if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
goto refill;
- /* if size can vary use frag.size else just use PAGE_SIZE */
- size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
-
+#if (PAGE_SIZE < NETDEV_FRAG_PAGE_MAX_SIZE)
+ /* if size can vary use size else just use PAGE_SIZE */
+ size = nc->size;
+#endif
/* OK, page count is 0, we can safely set it */
atomic_set(&page->_count, size);
@@ -424,9 +433,9 @@ refill:
}
nc->pagecnt_bias--;
- nc->frag.offset = offset;
+ nc->offset = offset;
- return page_address(page) + offset;
+ return nc->va + offset;
}
static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)