/* * (C) 2001 Clemson University and The University of Chicago * * See COPYING in top-level directory. */ #include "protocol.h" #include "orangefs-kernel.h" #include "orangefs-bufmap.h" struct slot_map { int c; wait_queue_head_t q; int count; unsigned long *map; }; static struct slot_map rw_map = { .c = -1, .q = __WAIT_QUEUE_HEAD_INITIALIZER(rw_map.q) }; static struct slot_map readdir_map = { .c = -1, .q = __WAIT_QUEUE_HEAD_INITIALIZER(readdir_map.q) }; static void install(struct slot_map *m, int count, unsigned long *map) { spin_lock(&m->q.lock); m->c = m->count = count; m->map = map; wake_up_all_locked(&m->q); spin_unlock(&m->q.lock); } static void mark_killed(struct slot_map *m) { spin_lock(&m->q.lock); m->c -= m->count + 1; spin_unlock(&m->q.lock); } static void run_down(struct slot_map *m) { DEFINE_WAIT(wait); spin_lock(&m->q.lock); if (m->c != -1) { for (;;) { if (likely(list_empty(&wait.task_list))) __add_wait_queue_tail(&m->q, &wait); set_current_state(TASK_UNINTERRUPTIBLE); if (m->c == -1) break; spin_unlock(&m->q.lock); schedule(); spin_lock(&m->q.lock); } __remove_wait_queue(&m->q, &wait); __set_current_state(TASK_RUNNING); } m->map = NULL; spin_unlock(&m->q.lock); } static void put(struct slot_map *m, int slot) { int v; spin_lock(&m->q.lock); __clear_bit(slot, m->map); v = ++m->c; if (unlikely(v == 1)) /* no free slots -> one free slot */ wake_up_locked(&m->q); else if (unlikely(v == -1)) /* finished dying */ wake_up_all_locked(&m->q); spin_unlock(&m->q.lock); } static int wait_for_free(struct slot_map *m) { long left = slot_timeout_secs * HZ; DEFINE_WAIT(wait); do { long n = left, t; if (likely(list_empty(&wait.task_list))) __add_wait_queue_tail_exclusive(&m->q, &wait); set_current_state(TASK_INTERRUPTIBLE); if (m->c > 0) break; if (m->c < 0) { /* we are waiting for map to be installed */ /* it would better be there soon, or we go away */ if (n > ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS * HZ) n = ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS * HZ; } spin_unlock(&m->q.lock); t = schedule_timeout(n); spin_lock(&m->q.lock); if (unlikely(!t) && n != left && m->c < 0) left = t; else left = t + (left - n); if (unlikely(signal_pending(current))) left = -EINTR; } while (left > 0); if (!list_empty(&wait.task_list)) list_del(&wait.task_list); else if (left <= 0 && waitqueue_active(&m->q)) __wake_up_locked_key(&m->q, TASK_INTERRUPTIBLE, NULL); __set_current_state(TASK_RUNNING); if (likely(left > 0)) return 0; return left < 0 ? -EINTR : -ETIMEDOUT; } static int get(struct slot_map *m) { int res = 0; spin_lock(&m->q.lock); if (unlikely(m->c <= 0)) res = wait_for_free(m); if (likely(!res)) { m->c--; res = find_first_zero_bit(m->map, m->count); __set_bit(res, m->map); } spin_unlock(&m->q.lock); return res; } /* used to describe mapped buffers */ struct orangefs_bufmap_desc { void *uaddr; /* user space address pointer */ struct page **page_array; /* array of mapped pages */ int array_count; /* size of above arrays */ struct list_head list_link; }; static struct orangefs_bufmap { int desc_size; int desc_shift; int desc_count; int total_size; int page_count; struct page **page_array; struct orangefs_bufmap_desc *desc_array; /* array to track usage of buffer descriptors */ unsigned long *buffer_index_array; /* array to track usage of buffer descriptors for readdir */ #define N DIV_ROUND_UP(ORANGEFS_READDIR_DEFAULT_DESC_COUNT, BITS_PER_LONG) unsigned long readdir_index_array[N]; #undef N } *__orangefs_bufmap; static DEFINE_SPINLOCK(orangefs_bufmap_lock); static void orangefs_bufmap_unmap(struct orangefs_bufmap *bufmap) { int i; for (i = 0; i < bufmap->page_count; i++) put_page(bufmap->page_array[i]); } static void orangefs_bufmap_free(struct orangefs_bufmap *bufmap) { kfree(bufmap->page_array); kfree(bufmap->desc_array); kfree(bufmap->buffer_index_array); kfree(bufmap); } /* * XXX: Can the size and shift change while the caller gives up the * XXX: lock between calling this and doing something useful? */ int orangefs_bufmap_size_query(void) { struct orangefs_bufmap *bufmap; int size = 0; spin_lock(&orangefs_bufmap_lock); bufmap = __orangefs_bufmap; if (bufmap) size = bufmap->desc_size; spin_unlock(&orangefs_bufmap_lock); return size; } int orangefs_bufmap_shift_query(void) { struct orangefs_bufmap *bufmap; int shift = 0; spin_lock(&orangefs_bufmap_lock); bufmap = __orangefs_bufmap; if (bufmap) shift = bufmap->desc_shift; spin_unlock(&orangefs_bufmap_lock); return shift; } static DECLARE_WAIT_QUEUE_HEAD(bufmap_waitq); static DECLARE_WAIT_QUEUE_HEAD(readdir_waitq); /* * orangefs_get_bufmap_init * * If bufmap_init is 1, then the shared memory system, including the * buffer_index_array, is available. Otherwise, it is not. * * returns the value of bufmap_init */ int orangefs_get_bufmap_init(void) { return __orangefs_bufmap ? 1 : 0; } static struct orangefs_bufmap * orangefs_bufmap_alloc(struct ORANGEFS_dev_map_desc *user_desc) { struct orangefs_bufmap *bufmap; bufmap = kzalloc(sizeof(*bufmap), GFP_KERNEL); if (!bufmap) goto out; bufmap->total_size = user_desc->total_size; bufmap->desc_count = user_desc->count; bufmap->desc_size = user_desc->size; bufmap->desc_shift = ilog2(bufmap->desc_size); bufmap->buffer_index_array = kzalloc(DIV_ROUND_UP(bufmap->desc_count, BITS_PER_LONG), GFP_KERNEL); if (!bufmap->buffer_index_array) { gossip_err("orangefs: could not allocate %d buffer indices\n", bufmap->desc_count); goto out_free_bufmap; } bufmap->desc_array = kcalloc(bufmap->desc_count, sizeof(struct orangefs_bufmap_desc), GFP_KERNEL); if (!bufmap->desc_array) { gossip_err("orangefs: could not allocate %d descriptors\n", bufmap->desc_count); goto out_free_index_array; } bufmap->page_count = bufmap->total_size / PAGE_SIZE; /* allocate storage to track our page mappings */ bufmap->page_array = kcalloc(bufmap->page_count, sizeof(struct page *), GFP_KERNEL); if (!bufmap->page_array) goto out_free_desc_array; return bufmap; out_free_desc_array: kfree(bufmap->desc_array); out_free_index_array: kfree(bufmap->buffer_index_array); out_free_bufmap: kfree(bufmap); out: return NULL; } static int orangefs_bufmap_map(struct orangefs_bufmap *bufmap, struct ORANGEFS_dev_map_desc *user_desc) { int pages_per_desc = bufmap->desc_size / PAGE_SIZE; int offset = 0, ret, i; /* map the pages */ ret = get_user_pages_fast((unsigned long)user_desc->ptr, bufmap->page_count, 1, bufmap->page_array); if (ret < 0) return ret; if (ret != bufmap->page_count) { gossip_err("orangefs error: asked for %d pages, only got %d.\n", bufmap->page_count, ret); for (i = 0; i < ret; i++) { SetPageError(bufmap->page_array[i]); put_page(bufmap->page_array[i]); } return -ENOMEM; } /* * ideally we want to get kernel space pointers for each page, but * we can't kmap that many pages at once if highmem is being used. * so instead, we just kmap/kunmap the page address each time the * kaddr is needed. */ for (i = 0; i < bufmap->page_count; i++) flush_dcache_page(bufmap->page_array[i]); /* build a list of available descriptors */ for (offset = 0, i = 0; i < bufmap->desc_count; i++) { bufmap->desc_array[i].page_array = &bufmap->page_array[offset]; bufmap->desc_array[i].array_count = pages_per_desc; bufmap->desc_array[i].uaddr = (user_desc->ptr + (i * pages_per_desc * PAGE_SIZE)); offset += pages_per_desc; } return 0; } /* * orangefs_bufmap_initialize() * * initializes the mapped buffer interface * * returns 0 on success, -errno on failure */ int orangefs_bufmap_initialize(struct ORANGEFS_dev_map_desc *user_desc) { struct orangefs_bufmap *bufmap; int ret = -EINVAL; gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_initialize: called (ptr (" "%p) sz (%d) cnt(%d).\n", user_desc->ptr, user_desc->size, user_desc->count); /* * sanity check alignment and size of buffer that caller wants to * work with */ if (PAGE_ALIGN((unsigned long)user_desc->ptr) != (unsigned long)user_desc->ptr) { gossip_err("orangefs error: memory alignment (front). %p\n", user_desc->ptr); goto out; } if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->total_size)) != (unsigned long)(user_desc->ptr + user_desc->total_size)) { gossip_err("orangefs error: memory alignment (back).(%p + %d)\n", user_desc->ptr, user_desc->total_size); goto out; } if (user_desc->total_size != (user_desc->size * user_desc->count)) { gossip_err("orangefs error: user provided an oddly sized buffer: (%d, %d, %d)\n", user_desc->total_size, user_desc->size, user_desc->count); goto out; } if ((user_desc->size % PAGE_SIZE) != 0) { gossip_err("orangefs error: bufmap size not page size divisible (%d).\n", user_desc->size); goto out; } ret = -ENOMEM; bufmap = orangefs_bufmap_alloc(user_desc); if (!bufmap) goto out; ret = orangefs_bufmap_map(bufmap, user_desc); if (ret) goto out_free_bufmap; spin_lock(&orangefs_bufmap_lock); if (__orangefs_bufmap) { spin_unlock(&orangefs_bufmap_lock); gossip_err("orangefs: error: bufmap already initialized.\n"); ret = -EINVAL; goto out_unmap_bufmap; } __orangefs_bufmap = bufmap; install(&rw_map, bufmap->desc_count, bufmap->buffer_index_array); install(&readdir_map, ORANGEFS_READDIR_DEFAULT_DESC_COUNT, bufmap->readdir_index_array); spin_unlock(&orangefs_bufmap_lock); gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_initialize: exiting normally\n"); return 0; out_unmap_bufmap: orangefs_bufmap_unmap(bufmap); out_free_bufmap: orangefs_bufmap_free(bufmap); out: return ret; } /* * orangefs_bufmap_finalize() * * shuts down the mapped buffer interface and releases any resources * associated with it * * no return value */ void orangefs_bufmap_finalize(void) { struct orangefs_bufmap *bufmap = __orangefs_bufmap; if (!bufmap) return; gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_finalize: called\n"); mark_killed(&rw_map); mark_killed(&readdir_map); gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_finalize: exiting normally\n"); } void orangefs_bufmap_run_down(void) { struct orangefs_bufmap *bufmap = __orangefs_bufmap; if (!bufmap) return; run_down(&rw_map); run_down(&readdir_map); spin_lock(&orangefs_bufmap_lock); __orangefs_bufmap = NULL; spin_unlock(&orangefs_bufmap_lock); orangefs_bufmap_unmap(bufmap); orangefs_bufmap_free(bufmap); } /* * orangefs_bufmap_get() * * gets a free mapped buffer descriptor, will sleep until one becomes * available if necessary * * returns slot on success, -errno on failure */ int orangefs_bufmap_get(void) { return get(&rw_map); } /* * orangefs_bufmap_put() * * returns a mapped buffer descriptor to the collection * * no return value */ void orangefs_bufmap_put(int buffer_index) { put(&rw_map, buffer_index); } /* * orangefs_readdir_index_get() * * gets a free descriptor, will sleep until one becomes * available if necessary. * Although the readdir buffers are not mapped into kernel space * we could do that at a later point of time. Regardless, these * indices are used by the client-core. * * returns slot on success, -errno on failure */ int orangefs_readdir_index_get(void) { return get(&readdir_map); } void orangefs_readdir_index_put(int buffer_index) { put(&readdir_map, buffer_index); } /* * we've been handed an iovec, we need to copy it to * the shared memory descriptor at "buffer_index". */ int orangefs_bufmap_copy_from_iovec(struct iov_iter *iter, int buffer_index, size_t size) { struct orangefs_bufmap_desc *to; int i; gossip_debug(GOSSIP_BUFMAP_DEBUG, "%s: buffer_index:%d: size:%zu:\n", __func__, buffer_index, size); to = &__orangefs_bufmap->desc_array[buffer_index]; for (i = 0; size; i++) { struct page *page = to->page_array[i]; size_t n = size; if (n > PAGE_SIZE) n = PAGE_SIZE; n = copy_page_from_iter(page, 0, n, iter); if (!n) return -EFAULT; size -= n; } return 0; } /* * we've been handed an iovec, we need to fill it from * the shared memory descriptor at "buffer_index". */ int orangefs_bufmap_copy_to_iovec(struct iov_iter *iter, int buffer_index, size_t size) { struct orangefs_bufmap_desc *from; int i; from = &__orangefs_bufmap->desc_array[buffer_index]; gossip_debug(GOSSIP_BUFMAP_DEBUG, "%s: buffer_index:%d: size:%zu:\n", __func__, buffer_index, size); for (i = 0; size; i++) { struct page *page = from->page_array[i]; size_t n = size; if (n > PAGE_SIZE) n = PAGE_SIZE; n = copy_page_to_iter(page, 0, n, iter); if (!n) return -EFAULT; size -= n; } return 0; } logplain -rw-r--r--exynos4415.h9828logplain -rw-r--r--exynos5250.h4616logplain -rw-r--r--exynos5260-clk.h14876logplain -rw-r--r--exynos5410.h1689logplain -rw-r--r--exynos5420.h6857logplain -rw-r--r--exynos5433.h45372logplain -rw-r--r--exynos5440.h1141logplain -rw-r--r--exynos7-clk.h5281logplain -rw-r--r--gxbb-aoclkc.h2866logplain -rw-r--r--gxbb-clkc.h592logplain -rw-r--r--hi3516cv300-clock.h1668logplain -rw-r--r--hi3519-clock.h1328logplain -rw-r--r--hi3620-clock.h4496logplain -rw-r--r--hi6220-clock.h4508logplain -rw-r--r--hip04-clock.h1137logplain -rw-r--r--histb-clock.h2012logplain -rw-r--r--hix5hd2-clock.h2415logplain -rw-r--r--imx1-clock.h1055logplain -rw-r--r--imx21-clock.h2461logplain -rw-r--r--imx27-clock.h3494logplain -rw-r--r--imx5-clock.h7212logplain -rw-r--r--imx6qdl-clock.h9593logplain -rw-r--r--imx6sl-clock.h5849logplain -rw-r--r--imx6sx-clock.h9099logplain -rw-r--r--imx6ul-clock.h8203logplain -rw-r--r--imx7d-clock.h15974logplain -rw-r--r--jz4740-cgu.h1028logplain -rw-r--r--jz4780-cgu.h2470logplain -rw-r--r--lpc18xx-ccu.h2134logplain -rw-r--r--lpc18xx-cgu.h1142logplain -rw-r--r--lpc32xx-clock.h1633logplain -rw-r--r--lsi,axm5516-clks.h974logplain -rw-r--r--marvell,mmp2.h2022logplain -rw-r--r--marvell,pxa168.h1654logplain -rw-r--r--marvell,pxa1928.h1535logplain -rw-r--r--marvell,pxa910.h1598logplain -rw-r--r--maxim,max77620.h632logplain -rw-r--r--maxim,max77686.h648logplain -rw-r--r--maxim,max77802.h630logplain -rw-r--r--meson8b-clkc.h523logplain -rw-r--r--microchip,pic32-clock.h1150logplain -rw-r--r--mpc512x-clock.h2236logplain -rw-r--r--mt2701-clk.h13832logplain -rw-r--r--mt8135-clk.h5641logplain -rw-r--r--mt8173-clk.h9293logplain -rw-r--r--oxsemi,ox810se.h1002logplain -rw-r--r--oxsemi,ox820.h1203logplain -rw-r--r--pistachio-clk.h4863logplain -rw-r--r--pxa-clock.h1715logplain -rw-r--r--qcom,gcc-apq8084.h12872logplain -rw-r--r--qcom,gcc-ipq4019.h5423logplain -rw-r--r--qcom,gcc-ipq806x.h8574logplain -rw-r--r--qcom,gcc-mdm9615.h9497logplain -rw-r--r--qcom,gcc-msm8660.h7932logplain -rw-r--r--qcom,gcc-msm8916.h6190logplain -rw-r--r--qcom,gcc-msm8960.h9342logplain -rw-r--r--qcom,gcc-msm8974.h12340logplain -rw-r--r--qcom,gcc-msm8994.h4858logplain -rw-r--r--qcom,gcc-msm8996.h12575logplain -rw-r--r--qcom,lcc-ipq806x.h899logplain -rw-r--r--qcom,lcc-mdm9615.h1701logplain -rw-r--r--qcom,lcc-msm8960.h1616logplain -rw-r--r--qcom,mmcc-apq8084.h5722logplain -rw-r--r--qcom,mmcc-msm8960.h4109logplain -rw-r--r--qcom,mmcc-msm8974.h5223logplain -rw-r--r--qcom,mmcc-msm8996.h9403logplain -rw-r--r--qcom,rpmcc.h2101logplain -rw-r--r--r7s72100-clock.h1218logplain -rw-r--r--r8a73a4-clock.h1596logplain -rw-r--r--r8a7740-clock.h1992logplain -rw-r--r--r8a7743-cpg-mssr.h1269logplain -rw-r--r--r8a7745-cpg-mssr.h1298logplain -rw-r--r--r8a7778-clock.h1855logplain -rw-r--r--r8a7779-clock.h1647logplain -rw-r--r--r8a7790-clock.h4367logplain -rw-r--r--r8a7791-clock.h4388logplain -rw-r--r--r8a7792-clock.h2562logplain -rw-r--r--r8a7793-clock.h4561logplain -rw-r--r--r8a7794-clock.h3679logplain -rw-r--r--r8a7795-cpg-mssr.h1890logplain -rw-r--r--r8a7796-cpg-mssr.h2066logplain -rw-r--r--renesas-cpg-mssr.h542logplain -rw-r--r--rk1108-cru.h6605logplain -rw-r--r--rk3036-cru.h4584logplain -rw-r--r--rk3066a-cru.h1068logplain -rw-r--r--rk3188-cru-common.h6105logplain -rw-r--r--rk3188-cru.h1435logplain