/* * Copyright (C) 2016 Red Hat, Inc. * Author: Michael S. Tsirkin * This work is licensed under the terms of the GNU GPL, version 2. * * Simple descriptor-based ring. virtio 0.9 compatible event index is used for * signalling, unconditionally. */ #define _GNU_SOURCE #include "main.h" #include #include #include /* Next - Where next entry will be written. * Prev - "Next" value when event triggered previously. * Event - Peer requested event after writing this entry. */ static inline bool need_event(unsigned short event, unsigned short next, unsigned short prev) { return (unsigned short)(next - event - 1) < (unsigned short)(next - prev); } /* Design: * Guest adds descriptors with unique index values and DESC_HW in flags. * Host overwrites used descriptors with correct len, index, and DESC_HW clear. * Flags are always set last. */ #define DESC_HW 0x1 struct desc { unsigned short flags; unsigned short index; unsigned len; unsigned long long addr; }; /* how much padding is needed to avoid false cache sharing */ #define HOST_GUEST_PADDING 0x80 /* Mostly read */ struct event { unsigned short kick_index; unsigned char reserved0[HOST_GUEST_PADDING - 2]; unsigned short call_index; unsigned char reserved1[HOST_GUEST_PADDING - 2]; }; struct data { void *buf; /* descriptor is writeable, we can't get buf from there */ void *data; } *data; struct desc *ring; struct event *event; struct guest { unsigned avail_idx; unsigned last_used_idx; unsigned num_free; unsigned kicked_avail_idx; unsigned char reserved[HOST_GUEST_PADDING - 12]; } guest; struct host { /* we do not need to track last avail index * unless we have more than one in flight. */ unsigned used_idx; unsigned called_used_idx; unsigned char reserved[HOST_GUEST_PADDING - 4]; } host; /* implemented by ring */ void alloc_ring(void) { int ret; int i; ret = posix_memalign((void **)&ring, 0x1000, ring_size * sizeof *ring); if (ret) { perror("Unable to allocate ring buffer.\n"); exit(3); } event = malloc(sizeof *event); if (!event) { perror("Unable to allocate event buffer.\n"); exit(3); } memset(event, 0, sizeof *event); guest.avail_idx = 0; guest.kicked_avail_idx = -1; guest.last_used_idx = 0; host.used_idx = 0; host.called_used_idx = -1; for (i = 0; i < ring_size; ++i) { struct desc desc = { .index = i, }; ring[i] = desc; } guest.num_free = ring_size; data = malloc(ring_size * sizeof *data); if (!data) { perror("Unable to allocate data buffer.\n"); exit(3); } memset(data, 0, ring_size * sizeof *data); } /* guest side */ int add_inbuf(unsigned len, void *buf, void *datap) { unsigned head, index; if (!guest.num_free) return -1; guest.num_free--; head = (ring_size - 1) & (guest.avail_idx++); /* Start with a write. On MESI architectures this helps * avoid a shared state with consumer that is polling this descriptor. */ ring[head].addr = (unsigned long)(void*)buf; ring[head].len = len; /* read below might bypass write above. That is OK because it's just an * optimization. If this happens, we will get the cache line in a * shared state which is unfortunate, but probably not worth it to * add an explicit full barrier to avoid this. */ barrier(); index = ring[head].index; data[index].buf = buf; data[index].data = datap; /* Barrier A (for pairing) */ smp_release(); ring[head].flags = DESC_HW; return 0; } void *get_buf(unsigned *lenp, void **bufp) { unsigned head = (ring_size - 1) & guest.last_used_idx; unsigned index; void *datap; if (ring[head].flags & DESC_HW) return NULL; /* Barrier B (for pairing) */ smp_acquire(); *lenp = ring[head].len; index = ring[head].index & (ring_size - 1); datap = data[index].data; *bufp = data[index].buf; data[index].buf = NULL; data[index].data = NULL; guest.num_free++; guest.last_used_idx++; return datap; } bool used_empty() { unsigned head = (ring_size - 1) & guest.last_used_idx; return (ring[head].flags & DESC_HW); } void disable_call() { /* Doing nothing to disable calls might cause * extra interrupts, but reduces the number of cache misses. */ } bool enable_call() { event->call_index = guest.last_used_idx; /* Flush call index write */ /* Barrier D (for pairing) */ smp_mb(); return used_empty(); } void kick_available(void) { /* Flush in previous flags write */ /* Barrier C (for pairing) */ smp_mb(); if (!need_event(event->kick_index, guest.avail_idx, guest.kicked_avail_idx)) return; guest.kicked_avail_idx = guest.avail_idx; kick(); } /* host side */ void disable_kick() { /* Doing nothing to disable kicks might cause * extra interrupts, but reduces the number of cache misses. */ } bool enable_kick() { event->kick_index = host.used_idx; /* Barrier C (for pairing) */ smp_mb(); return avail_empty(); } bool avail_empty() { unsigned head = (ring_size - 1) & host.used_idx; return !(ring[head].flags & DESC_HW); } bool use_buf(unsigned *lenp, void **bufp) { unsigned head = (ring_size - 1) & host.used_idx; if (!(ring[head].flags & DESC_HW)) return false; /* make sure length read below is not speculated */ /* Barrier A (for pairing) */ smp_acquire(); /* simple in-order completion: we don't need * to touch index at all. This also means we * can just modify the descriptor in-place. */ ring[head].len--; /* Make sure len is valid before flags. * Note: alternative is to write len and flags in one access - * possible on 64 bit architectures but wmb is free on Intel anyway * so I have no way to test whether it's a gain. */ /* Barrier B (for pairing) */ smp_release(); ring[head].flags = 0; host.used_idx++; return true; } void call_used(void) { /* Flush in previous flags write */ /* Barrier D (for pairing) */ smp_mb(); if (!need_event(event->call_index, host.used_idx, host.called_used_idx)) return; host.called_used_idx = host.used_idx; call(); } /dt-bindings/clock/bcm-cygnus.h?id=034dd34ff4916ec1f8f74e39ca3efb04eab2f791'>plain -rw-r--r--bcm-ns2.h2915logplain -rw-r--r--bcm-nsp.h2148logplain -rw-r--r--bcm21664.h1984logplain -rw-r--r--bcm281xx.h2456logplain -rw-r--r--bcm2835-aux.h635logplain -rw-r--r--bcm2835.h1962logplain -rw-r--r--berlin2.h1034logplain -rw-r--r--berlin2q.h695logplain -rw-r--r--clps711x-clock.h718logplain -rw-r--r--efm32-cmu.h1112logplain -rw-r--r--exynos-audss-clk.h597logplain -rw-r--r--exynos3250.h9083logplain -rw-r--r--exynos4.h8284logplain -rw-r--r--exynos4415.h9828logplain -rw-r--r--exynos5250.h4616logplain -rw-r--r--exynos5260-clk.h14876logplain -rw-r--r--exynos5410.h1689logplain -rw-r--r--exynos5420.h6857logplain -rw-r--r--exynos5433.h45372logplain -rw-r--r--exynos5440.h1141logplain -rw-r--r--exynos7-clk.h5281logplain -rw-r--r--gxbb-aoclkc.h2866logplain -rw-r--r--gxbb-clkc.h592logplain -rw-r--r--hi3516cv300-clock.h1668logplain -rw-r--r--hi3519-clock.h1328logplain -rw-r--r--hi3620-clock.h4496logplain -rw-r--r--hi6220-clock.h4508logplain -rw-r--r--hip04-clock.h1137logplain -rw-r--r--histb-clock.h2012logplain -rw-r--r--hix5hd2-clock.h2415logplain -rw-r--r--imx1-clock.h1055logplain -rw-r--r--imx21-clock.h2461logplain -rw-r--r--imx27-clock.h3494logplain -rw-r--r--imx5-clock.h7212logplain -rw-r--r--imx6qdl-clock.h9593logplain -rw-r--r--imx6sl-clock.h5849logplain -rw-r--r--imx6sx-clock.h9099logplain -rw-r--r--imx6ul-clock.h8203logplain -rw-r--r--imx7d-clock.h15974logplain -rw-r--r--jz4740-cgu.h1028logplain -rw-r--r--jz4780-cgu.h2470logplain -rw-r--r--lpc18xx-ccu.h2134logplain -rw-r--r--lpc18xx-cgu.h1142logplain -rw-r--r--lpc32xx-clock.h1633logplain -rw-r--r--lsi,axm5516-clks.h974logplain -rw-r--r--marvell,mmp2.h2022logplain -rw-r--r--marvell,pxa168.h1654logplain -rw-r--r--marvell,pxa1928.h1535logplain -rw-r--r--marvell,pxa910.h1598logplain -rw-r--r--maxim,max77620.h632logplain -rw-r--r--maxim,max77686.h648logplain -rw-r--r--maxim,max77802.h630logplain -rw-r--r--meson8b-clkc.h523logplain -rw-r--r--microchip,pic32-clock.h1150logplain -rw-r--r--mpc512x-clock.h2236logplain -rw-r--r--mt2701-clk.h13832logplain -rw-r--r--mt8135-clk.h5641logplain -rw-r--r--mt8173-clk.h9293logplain -rw-r--r--oxsemi,ox810se.h1002logplain -rw-r--r--oxsemi,ox820.h1203logplain -rw-r--r--pistachio-clk.h4863logplain -rw-r--r--pxa-clock.h1715logplain -rw-r--r--qcom,gcc-apq8084.h12872logplain -rw-r--r--qcom,gcc-ipq4019.h5423logplain -rw-r--r--qcom,gcc-ipq806x.h8574logplain -rw-r--r--qcom,gcc-mdm9615.h9497logplain -rw-r--r--qcom,gcc-msm8660.h7932logplain -rw-r--r--qcom,gcc-msm8916.h6190logplain -rw-r--r--qcom,gcc-msm8960.h9342logplain -rw-r--r--qcom,gcc-msm8974.h12340logplain -rw-r--r--qcom,gcc-msm8994.h4858logplain -rw-r--r--qcom,gcc-msm8996.h12575logplain -rw-r--r--qcom,lcc-ipq806x.h899logplain -rw-r--r--qcom,lcc-mdm9615.h1701logplain -rw-r--r--qcom,lcc-msm8960.h1616logplain -rw-r--r--qcom,mmcc-apq8084.h5722logplain -rw-r--r--qcom,mmcc-msm8960.h4109logplain -rw-r--r--qcom,mmcc-msm8974.h5223logplain -rw-r--r--qcom,mmcc-msm8996.h9403logplain -rw-r--r--qcom,rpmcc.h2101logplain -rw-r--r--r7s72100-clock.h1218logplain -rw-r--r--r8a73a4-clock.h1596logplain -rw-r--r--r8a7740-clock.h1992logplain -rw-r--r--r8a7743-cpg-mssr.h1269logplain -rw-r--r--r8a7745-cpg-mssr.h1298logplain -rw-r--r--r8a7778-clock.h1855logplain -rw-r--r--r8a7779-clock.h1647logplain -rw-r--r--r8a7790-clock.h4367logplain -rw-r--r--r8a7791-clock.h4388logplain -rw-r--r--r8a7792-clock.h2562logplain -rw-r--r--r8a7793-clock.h4561logplain -rw-r--r--r8a7794-clock.h3679logplain -rw-r--r--r8a7795-cpg-mssr.h1890logplain -rw-r--r--r8a7796-cpg-mssr.h2066logplain -rw-r--r--renesas-cpg-mssr.h542logplain -rw-r--r--rk1108-cru.h6605logplain -rw-r--r--rk3036-cru.h4584logplain -rw-r--r--rk3066a-cru.h1068logplain -rw-r--r--rk3188-cru-common.h6105logplain -rw-r--r--rk3188-cru.h1435logplain