percpu-refcount: fix reference leak during percpu-atomic transition
percpu_ref_tryget() and percpu_ref_tryget_live() should return
"true" IFF they acquire a reference. But the return value from
atomic_long_inc_not_zero() is a long and may have high bits set,
e.g. PERCPU_COUNT_BIAS, and the return value of the tryget routines
is bool so the reference may actually be acquired but the routines
return "false" which results in a reference leak since the caller
assumes it does not need to do a corresponding percpu_ref_put().
This was seen when performing CPU hotplug during I/O, as hangs in
blk_mq_freeze_queue_wait where percpu_ref_kill (blk_mq_freeze_queue_start)
raced with percpu_ref_tryget (blk_mq_timeout_work).
Sample stack trace:
__switch_to+0x2c0/0x450
__schedule+0x2f8/0x970
schedule+0x48/0xc0
blk_mq_freeze_queue_wait+0x94/0x120
blk_mq_queue_reinit_work+0xb8/0x180
blk_mq_queue_reinit_prepare+0x84/0xa0
cpuhp_invoke_callback+0x17c/0x600
cpuhp_up_callbacks+0x58/0x150
_cpu_up+0xf0/0x1c0
do_cpu_up+0x120/0x150
cpu_subsys_online+0x64/0xe0
device_online+0xb4/0x120
online_store+0xb4/0xc0
dev_attr_store+0x68/0xa0
sysfs_kf_write+0x80/0xb0
kernfs_fop_write+0x17c/0x250
__vfs_write+0x6c/0x1e0
vfs_write+0xd0/0x270
SyS_write+0x6c/0x110
system_call+0x38/0xe0
Examination of the queue showed a single reference (no PERCPU_COUNT_BIAS,
and __PERCPU_REF_DEAD, __PERCPU_REF_ATOMIC set) and no requests.
However, conditions at the time of the race are count of PERCPU_COUNT_BIAS + 0
and __PERCPU_REF_DEAD and __PERCPU_REF_ATOMIC set.
The fix is to make the tryget routines use an actual boolean internally instead
of the atomic long result truncated to a int.
Fixes: e625305b3907 percpu-refcount: make percpu_ref based on longs instead of ints
Link: https://bugzilla.kernel.org/show_bug.cgi?id=190751
Signed-off-by: Douglas Miller <dougmill@linux.vnet.ibm.com>
Reviewed-by: Jens Axboe <axboe@fb.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Fixes: e625305b3907 ("percpu-refcount: make percpu_ref based on longs instead of ints")
Cc: stable@vger.kernel.org # v3.18+
in
| -rw-r--r-- | bcm2835.h | 1962 | logplain |
| -rw-r--r-- | berlin2.h | 1034 | logplain |
| -rw-r--r-- | berlin2q.h | 695 | logplain |
| -rw-r--r-- | clps711x-clock.h | 718 | logplain |
| -rw-r--r-- | efm32-cmu.h | 1112 | logplain |
| -rw-r--r-- | exynos-audss-clk.h | 597 | logplain |
| -rw-r--r-- | exynos3250.h | 9083 | logplain |
| -rw-r--r-- | exynos4.h | 8284 | logplain |
| -rw-r--r-- | exynos4415.h | 9828 | logplain |
| -rw-r--r-- | exynos5250.h | 4616 | logplain |
| -rw-r--r-- | exynos5260-clk.h | 14876 | logplain |
| -rw-r--r-- | exynos5410.h | 1689 | logplain |
| -rw-r--r-- | exynos5420.h | 6857 | logplain |
| -rw-r--r-- | exynos5433.h | 45372 | logplain |
| -rw-r--r-- | exynos5440.h | 1141 | logplain |
| -rw-r--r-- | exynos7-clk.h | 5281 | logplain |
| -rw-r--r-- | gxbb-aoclkc.h | 2866 | logplain |
| -rw-r--r-- | gxbb-clkc.h | 592 | logplain |
| -rw-r--r-- | hi3516cv300-clock.h | 1668 | logplain |
| -rw-r--r-- | hi3519-clock.h | 1328 | logplain |
| -rw-r--r-- | hi3620-clock.h | 4496 | logplain |
| -rw-r--r-- | hi6220-clock.h | 4508 | logplain |
| -rw-r--r-- | hip04-clock.h | 1137 | logplain |
| -rw-r--r-- | histb-clock.h | 2012 | logplain |
| -rw-r--r-- | hix5hd2-clock.h | 2415 | logplain |
| -rw-r--r-- | imx1-clock.h | 1055 | logplain |
| -rw-r--r-- | imx21-clock.h | 2461 | logplain |
| -rw-r--r-- | imx27-clock.h | 3494 | logplain |
| -rw-r--r-- | imx5-clock.h | 7212 | logplain |
| -rw-r--r-- | imx6qdl-clock.h | 9593 | logplain |
| -rw-r--r-- | imx6sl-clock.h | 5849 | logplain |
| -rw-r--r-- | imx6sx-clock.h | 9099 | logplain |
| -rw-r--r-- | imx6ul-clock.h | 8203 | logplain |
| -rw-r--r-- | imx7d-clock.h | 15974 | logplain |
| -rw-r--r-- | jz4740-cgu.h | 1028 | logplain |
| -rw-r--r-- | jz4780-cgu.h | 2470 | logplain |
| -rw-r--r-- | lpc18xx-ccu.h | 2134 | logplain |
| -rw-r--r-- | lpc18xx-cgu.h | 1142 | logplain |
| -rw-r--r-- | lpc32xx-clock.h | 1633 | logplain |
| -rw-r--r-- | lsi,axm5516-clks.h | 974 | logplain |
| -rw-r--r-- | marvell,mmp2.h | 2022 | logplain |
| -rw-r--r-- | marvell,pxa168.h | 1654 | logplain |
| -rw-r--r-- | marvell,pxa1928.h | 1535 | logplain |
| -rw-r--r-- | marvell,pxa910.h | 1598 | logplain |
| -rw-r--r-- | maxim,max77620.h | 632 | logplain |
| -rw-r--r-- | maxim,max77686.h | 648 | logplain |
| -rw-r--r-- | maxim,max77802.h | 630 | logplain |
| -rw-r--r-- | meson8b-clkc.h | 523 | logplain |
| -rw-r--r-- | microchip,pic32-clock.h | 1150 | logplain |
| -rw-r--r-- | mpc512x-clock.h | 2236 | logplain |
| -rw-r--r-- | mt2701-clk.h | 13832 | logplain |
| -rw-r--r-- | mt8135-clk.h | 5641 | logplain |
| -rw-r--r-- | mt8173-clk.h | 9293 | logplain |
| -rw-r--r-- | oxsemi,ox810se.h | 1002 | logplain |
| -rw-r--r-- | oxsemi,ox820.h | 1203 | logplain |
| -rw-r--r-- | pistachio-clk.h | 4863 | logplain |
| -rw-r--r-- | pxa-clock.h | 1715 | logplain |
| -rw-r--r-- | qcom,gcc-apq8084.h | 12872 | logplain |
| -rw-r--r-- | qcom,gcc-ipq4019.h | 5423 | logplain |
| -rw-r--r-- | qcom,gcc-ipq806x.h | 8574 | logplain |
| -rw-r--r-- | qcom,gcc-mdm9615.h | 9497 | logplain |
| -rw-r--r-- | qcom,gcc-msm8660.h | 7932 | logplain |
| -rw-r--r-- | qcom,gcc-msm8916.h | 6190 | logplain |
| -rw-r--r-- | qcom,gcc-msm8960.h | 9342 | logplain |
| -rw-r--r-- | qcom,gcc-msm8974.h | 12340 | logplain |
| -rw-r--r-- | qcom,gcc-msm8994.h | 4858 | logplain |
| -rw-r--r-- | qcom,gcc-msm8996.h | 12575 | logplain |
| -rw-r--r-- | qcom,lcc-ipq806x.h | 899 | logplain |
| -rw-r--r-- | qcom,lcc-mdm9615.h | 1701 | logplain |
| -rw-r--r-- | qcom,lcc-msm8960.h | 1616 | logplain |
| -rw-r--r-- | qcom,mmcc-apq8084.h | 5722 | logplain |
| -rw-r--r-- | qcom,mmcc-msm8960.h | 4109 | logplain |
| -rw-r--r-- | qcom,mmcc-msm8974.h | 5223 | logplain |
| -rw-r--r-- | qcom,mmcc-msm8996.h | 9403 | logplain |
| -rw-r--r-- | qcom,rpmcc.h | 2101 | logplain |
| -rw-r--r-- | r7s72100-clock.h | 1218 | logplain |
| -rw-r--r-- | r8a73a4-clock.h | 1596 | logplain |
| -rw-r--r-- | r8a7740-clock.h | 1992 | logplain |
| -rw-r--r-- | r8a7743-cpg-mssr.h | 1269 | logplain |
| -rw-r--r-- | r8a7745-cpg-mssr.h | 1298 | logplain |
| -rw-r--r-- | r8a7778-clock.h | 1855 | logplain |
| -rw-r--r-- | r8a7779-clock.h | 1647 | logplain |
| -rw-r--r-- | r8a7790-clock.h | 4367 | logplain |
| -rw-r--r-- | r8a7791-clock.h | 4388 | logplain |
| -rw-r--r-- | r8a7792-clock.h | 2562 | logplain |
| -rw-r--r-- | r8a7793-clock.h | 4561 | logplain |
| -rw-r--r-- | r8a7794-clock.h | 3679 | logplain |
| -rw-r--r-- | r8a7795-cpg-mssr.h | 1890 | logplain |
| -rw-r--r-- | r8a7796-cpg-mssr.h | 2066 | logplain |
| -rw-r--r-- | renesas-cpg-mssr.h | 542 | logplain |
| -rw-r--r-- | rk1108-cru.h | 6605 | logplain |
| -rw-r--r-- | rk3036-cru.h | 4584 | logplain |
| -rw-r--r-- | rk3066a-cru.h | 1068 | logplain |
| -rw-r--r-- | rk3188-cru-common.h | 6105 | logplain |
| -rw-r--r-- | rk3188-cru.h | 1435 | logplain |