From a86d27693066a34a29be86f394bbad847b2d1749 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 7 May 2024 14:13:46 +0200 Subject: [PATCH 1/4] gpiolib: fix the speed of descriptor label setting with SRCU Commit 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU") caused a massive drop in performance of requesting GPIO lines due to the call to synchronize_srcu() on each label change. Rework the code to not wait until all read-only users are done with reading the label but instead atomically replace the label pointer and schedule its release after all read-only critical sections are done. To that end wrap the descriptor label in a struct that also contains the rcu_head struct required for deferring tasks using call_srcu() and stop using kstrdup_const() as we're required to allocate memory anyway. Just allocate enough for the label string and rcu_head in one go. Reported-by: Neil Armstrong Closes: https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@mail.gmail.com/ Fixes: 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU") Suggested-by: "Paul E. McKenney" Tested-by: Neil Armstrong # on SM8650-QRD Acked-by: "Paul E. McKenney" Link: https://lore.kernel.org/r/20240507121346.16969-1-brgl@bgdev.pl Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 31 ++++++++++++++++++++++++------- drivers/gpio/gpiolib.h | 7 ++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 94903fc1c145..2fa3756c9073 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -101,6 +101,7 @@ static bool gpiolib_initialized; const char *gpiod_get_label(struct gpio_desc *desc) { + struct gpio_desc_label *label; unsigned long flags; flags = READ_ONCE(desc->flags); @@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags)) return "interrupt"; - return test_bit(FLAG_REQUESTED, &flags) ? - srcu_dereference(desc->label, &desc->srcu) : NULL; + if (!test_bit(FLAG_REQUESTED, &flags)) + return NULL; + + label = srcu_dereference_check(desc->label, &desc->srcu, + srcu_read_lock_held(&desc->srcu)); + + return label->str; +} + +static void desc_free_label(struct rcu_head *rh) +{ + kfree(container_of(rh, struct gpio_desc_label, rh)); } static int desc_set_label(struct gpio_desc *desc, const char *label) { - const char *new = NULL, *old; + struct gpio_desc_label *new = NULL, *old; if (label) { - new = kstrdup_const(label, GFP_KERNEL); + new = kzalloc(struct_size(new, str, strlen(label) + 1), + GFP_KERNEL); if (!new) return -ENOMEM; + + strcpy(new->str, label); } old = rcu_replace_pointer(desc->label, new, 1); - synchronize_srcu(&desc->srcu); - kfree_const(old); + if (old) + call_srcu(&desc->srcu, &old->rh, desc_free_label); return 0; } @@ -697,8 +711,11 @@ static void gpiodev_release(struct device *dev) struct gpio_device *gdev = to_gpio_device(dev); unsigned int i; - for (i = 0; i < gdev->ngpio; i++) + for (i = 0; i < gdev->ngpio; i++) { + /* Free pending label. */ + synchronize_srcu(&gdev->descs[i].srcu); cleanup_srcu_struct(&gdev->descs[i].srcu); + } ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index f67d5991ab1c..69a353c789f0 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -137,6 +137,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory); void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); +struct gpio_desc_label { + struct rcu_head rh; + char str[]; +}; + /** * struct gpio_desc - Opaque descriptor for a GPIO * @@ -177,7 +182,7 @@ struct gpio_desc { #define FLAG_EVENT_CLOCK_HTE 19 /* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ - const char __rcu *label; + struct gpio_desc_label __rcu *label; /* Name of the GPIO */ const char *name; #ifdef CONFIG_OF_DYNAMIC From 7765ffed533d4a9f0291a0edc660496d104396ec Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 7 May 2024 19:24:14 +0200 Subject: [PATCH 2/4] gpiolib: use a single SRCU struct for all GPIO descriptors We used a per-descriptor SRCU struct in order to not impose a wait with synchronize_srcu() for descriptor X on read-only operations of descriptor Y. Now that we no longer call synchronize_srcu() on descriptor label change but only when releasing descriptor resources, we can use a single SRCU structure for all GPIO descriptors in a given chip. Suggested-by: "Paul E. McKenney" Acked-by: "Paul E. McKenney" Link: https://lore.kernel.org/r/20240507172414.28513-1-brgl@bgdev.pl Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-cdev.c | 2 +- drivers/gpio/gpiolib.c | 41 +++++++++++++++++-------------------- drivers/gpio/gpiolib.h | 10 ++++----- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index d09c7d728365..fea149ae7774 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2351,7 +2351,7 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, dflags = READ_ONCE(desc->flags); - scoped_guard(srcu, &desc->srcu) { + scoped_guard(srcu, &desc->gdev->desc_srcu) { label = gpiod_get_label(desc); if (label && test_bit(FLAG_REQUESTED, &dflags)) strscpy(info->consumer, label, diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 2fa3756c9073..fa50db0c3605 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -112,8 +112,8 @@ const char *gpiod_get_label(struct gpio_desc *desc) if (!test_bit(FLAG_REQUESTED, &flags)) return NULL; - label = srcu_dereference_check(desc->label, &desc->srcu, - srcu_read_lock_held(&desc->srcu)); + label = srcu_dereference_check(desc->label, &desc->gdev->desc_srcu, + srcu_read_lock_held(&desc->gdev->desc_srcu)); return label->str; } @@ -138,7 +138,7 @@ static int desc_set_label(struct gpio_desc *desc, const char *label) old = rcu_replace_pointer(desc->label, new, 1); if (old) - call_srcu(&desc->srcu, &old->rh, desc_free_label); + call_srcu(&desc->gdev->desc_srcu, &old->rh, desc_free_label); return 0; } @@ -709,13 +709,10 @@ EXPORT_SYMBOL_GPL(gpiochip_line_is_valid); static void gpiodev_release(struct device *dev) { struct gpio_device *gdev = to_gpio_device(dev); - unsigned int i; - for (i = 0; i < gdev->ngpio; i++) { - /* Free pending label. */ - synchronize_srcu(&gdev->descs[i].srcu); - cleanup_srcu_struct(&gdev->descs[i].srcu); - } + /* Call pending kfree()s for descriptor labels. */ + synchronize_srcu(&gdev->desc_srcu); + cleanup_srcu_struct(&gdev->desc_srcu); ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); @@ -992,6 +989,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_remove_from_list; + ret = init_srcu_struct(&gdev->desc_srcu); + if (ret) + goto err_cleanup_gdev_srcu; + #ifdef CONFIG_PINCTRL INIT_LIST_HEAD(&gdev->pin_ranges); #endif @@ -999,23 +1000,19 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (gc->names) { ret = gpiochip_set_desc_names(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; } ret = gpiochip_set_names(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; ret = gpiochip_init_valid_mask(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; for (desc_index = 0; desc_index < gc->ngpio; desc_index++) { struct gpio_desc *desc = &gdev->descs[desc_index]; - ret = init_srcu_struct(&desc->srcu); - if (ret) - goto err_cleanup_desc_srcu; - if (gc->get_direction && gpiochip_line_is_valid(gc, desc_index)) { assign_bit(FLAG_IS_OUT, &desc->flags, !gc->get_direction(gc, desc_index)); @@ -1027,7 +1024,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = of_gpiochip_add(gc); if (ret) - goto err_cleanup_desc_srcu; + goto err_free_valid_mask; ret = gpiochip_add_pin_ranges(gc); if (ret) @@ -1074,10 +1071,10 @@ err_free_hogs: gpiochip_remove_pin_ranges(gc); err_remove_of_chip: of_gpiochip_remove(gc); -err_cleanup_desc_srcu: - while (desc_index--) - cleanup_srcu_struct(&gdev->descs[desc_index].srcu); +err_free_valid_mask: gpiochip_free_valid_mask(gc); +err_cleanup_desc_srcu: + cleanup_srcu_struct(&gdev->desc_srcu); err_cleanup_gdev_srcu: cleanup_srcu_struct(&gdev->srcu); err_remove_from_list: @@ -2407,7 +2404,7 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) if (!test_bit(FLAG_REQUESTED, &desc->flags)) return NULL; - guard(srcu)(&desc->srcu); + guard(srcu)(&desc->gdev->desc_srcu); label = kstrdup(gpiod_get_label(desc), GFP_KERNEL); if (!label) @@ -4798,7 +4795,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) } for_each_gpio_desc(gc, desc) { - guard(srcu)(&desc->srcu); + guard(srcu)(&desc->gdev->desc_srcu); if (test_bit(FLAG_REQUESTED, &desc->flags)) { gpiod_get_direction(desc); is_out = test_bit(FLAG_IS_OUT, &desc->flags); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 69a353c789f0..8e0e211ebf08 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -31,6 +31,7 @@ * @chip: pointer to the corresponding gpiochip, holding static * data for this device * @descs: array of ngpio descriptors. + * @desc_srcu: ensures consistent state of GPIO descriptors exposed to users * @ngpio: the number of GPIO lines on this GPIO device, equal to the size * of the @descs array. * @can_sleep: indicate whether the GPIO chip driver's callbacks can sleep @@ -61,6 +62,7 @@ struct gpio_device { struct module *owner; struct gpio_chip __rcu *chip; struct gpio_desc *descs; + struct srcu_struct desc_srcu; int base; u16 ngpio; bool can_sleep; @@ -150,7 +152,6 @@ struct gpio_desc_label { * @label: Name of the consumer * @name: Line name * @hog: Pointer to the device node that hogs this line (if any) - * @srcu: SRCU struct protecting the label pointer. * * These are obtained using gpiod_get() and are preferable to the old * integer-based handles. @@ -188,7 +189,6 @@ struct gpio_desc { #ifdef CONFIG_OF_DYNAMIC struct device_node *hog; #endif - struct srcu_struct srcu; }; #define gpiod_not_found(desc) (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) @@ -256,7 +256,7 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) #define gpiod_err(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ @@ -264,7 +264,7 @@ do { \ #define gpiod_warn(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ @@ -272,7 +272,7 @@ do { \ #define gpiod_dbg(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ From 02f6b0e1ec7e0e7d059dddc893645816552039da Mon Sep 17 00:00:00 2001 From: Zhongqiu Han Date: Sun, 5 May 2024 22:11:56 +0800 Subject: [PATCH 3/4] gpiolib: cdev: Fix use after free in lineinfo_changed_notify The use-after-free issue occurs as follows: when the GPIO chip device file is being closed by invoking gpio_chrdev_release(), watched_lines is freed by bitmap_free(), but the unregistration of lineinfo_changed_nb notifier chain failed due to waiting write rwsem. Additionally, one of the GPIO chip's lines is also in the release process and holds the notifier chain's read rwsem. Consequently, a race condition leads to the use-after-free of watched_lines. Here is the typical stack when issue happened: [free] gpio_chrdev_release() --> bitmap_free(cdev->watched_lines) <-- freed --> blocking_notifier_chain_unregister() --> down_write(&nh->rwsem) <-- waiting rwsem --> __down_write_common() --> rwsem_down_write_slowpath() --> schedule_preempt_disabled() --> schedule() [use] st54spi_gpio_dev_release() --> gpio_free() --> gpiod_free() --> gpiod_free_commit() --> gpiod_line_state_notify() --> blocking_notifier_call_chain() --> down_read(&nh->rwsem); <-- held rwsem --> notifier_call_chain() --> lineinfo_changed_notify() --> test_bit(xxxx, cdev->watched_lines) <-- use after free The side effect of the use-after-free issue is that a GPIO line event is being generated for userspace where it shouldn't. However, since the chrdev is being closed, userspace won't have the chance to read that event anyway. To fix the issue, call the bitmap_free() function after the unregistration of lineinfo_changed_nb notifier chain. Fixes: 51c1064e82e7 ("gpiolib: add new ioctl() for monitoring changes in line info") Signed-off-by: Zhongqiu Han Link: https://lore.kernel.org/r/20240505141156.2944912-1-quic_zhonhan@quicinc.com Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-cdev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index fea149ae7774..46a45093d4d0 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2799,11 +2799,11 @@ static int gpio_chrdev_release(struct inode *inode, struct file *file) struct gpio_chardev_data *cdev = file->private_data; struct gpio_device *gdev = cdev->gdev; - bitmap_free(cdev->watched_lines); blocking_notifier_chain_unregister(&gdev->device_notifier, &cdev->device_unregistered_nb); blocking_notifier_chain_unregister(&gdev->line_state_notifier, &cdev->lineinfo_changed_nb); + bitmap_free(cdev->watched_lines); gpio_device_put(gdev); kfree(cdev); From ee0166b637a5e376118e9659e5b4148080f1d27e Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Fri, 10 May 2024 14:53:42 +0800 Subject: [PATCH 4/4] gpiolib: cdev: fix uninitialised kfifo If a line is requested with debounce, and that results in debouncing in software, and the line is subsequently reconfigured to enable edge detection then the allocation of the kfifo to contain edge events is overlooked. This results in events being written to and read from an uninitialised kfifo. Read events are returned to userspace. Initialise the kfifo in the case where the software debounce is already active. Fixes: 65cff7046406 ("gpiolib: cdev: support setting debounce") Signed-off-by: Kent Gibson Link: https://lore.kernel.org/r/20240510065342.36191-1-warthog618@gmail.com Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-cdev.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 46a45093d4d0..9dad67ea2597 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -1193,6 +1193,8 @@ static int edge_detector_update(struct line *line, struct gpio_v2_line_config *lc, unsigned int line_idx, u64 edflags) { + u64 eflags; + int ret; u64 active_edflags = READ_ONCE(line->edflags); unsigned int debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx); @@ -1204,6 +1206,18 @@ static int edge_detector_update(struct line *line, /* sw debounced and still will be...*/ if (debounce_period_us && READ_ONCE(line->sw_debounced)) { line_set_debounce_period(line, debounce_period_us); + /* + * ensure event fifo is initialised if edge detection + * is now enabled. + */ + eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS; + if (eflags && !kfifo_initialized(&line->req->events)) { + ret = kfifo_alloc(&line->req->events, + line->req->event_buffer_size, + GFP_KERNEL); + if (ret) + return ret; + } return 0; }