diff options
| author | Mark Brown <broonie@kernel.org> | 2026-05-29 23:00:38 +0100 |
|---|---|---|
| committer | Mark Brown <broonie@kernel.org> | 2026-05-29 23:00:38 +0100 |
| commit | 1aef57bf376607540d77304cc65b44996746cbaf (patch) | |
| tree | 23cc26befc9e65aa47eade8207f136a4942f75d5 | |
| parent | 5438eb458830693c5f8481aa36097245a727e2c2 (diff) | |
| parent | fa09f08ede3db3050ae16ae1ed92c902d0cada23 (diff) | |
| download | linux-next-history-1aef57bf376607540d77304cc65b44996746cbaf.tar.gz | |
Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-catu.c | 2 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-core.c | 574 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-cti-core.c | 9 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-cti-platform.c | 1 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-etb10.c | 6 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-etm-perf.c | 287 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-etm3x-core.c | 73 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-etm4x-core.c | 214 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-platform.c | 12 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-priv.h | 8 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-syscfg.c | 38 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-syscfg.h | 2 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-sysfs.c | 135 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-tmc-etr.c | 4 | ||||
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-trbe.c | 61 | ||||
| -rw-r--r-- | include/linux/coresight.h | 27 | ||||
| -rw-r--r-- | include/linux/cpuhotplug.h | 2 |
17 files changed, 936 insertions, 519 deletions
diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index ce71dcddfca25..43abe13995cf3 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -514,7 +514,7 @@ static int __catu_probe(struct device *dev, struct resource *res) int ret = 0; u32 dma_mask; struct catu_drvdata *drvdata; - struct coresight_desc catu_desc; + struct coresight_desc catu_desc = { 0 }; struct coresight_platform_data *pdata = NULL; void __iomem *base; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 46f247f73cf64..f7b1308a759c3 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -6,6 +6,7 @@ #include <linux/acpi.h> #include <linux/bitfield.h> #include <linux/build_bug.h> +#include <linux/cpu_pm.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/types.h> @@ -35,6 +36,10 @@ DEFINE_MUTEX(coresight_mutex); static DEFINE_PER_CPU(struct coresight_device *, csdev_sink); +static DEFINE_RAW_SPINLOCK(coresight_dev_lock); +static DEFINE_PER_CPU(struct coresight_device *, csdev_source); +static DEFINE_PER_CPU(bool, percpu_pm_failed); + /** * struct coresight_node - elements of a path, from source to sink * @csdev: Address of an element. @@ -58,6 +63,24 @@ static LIST_HEAD(coresight_dev_idx_list); static const struct cti_assoc_op *cti_assoc_ops; +static struct coresight_node * +coresight_path_first_node(struct coresight_path *path) +{ + if (list_empty(&path->path_list)) + return NULL; + + return list_first_entry(&path->path_list, struct coresight_node, link); +} + +static struct coresight_node * +coresight_path_last_node(struct coresight_path *path) +{ + if (list_empty(&path->path_list)) + return NULL; + + return list_last_entry(&path->path_list, struct coresight_node, link); +} + void coresight_set_cti_ops(const struct cti_assoc_op *cti_op) { cti_assoc_ops = cti_op; @@ -82,14 +105,89 @@ struct coresight_device *coresight_get_percpu_sink(int cpu) } EXPORT_SYMBOL_GPL(coresight_get_percpu_sink); -static struct coresight_device *coresight_get_source(struct coresight_path *path) +static void coresight_set_percpu_source(struct coresight_device *csdev) +{ + if (!csdev || !coresight_is_percpu_source(csdev)) + return; + + guard(raw_spinlock_irqsave)(&coresight_dev_lock); + + /* Expect no device to be set yet */ + WARN_ON(per_cpu(csdev_source, csdev->cpu)); + per_cpu(csdev_source, csdev->cpu) = csdev; +} + +static void coresight_clear_percpu_source(struct coresight_device *csdev) +{ + if (!csdev || !coresight_is_percpu_source(csdev)) + return; + + /* Clear percpu_pm_failed */ + per_cpu(percpu_pm_failed, csdev->cpu) = false; + + guard(raw_spinlock_irqsave)(&coresight_dev_lock); + + /* The per-CPU pointer should contain the same csdev */ + WARN_ON(per_cpu(csdev_source, csdev->cpu) != csdev); + per_cpu(csdev_source, csdev->cpu) = NULL; +} + +struct coresight_device *coresight_get_percpu_source_ref(int cpu) +{ + struct coresight_device *csdev; + + if (WARN_ON(cpu < 0)) + return NULL; + + guard(raw_spinlock_irqsave)(&coresight_dev_lock); + + csdev = per_cpu(csdev_source, cpu); + if (!csdev) + return NULL; + + /* + * Holding a reference to the csdev->dev ensures that the + * coresight_device is live for the caller. The path building + * logic can safely either build a path to the sink or fail + * if the device is being unregistered (if there was a race). + * The caller can skip the "source" device, if no path could + * be built. + */ + get_device(&csdev->dev); + + return csdev; +} + +void coresight_put_percpu_source_ref(struct coresight_device *csdev) +{ + if (!csdev || !coresight_is_percpu_source(csdev)) + return; + + guard(raw_spinlock_irqsave)(&coresight_dev_lock); + + /* + * TODO: coresight_device_release() is invoked to release resources when + * the device's refcount reaches zero. It then calls free_percpu(), + * which acquires pcpu_lock — a sleepable lock when PREEMPT_RT is + * enabled. Since the raw spinlock coresight_dev_lock is held, this can + * lead to a potential "scheduling while atomic" issue. + */ + put_device(&csdev->dev); +} + +struct coresight_device *coresight_get_source(struct coresight_path *path) { struct coresight_device *csdev; + struct coresight_node *nd; if (!path) return NULL; - csdev = list_first_entry(&path->path_list, struct coresight_node, link)->csdev; + nd = coresight_path_first_node(path); + if (!nd) + return NULL; + + csdev = nd->csdev; if (!coresight_is_device_source(csdev)) return NULL; @@ -385,19 +483,47 @@ static void coresight_disable_helpers(struct coresight_device *csdev, } /* - * Helper function to call source_ops(csdev)->disable and also disable the - * helpers. - * - * There is an imbalance between coresight_enable_path() and - * coresight_disable_path(). Enabling also enables the source's helpers as part - * of the path, but disabling always skips the first item in the path (which is - * the source), so sources and their helpers don't get disabled as part of that - * function and we need the extra step here. + * coresight_enable_source() and coresight_disable_source() only enable and + * disable the source, but do nothing for the associated helpers, which are + * controlled as part of the path. */ +int coresight_enable_source(struct coresight_device *csdev, + struct perf_event *event, enum cs_mode mode, + struct coresight_path *path) +{ + int ret; + + if (!coresight_is_device_source(csdev)) + return -EINVAL; + + ret = source_ops(csdev)->enable(csdev, event, mode, path); + if (ret) + return ret; + + /* + * Update the path pointer until after the source is enabled to avoid + * races where multiple paths attempt to enable the same source. + * + * Do not set the path pointer here for per-CPU sources; set it locally + * on the CPU instead. Otherwise, there is a window where the path is + * enabled but the pointer is not yet set, causing CPU PM notifiers to + * miss PM operations due to reading a NULL pointer. + */ + if (!coresight_is_percpu_source(csdev)) + csdev->path = path; + + return 0; +} + void coresight_disable_source(struct coresight_device *csdev, void *data) { + if (!coresight_is_device_source(csdev)) + return; + + if (!coresight_is_percpu_source(csdev)) + csdev->path = NULL; + source_ops(csdev)->disable(csdev, data); - coresight_disable_helpers(csdev, NULL); } EXPORT_SYMBOL_GPL(coresight_disable_source); @@ -424,20 +550,42 @@ int coresight_resume_source(struct coresight_device *csdev) EXPORT_SYMBOL_GPL(coresight_resume_source); /* - * coresight_disable_path_from : Disable components in the given path beyond - * @nd in the list. If @nd is NULL, all the components, except the SOURCE are - * disabled. + * Callers must fetch nodes from the path and pass @from and @to to the path + * enable/disable functions. Walk the path from @from to locate @to. If @to + * is found, it indicates @from and @to are in order. Otherwise, they are out + * of order. */ -static void coresight_disable_path_from(struct coresight_path *path, - struct coresight_node *nd) +static bool coresight_path_nodes_in_order(struct coresight_path *path, + struct coresight_node *from, + struct coresight_node *to) +{ + struct coresight_node *nd; + + if (WARN_ON_ONCE(!from || !to)) + return false; + + nd = from; + list_for_each_entry_from(nd, &path->path_list, link) { + if (nd == to) + return true; + } + + return false; +} + +static void coresight_disable_path_from_to(struct coresight_path *path, + struct coresight_node *from, + struct coresight_node *to) { u32 type; struct coresight_device *csdev, *parent, *child; + struct coresight_node *nd; - if (!nd) - nd = list_first_entry(&path->path_list, struct coresight_node, link); + if (!coresight_path_nodes_in_order(path, from, to)) + return; - list_for_each_entry_continue(nd, &path->path_list, link) { + nd = from; + list_for_each_entry_from(nd, &path->path_list, link) { csdev = nd->csdev; type = csdev->type; @@ -457,12 +605,6 @@ static void coresight_disable_path_from(struct coresight_path *path, coresight_disable_sink(csdev); break; case CORESIGHT_DEV_TYPE_SOURCE: - /* - * We skip the first node in the path assuming that it - * is the source. So we don't expect a source device in - * the middle of a path. - */ - WARN_ON(1); break; case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; @@ -476,12 +618,18 @@ static void coresight_disable_path_from(struct coresight_path *path, /* Disable all helpers adjacent along the path last */ coresight_disable_helpers(csdev, path); + + /* Iterate up to and including @to */ + if (nd == to) + break; } } void coresight_disable_path(struct coresight_path *path) { - coresight_disable_path_from(path, NULL); + coresight_disable_path_from_to(path, + coresight_path_first_node(path), + coresight_path_last_node(path)); } EXPORT_SYMBOL_GPL(coresight_disable_path); @@ -499,22 +647,36 @@ static int coresight_enable_helpers(struct coresight_device *csdev, ret = coresight_enable_helper(helper, mode, path); if (ret) - return ret; + goto err; } return 0; + +err: + while (i--) { + helper = csdev->pdata->out_conns[i]->dest_dev; + if (helper && coresight_is_helper(helper)) + coresight_disable_helper(helper, path); + } + + return ret; } -int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) +static int coresight_enable_path_from_to(struct coresight_path *path, + enum cs_mode mode, + struct coresight_node *from, + struct coresight_node *to) { int ret = 0; u32 type; struct coresight_node *nd; struct coresight_device *csdev, *parent, *child; - struct coresight_device *source; - source = coresight_get_source(path); - list_for_each_entry_reverse(nd, &path->path_list, link) { + if (!coresight_path_nodes_in_order(path, from, to)) + return -EINVAL; + + nd = to; + list_for_each_entry_from_reverse(nd, &path->path_list, link) { csdev = nd->csdev; type = csdev->type; @@ -553,7 +715,8 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; child = list_next_entry(nd, link)->csdev; - ret = coresight_enable_link(csdev, parent, child, source); + ret = coresight_enable_link(csdev, parent, child, + coresight_get_source(path)); if (ret) goto err_disable_helpers; break; @@ -561,6 +724,10 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) ret = -EINVAL; goto err_disable_helpers; } + + /* Iterate down to and including @from */ + if (nd == from) + break; } out: @@ -568,18 +735,36 @@ out: err_disable_helpers: coresight_disable_helpers(csdev, path); err_disable_path: - coresight_disable_path_from(path, nd); + /* No device is actually enabled */ + if (nd == to) + goto out; + + /* Fetch the previous node, the last successfully enabled one */ + nd = list_next_entry(nd, link); + coresight_disable_path_from_to(path, nd, to); goto out; } +int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) +{ + return coresight_enable_path_from_to(path, mode, + coresight_path_first_node(path), + coresight_path_last_node(path)); +} + struct coresight_device *coresight_get_sink(struct coresight_path *path) { struct coresight_device *csdev; + struct coresight_node *nd; if (!path) return NULL; - csdev = list_last_entry(&path->path_list, struct coresight_node, link)->csdev; + nd = coresight_path_last_node(path); + if (!nd) + return NULL; + + csdev = nd->csdev; if (csdev->type != CORESIGHT_DEV_TYPE_SINK && csdev->type != CORESIGHT_DEV_TYPE_LINKSINK) return NULL; @@ -642,15 +827,29 @@ struct coresight_device *coresight_get_sink_by_id(u32 id) */ static bool coresight_get_ref(struct coresight_device *csdev) { - struct device *dev = csdev->dev.parent; + struct device *dev = &csdev->dev; + struct device *parent = csdev->dev.parent; + struct device_driver *drv; - /* Make sure the driver can't be removed */ - if (!try_module_get(dev->driver->owner)) - return false; - /* Make sure the device can't go away */ + /* Make sure csdev can't go away */ get_device(dev); - pm_runtime_get_sync(dev); + + /* Make sure parent device can't go away */ + get_device(parent); + + /* Make sure the driver can't be removed */ + drv = parent->driver; + if (!drv || !try_module_get(drv->owner)) + goto err_module; + + /* Make sure the device is powered on */ + pm_runtime_get_sync(parent); return true; + +err_module: + put_device(parent); + put_device(dev); + return false; } /** @@ -661,11 +860,15 @@ static bool coresight_get_ref(struct coresight_device *csdev) */ static void coresight_put_ref(struct coresight_device *csdev) { - struct device *dev = csdev->dev.parent; + struct device *dev = &csdev->dev; + struct device *parent = csdev->dev.parent; + struct device_driver *drv = parent->driver; - pm_runtime_put(dev); + pm_runtime_put(parent); + if (drv) + module_put(drv->owner); + put_device(parent); put_device(dev); - module_put(dev->driver->owner); } /* @@ -739,8 +942,8 @@ static int coresight_get_trace_id(struct coresight_device *csdev, * Call this after creating the path and before enabling it. This leaves * the trace ID set on the path, or it remains 0 if it couldn't be assigned. */ -void coresight_path_assign_trace_id(struct coresight_path *path, - enum cs_mode mode) +int coresight_path_assign_trace_id(struct coresight_path *path, + enum cs_mode mode) { struct coresight_device *sink = coresight_get_sink(path); struct coresight_node *nd; @@ -750,15 +953,18 @@ void coresight_path_assign_trace_id(struct coresight_path *path, /* Assign a trace ID to the path for the first device that wants to do it */ trace_id = coresight_get_trace_id(nd->csdev, mode, sink); - /* - * 0 in this context is that it didn't want to assign so keep searching. - * Non 0 is either success or fail. - */ - if (trace_id != 0) { - path->trace_id = trace_id; - return; - } + /* 0 means the device has no ID assignment, so keep searching */ + if (trace_id == 0) + continue; + + if (!IS_VALID_CS_TRACE_ID(trace_id)) + return -EINVAL; + + path->trace_id = trace_id; + return 0; } + + return -EINVAL; } /** @@ -787,7 +993,7 @@ static int _coresight_build_path(struct coresight_device *csdev, goto out; if (coresight_is_percpu_source(csdev) && coresight_is_percpu_sink(sink) && - sink == per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev))) { + sink == per_cpu(csdev_sink, csdev->cpu)) { if (_coresight_build_path(sink, source, sink, path) == 0) { found = true; goto out; @@ -1014,7 +1220,7 @@ coresight_find_default_sink(struct coresight_device *csdev) /* look for a default sink if we have not found for this device */ if (!csdev->def_sink) { if (coresight_is_percpu_source(csdev)) - csdev->def_sink = per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev)); + csdev->def_sink = per_cpu(csdev_sink, csdev->cpu); if (!csdev->def_sink) csdev->def_sink = coresight_find_sink(csdev, &depth); } @@ -1322,32 +1528,56 @@ void coresight_release_platform_data(struct device *dev, devm_kfree(dev, pdata); } -struct coresight_device *coresight_register(struct coresight_desc *desc) +static struct coresight_device * +coresight_init_device(struct coresight_desc *desc) { - int ret; struct coresight_device *csdev; - bool registered = false; csdev = kzalloc_obj(*csdev); - if (!csdev) { - ret = -ENOMEM; - goto err_out; - } + if (!csdev) + return ERR_PTR(-ENOMEM); csdev->pdata = desc->pdata; - csdev->type = desc->type; csdev->subtype = desc->subtype; csdev->ops = desc->ops; csdev->access = desc->access; csdev->orphan = true; + if (desc->flags & CORESIGHT_DESC_CPU_BOUND) { + csdev->cpu = desc->cpu; + } else { + /* A per-CPU source or sink must set CPU_BOUND flag */ + if (coresight_is_percpu_source(csdev) || + coresight_is_percpu_sink(csdev)) { + kfree(csdev); + return ERR_PTR(-EINVAL); + } + + csdev->cpu = -1; + } + csdev->dev.type = &coresight_dev_type[desc->type]; csdev->dev.groups = desc->groups; csdev->dev.parent = desc->dev; csdev->dev.release = coresight_device_release; csdev->dev.bus = &coresight_bustype; + return csdev; +} + +struct coresight_device *coresight_register(struct coresight_desc *desc) +{ + int ret; + struct coresight_device *csdev; + bool registered = false; + + csdev = coresight_init_device(desc); + if (IS_ERR(csdev)) { + ret = PTR_ERR(csdev); + goto err_out; + } + if (csdev->type == CORESIGHT_DEV_TYPE_SINK || csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) { raw_spin_lock_init(&csdev->perf_sink_id_map.lock); @@ -1398,6 +1628,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) if (ret) goto out_unlock; + coresight_set_percpu_source(csdev); mutex_unlock(&coresight_mutex); if (cti_assoc_ops && cti_assoc_ops->add) @@ -1427,6 +1658,7 @@ void coresight_unregister(struct coresight_device *csdev) cti_assoc_ops->remove(csdev); mutex_lock(&coresight_mutex); + coresight_clear_percpu_source(csdev); etm_perf_del_symlink_sink(csdev); coresight_remove_conns(csdev); coresight_clear_default_sink(csdev); @@ -1615,6 +1847,211 @@ static void coresight_release_device_list(void) } } +static struct coresight_path *coresight_cpu_get_active_path(enum cs_mode mode) +{ + struct coresight_device *source; + bool is_active = false; + + source = coresight_get_percpu_source_ref(smp_processor_id()); + if (!source) + return NULL; + + if (coresight_get_mode(source) & mode) + is_active = true; + + coresight_put_percpu_source_ref(source); + + /* + * It is expected to run in atomic context or with the CPU lock held for + * sysfs mode, so it cannot be preempted to disable the path. Here + * returns the active path pointer without concern that its state may + * change. Since the build path has taken a reference on the component, + * the path can be safely used by the caller. + */ + return is_active ? source->path : NULL; +} + +/* Return: 1 if PM is required, 0 if skip, or a negative error */ +static int coresight_pm_is_needed(struct coresight_path *path) +{ + struct coresight_device *source, *sink; + + if (this_cpu_read(percpu_pm_failed)) + return -EIO; + + if (!path) + return 0; + + source = coresight_get_source(path); + sink = coresight_get_sink(path); + if (!source || !sink) + return 0; + + /* pm_save_disable() and pm_restore_enable() must be paired */ + if (coresight_ops(source)->pm_save_disable && + coresight_ops(source)->pm_restore_enable) + return 1; + + /* + * It is not permitted that the source has no callbacks while the sink + * does, as the sink cannot be disabled without disabling the source, + * which may lead to lockups. Fix this by enabling self-hosted PM + * mode for ETM (see etm4_probe()). + */ + if (coresight_ops(sink)->pm_save_disable && + coresight_ops(sink)->pm_restore_enable) { + pr_warn_once("coresight PM failed: source has no PM callbacks; " + "cannot safely control sink\n"); + return -EINVAL; + } + + return 0; +} + +static int coresight_pm_device_save(struct coresight_device *csdev) +{ + if (!csdev || !coresight_ops(csdev)->pm_save_disable) + return 0; + + return coresight_ops(csdev)->pm_save_disable(csdev); +} + +static void coresight_pm_device_restore(struct coresight_device *csdev) +{ + if (!csdev || !coresight_ops(csdev)->pm_restore_enable) + return; + + coresight_ops(csdev)->pm_restore_enable(csdev); +} + +static int coresight_pm_save(struct coresight_path *path) +{ + struct coresight_device *source = coresight_get_source(path); + struct coresight_node *from, *to; + int ret; + + ret = coresight_pm_device_save(source); + if (ret) + return ret; + + from = coresight_path_first_node(path); + /* Disable up to the node before sink */ + to = list_prev_entry(coresight_path_last_node(path), link); + coresight_disable_path_from_to(path, from, to); + + /* + * Save the sink. Most sinks do not implement a save callback to avoid + * latency from memory copying. We assume the sink's save and restore + * always succeed. + */ + coresight_pm_device_save(coresight_get_sink(path)); + return 0; +} + +static void coresight_pm_restore(struct coresight_path *path) +{ + struct coresight_device *source = coresight_get_source(path); + struct coresight_device *sink = coresight_get_sink(path); + struct coresight_node *from, *to; + int ret; + + coresight_pm_device_restore(sink); + + from = coresight_path_first_node(path); + /* Enable up to the node before sink */ + to = list_prev_entry(coresight_path_last_node(path), link); + ret = coresight_enable_path_from_to(path, coresight_get_mode(source), + from, to); + if (ret) + goto path_failed; + + coresight_pm_device_restore(source); + return; + +path_failed: + coresight_pm_device_save(sink); + + pr_err("Failed in coresight PM restore on CPU%d: %d\n", + smp_processor_id(), ret); + + /* + * Once PM fails on a CPU, set percpu_pm_failed and leave it set until + * reboot. This prevents repeated partial transitions during idle + * entry and exit. + */ + this_cpu_write(percpu_pm_failed, true); +} + +static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, + void *v) +{ + struct coresight_path *path = + coresight_cpu_get_active_path(CS_MODE_SYSFS | CS_MODE_PERF); + int ret; + + ret = coresight_pm_is_needed(path); + if (ret <= 0) + return ret ? NOTIFY_BAD : NOTIFY_DONE; + + switch (cmd) { + case CPU_PM_ENTER: + if (coresight_pm_save(path)) + return NOTIFY_BAD; + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + coresight_pm_restore(path); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block coresight_cpu_pm_nb = { + .notifier_call = coresight_cpu_pm_notify, +}; + +static int coresight_dying_cpu(unsigned int cpu) +{ + struct coresight_path *path; + + /* + * The perf event layer will disable PMU events in the CPU + * hotplug. Here only handles SYSFS case. + */ + path = coresight_cpu_get_active_path(CS_MODE_SYSFS); + if (!path) + return 0; + + coresight_disable_sysfs(coresight_get_source(path)); + return 0; +} + +static int __init coresight_pm_setup(void) +{ + int ret; + + ret = cpu_pm_register_notifier(&coresight_cpu_pm_nb); + if (ret) + return ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE, + "arm/coresight-core:dying", + NULL, coresight_dying_cpu); + if (ret) + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); + + return ret; +} + +static void coresight_pm_cleanup(void) +{ + cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE); + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); +} + const struct bus_type coresight_bustype = { .name = "coresight", }; @@ -1669,9 +2106,15 @@ static int __init coresight_init(void) /* initialise the coresight syscfg API */ ret = cscfg_init(); + if (ret) + goto exit_notifier; + + ret = coresight_pm_setup(); if (!ret) return 0; + cscfg_exit(); +exit_notifier: atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); exit_perf: @@ -1683,6 +2126,7 @@ exit_bus_unregister: static void __exit coresight_exit(void) { + coresight_pm_cleanup(); cscfg_exit(); atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); @@ -1728,10 +2172,10 @@ int coresight_etm_get_trace_id(struct coresight_device *csdev, enum cs_mode mode { int cpu, trace_id; - if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE || !source_ops(csdev)->cpu_id) + if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE) return -EINVAL; - cpu = source_ops(csdev)->cpu_id(csdev); + cpu = csdev->cpu; switch (mode) { case CS_MODE_SYSFS: trace_id = coresight_trace_id_get_cpu_id(cpu); diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 2f4c9362709a9..b2c9a4db13b4e 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -659,7 +659,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) void __iomem *base; struct device *dev = &adev->dev; struct cti_drvdata *drvdata = NULL; - struct coresight_desc cti_desc; + struct coresight_desc cti_desc = { 0 }; struct coresight_platform_data *pdata = NULL; struct resource *res = &adev->res; @@ -702,11 +702,14 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) * eCPU ID. System CTIs will have the name cti_sys<I> where I is an * index allocated by order of discovery. */ - if (drvdata->ctidev.cpu >= 0) + if (drvdata->ctidev.cpu >= 0) { + cti_desc.cpu = drvdata->ctidev.cpu; + cti_desc.flags = CORESIGHT_DESC_CPU_BOUND; cti_desc.name = devm_kasprintf(dev, GFP_KERNEL, "cti_cpu%d", drvdata->ctidev.cpu); - else + } else { cti_desc.name = coresight_alloc_device_name("cti_sys", dev); + } if (!cti_desc.name) return -ENOMEM; diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c index 4eff96f48594e..d6d5388705c3e 100644 --- a/drivers/hwtracing/coresight/coresight-cti-platform.c +++ b/drivers/hwtracing/coresight/coresight-cti-platform.c @@ -329,6 +329,7 @@ static int cti_plat_process_filter_sigs(struct cti_drvdata *drvdata, if (!tg) return -ENOMEM; + tg->nr_sigs = nr_filter_sigs; err = cti_plat_read_trig_group(tg, fwnode, CTI_DT_FILTER_OUT_SIGS); if (!err) drvdata->config.trig_out_filter |= tg->used_mask; diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index b952a1d47f12f..a827f76b8144a 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -83,7 +83,7 @@ struct etb_drvdata { struct coresight_device *csdev; struct miscdevice miscdev; raw_spinlock_t spinlock; - local_t reading; + atomic_t reading; pid_t pid; u8 *buf; u32 buffer_depth; @@ -601,7 +601,7 @@ static int etb_open(struct inode *inode, struct file *file) struct etb_drvdata *drvdata = container_of(file->private_data, struct etb_drvdata, miscdev); - if (local_cmpxchg(&drvdata->reading, 0, 1)) + if (atomic_cmpxchg(&drvdata->reading, 0, 1)) return -EBUSY; dev_dbg(&drvdata->csdev->dev, "%s: successfully opened\n", __func__); @@ -639,7 +639,7 @@ static int etb_release(struct inode *inode, struct file *file) { struct etb_drvdata *drvdata = container_of(file->private_data, struct etb_drvdata, miscdev); - local_set(&drvdata->reading, 0); + atomic_set(&drvdata->reading, 0); dev_dbg(&drvdata->csdev->dev, "%s: released\n", __func__); return 0; diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index f85dedf89a3f9..09b21a711a876 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -49,7 +49,6 @@ struct etm_ctxt { }; static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt); -static DEFINE_PER_CPU(struct coresight_device *, csdev_src); GEN_PMU_FORMAT_ATTR(cycacc); GEN_PMU_FORMAT_ATTR(timestamp); @@ -315,6 +314,115 @@ static bool sinks_compatible(struct coresight_device *a, (sink_ops(a) == sink_ops(b)); } +/* + * This helper is used for fetching the path pointer via the ctxt. + * + * Perf event callbacks run on the same CPU in atomic context, but AUX pause + * and resume may run in NMI context and preempt other callbacks. Since the + * event stop callback clears ctxt->event_data before the data is released, + * AUX pause/resume will either observe a NULL pointer and stop fetching the + * path pointer, or safely access event_data and the path, as the data has + * not yet been freed. + */ +static struct coresight_path *etm_event_get_ctxt_path(struct etm_ctxt *ctxt) +{ + struct etm_event_data *event_data; + struct coresight_path *path; + + if (!ctxt) + return NULL; + + event_data = READ_ONCE(ctxt->event_data); + if (!event_data) + return NULL; + + path = etm_event_cpu_path(event_data, smp_processor_id()); + if (!path) + return NULL; + + return path; +} + +static struct coresight_path * +etm_event_build_path(struct perf_event *event, int cpu, + struct coresight_device *user_sink, + struct coresight_device *match_sink) +{ + struct coresight_path *path = NULL; + struct coresight_device *source, *sink; + int ret; + + source = coresight_get_percpu_source_ref(cpu); + + /* + * If there is no ETM associated with this CPU or ever we try to trace + * on this CPU, we handle it accordingly. + */ + if (!source) + return NULL; + + /* + * If AUX pause feature is enabled but the ETM driver does not + * support the operations, skip for this source. + */ + if (event->attr.aux_start_paused && + (!source_ops(source)->pause_perf || + !source_ops(source)->resume_perf)) { + dev_err_once(&source->dev, "AUX pause is not supported.\n"); + goto out; + } + + /* If sink has been specified by user, directly use it */ + if (user_sink) { + sink = user_sink; + } else { + /* + * No sink provided - look for a default sink for all the ETMs, + * where this event can be scheduled. + * + * We allocate the sink specific buffers only once for this + * event. If the ETMs have different default sink devices, we + * can only use a single "type" of sink as the event can carry + * only one sink specific buffer. Thus we have to make sure + * that the sinks are of the same type and driven by the same + * driver, as the one we allocate the buffer for. We don't + * trace on a CPU if the sink is not compatible. + */ + + /* Find the default sink for this ETM */ + sink = coresight_find_default_sink(source); + if (!sink) + goto out; + + /* Check if this sink compatible with the last sink */ + if (match_sink && !sinks_compatible(match_sink, sink)) + goto out; + } + + /* + * Building a path doesn't enable it, it simply builds a + * list of devices from source to sink that can be + * referenced later when the path is actually needed. + */ + path = coresight_build_path(source, sink); + if (IS_ERR(path)) + goto out; + + /* ensure we can allocate a trace ID for this CPU */ + ret = coresight_path_assign_trace_id(path, CS_MODE_PERF); + if (ret) { + coresight_release_path(path); + path = NULL; + goto out; + } + + coresight_trace_id_perf_start(&sink->perf_sink_id_map); + +out: + coresight_put_percpu_source_ref(source); + return IS_ERR_OR_NULL(path) ? NULL : path; +} + static void *etm_setup_aux(struct perf_event *event, void **pages, int nr_pages, bool overwrite) { @@ -322,7 +430,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, int cpu = event->cpu; cpumask_t *mask; struct coresight_device *sink = NULL; - struct coresight_device *user_sink = NULL, *last_sink = NULL; + struct coresight_device *user_sink = NULL; struct etm_event_data *event_data = NULL; event_data = alloc_event_data(cpu); @@ -354,80 +462,25 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, */ for_each_cpu(cpu, mask) { struct coresight_path *path; - struct coresight_device *csdev; - - csdev = per_cpu(csdev_src, cpu); - /* - * If there is no ETM associated with this CPU clear it from - * the mask and continue with the rest. If ever we try to trace - * on this CPU, we handle it accordingly. - */ - if (!csdev) { - cpumask_clear_cpu(cpu, mask); - continue; - } - /* - * If AUX pause feature is enabled but the ETM driver does not - * support the operations, clear this CPU from the mask and - * continue to next one. - */ - if (event->attr.aux_start_paused && - (!source_ops(csdev)->pause_perf || !source_ops(csdev)->resume_perf)) { - dev_err_once(&csdev->dev, "AUX pause is not supported.\n"); + path = etm_event_build_path(event, cpu, user_sink, sink); + if (!path) { + /* + * Failed to create a path for the CPU, clear it from + * the mask and continue to next one. + */ cpumask_clear_cpu(cpu, mask); continue; } /* - * No sink provided - look for a default sink for all the ETMs, - * where this event can be scheduled. - * We allocate the sink specific buffers only once for this - * event. If the ETMs have different default sink devices, we - * can only use a single "type" of sink as the event can carry - * only one sink specific buffer. Thus we have to make sure - * that the sinks are of the same type and driven by the same - * driver, as the one we allocate the buffer for. As such - * we choose the first sink and check if the remaining ETMs - * have a compatible default sink. We don't trace on a CPU - * if the sink is not compatible. + * The first found sink is saved here and passed to + * etm_event_build_path() to check whether the remaining ETMs + * have a compatible default sink. */ - if (!user_sink) { - /* Find the default sink for this ETM */ - sink = coresight_find_default_sink(csdev); - if (!sink) { - cpumask_clear_cpu(cpu, mask); - continue; - } + if (!user_sink && !sink) + sink = coresight_get_sink(path); - /* Check if this sink compatible with the last sink */ - if (last_sink && !sinks_compatible(last_sink, sink)) { - cpumask_clear_cpu(cpu, mask); - continue; - } - last_sink = sink; - } - - /* - * Building a path doesn't enable it, it simply builds a - * list of devices from source to sink that can be - * referenced later when the path is actually needed. - */ - path = coresight_build_path(csdev, sink); - if (IS_ERR(path)) { - cpumask_clear_cpu(cpu, mask); - continue; - } - - /* ensure we can allocate a trace ID for this CPU */ - coresight_path_assign_trace_id(path, CS_MODE_PERF); - if (!IS_VALID_CS_TRACE_ID(path->trace_id)) { - cpumask_clear_cpu(cpu, mask); - coresight_release_path(path); - continue; - } - - coresight_trace_id_perf_start(&sink->perf_sink_id_map); *etm_event_cpu_path_ptr(event_data, cpu) = path; } @@ -464,13 +517,23 @@ err: goto out; } -static int etm_event_resume(struct coresight_device *csdev, - struct etm_ctxt *ctxt) +static int etm_event_resume(struct coresight_path *path) { - if (!ctxt->event_data) + struct coresight_device *source; + int ret; + + if (!path) return 0; - return coresight_resume_source(csdev); + source = coresight_get_source(path); + if (!source) + return 0; + + ret = coresight_resume_source(source); + if (ret < 0) + dev_err(&source->dev, "Failed to resume ETM event.\n"); + + return ret; } static void etm_event_start(struct perf_event *event, int flags) @@ -479,23 +542,19 @@ static void etm_event_start(struct perf_event *event, int flags) struct etm_event_data *event_data; struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; - struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); + struct coresight_device *source, *sink; struct coresight_path *path; u64 hw_id; - if (!csdev) - goto fail; - if (flags & PERF_EF_RESUME) { - if (etm_event_resume(csdev, ctxt) < 0) { - dev_err(&csdev->dev, "Failed to resume ETM event.\n"); + path = etm_event_get_ctxt_path(ctxt); + if (etm_event_resume(path) < 0) goto fail; - } return; } /* Have we messed up our tracking ? */ - if (WARN_ON(ctxt->event_data)) + if (WARN_ON(READ_ONCE(ctxt->event_data))) goto fail; /* @@ -523,9 +582,10 @@ static void etm_event_start(struct perf_event *event, int flags) path = etm_event_cpu_path(event_data, cpu); path->handle = handle; - /* We need a sink, no need to continue without one */ + /* We need source and sink, no need to continue if any is not set */ + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (WARN_ON_ONCE(!sink)) + if (WARN_ON_ONCE(!source || !sink)) goto fail_end_stop; /* Nothing will happen without a path */ @@ -533,7 +593,7 @@ static void etm_event_start(struct perf_event *event, int flags) goto fail_end_stop; /* Finally enable the tracer */ - if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF, path)) + if (coresight_enable_source(source, event, CS_MODE_PERF, path)) goto fail_disable_path; /* @@ -557,7 +617,7 @@ out: /* Tell the perf core the event is alive */ event->hw.state = 0; /* Save the event_data for this ETM */ - ctxt->event_data = event_data; + WRITE_ONCE(ctxt->event_data, event_data); return; fail_disable_path: @@ -577,27 +637,26 @@ fail: return; } -static void etm_event_pause(struct perf_event *event, - struct coresight_device *csdev, +static void etm_event_pause(struct coresight_path *path, + struct perf_event *event, struct etm_ctxt *ctxt) { - int cpu = smp_processor_id(); - struct coresight_device *sink; struct perf_output_handle *handle = &ctxt->handle; - struct coresight_path *path; + struct coresight_device *source, *sink; + struct etm_event_data *event_data; unsigned long size; - if (!ctxt->event_data) + if (!path) return; - /* Stop tracer */ - coresight_pause_source(csdev); - - path = etm_event_cpu_path(ctxt->event_data, cpu); + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (WARN_ON_ONCE(!sink)) + if (WARN_ON_ONCE(!source || !sink)) return; + /* Stop tracer */ + coresight_pause_source(source); + /* * The per CPU sink has own interrupt handling, it might have * race condition with updating buffer on AUX trace pause if @@ -613,8 +672,9 @@ static void etm_event_pause(struct perf_event *event, if (!sink_ops(sink)->update_buffer) return; + event_data = READ_ONCE(ctxt->event_data); size = sink_ops(sink)->update_buffer(sink, handle, - ctxt->event_data->snk_config); + event_data->snk_config); if (READ_ONCE(handle->event)) { if (!size) return; @@ -630,14 +690,14 @@ static void etm_event_stop(struct perf_event *event, int mode) { int cpu = smp_processor_id(); unsigned long size; - struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); + struct coresight_device *source, *sink; struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; + struct coresight_path *path = etm_event_get_ctxt_path(ctxt); struct etm_event_data *event_data; - struct coresight_path *path; if (mode & PERF_EF_PAUSE) - return etm_event_pause(event, csdev, ctxt); + return etm_event_pause(path, event, ctxt); /* * If we still have access to the event_data via handle, @@ -647,9 +707,9 @@ static void etm_event_stop(struct perf_event *event, int mode) WARN_ON(perf_get_aux(handle) != ctxt->event_data)) return; - event_data = ctxt->event_data; + event_data = READ_ONCE(ctxt->event_data); /* Clear the event_data as this ETM is stopping the trace. */ - ctxt->event_data = NULL; + WRITE_ONCE(ctxt->event_data, NULL); if (event->hw.state == PERF_HES_STOPPED) return; @@ -671,19 +731,13 @@ static void etm_event_stop(struct perf_event *event, int mode) return; } - if (!csdev) - return; - - path = etm_event_cpu_path(event_data, cpu); - if (!path) - return; - + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (!sink) + if (!source || !sink) return; /* stop tracer */ - coresight_disable_source(csdev, event); + coresight_disable_source(source, event); /* tell the core */ event->hw.state = PERF_HES_STOPPED; @@ -824,7 +878,7 @@ static void etm_addr_filters_sync(struct perf_event *event) int etm_perf_symlink(struct coresight_device *csdev, bool link) { char entry[sizeof("cpu9999999")]; - int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev); + int ret = 0, cpu = csdev->cpu; struct device *pmu_dev = etm_pmu.dev; struct device *cs_dev = &csdev->dev; @@ -833,17 +887,12 @@ int etm_perf_symlink(struct coresight_device *csdev, bool link) if (!etm_perf_up) return -EPROBE_DEFER; - if (link) { + if (link) ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry); - if (ret) - return ret; - per_cpu(csdev_src, cpu) = csdev; - } else { + else sysfs_remove_link(&pmu_dev->kobj, entry); - per_cpu(csdev_src, cpu) = NULL; - } - return 0; + return ret; } EXPORT_SYMBOL_GPL(etm_perf_symlink); diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index a547a6d2e0bde..862ad0786699c 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -441,6 +441,7 @@ done: struct etm_enable_arg { struct etm_drvdata *drvdata; + struct coresight_path *path; int rc; }; @@ -462,15 +463,12 @@ static void etm_enable_sysfs_smp_call(void *info) arg->rc = etm_enable_hw(arg->drvdata); /* The tracer didn't start */ - if (arg->rc) + if (arg->rc) { coresight_set_mode(csdev, CS_MODE_DISABLED); -} - -static int etm_cpu_id(struct coresight_device *csdev) -{ - struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + return; + } - return drvdata->cpu; + csdev->path = arg->path; } void etm_release_trace_id(struct etm_drvdata *drvdata) @@ -499,10 +497,13 @@ static int etm_enable_perf(struct coresight_device *csdev, ret = etm_enable_hw(drvdata); /* Failed to start tracer; roll back to DISABLED mode */ - if (ret) + if (ret) { coresight_set_mode(csdev, CS_MODE_DISABLED); + return ret; + } - return ret; + csdev->path = path; + return 0; } static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -521,6 +522,7 @@ static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_pat */ if (cpu_online(drvdata->cpu)) { arg.drvdata = drvdata; + arg.path = path; ret = smp_call_function_single(drvdata->cpu, etm_enable_sysfs_smp_call, &arg, 1); if (!ret) @@ -590,6 +592,7 @@ static void etm_disable_sysfs_smp_call(void *info) etm_disable_hw(drvdata); + drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); } @@ -614,6 +617,7 @@ static void etm_disable_perf(struct coresight_device *csdev) CS_LOCK(drvdata->csa.base); + drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); /* @@ -627,13 +631,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - /* - * Taking hotplug lock here protects from clocks getting disabled - * with tracing being left on (crash scenario) if user disable occurs - * after cpu online mask indicates the cpu is offline but before the - * DYING hotplug callback is serviced by the ETM driver. - */ - cpus_read_lock(); spin_lock(&drvdata->spinlock); /* @@ -644,7 +641,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev) drvdata, 1); spin_unlock(&drvdata->spinlock); - cpus_read_unlock(); /* * we only release trace IDs when resetting sysfs. @@ -684,7 +680,6 @@ static void etm_disable(struct coresight_device *csdev, } static const struct coresight_ops_source etm_source_ops = { - .cpu_id = etm_cpu_id, .enable = etm_enable, .disable = etm_disable, }; @@ -704,35 +699,6 @@ static int etm_online_cpu(unsigned int cpu) return 0; } -static int etm_starting_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - spin_lock(&etmdrvdata[cpu]->spinlock); - if (!etmdrvdata[cpu]->os_unlock) { - etm_os_unlock(etmdrvdata[cpu]); - etmdrvdata[cpu]->os_unlock = true; - } - - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm_enable_hw(etmdrvdata[cpu]); - spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - -static int etm_dying_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - spin_lock(&etmdrvdata[cpu]->spinlock); - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm_disable_hw(etmdrvdata[cpu]); - spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - static bool etm_arch_supported(u8 arch) { switch (arch) { @@ -800,13 +766,6 @@ static int __init etm_hp_setup(void) { int ret; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); - - if (ret) - return ret; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight:online", etm_online_cpu, NULL); @@ -817,15 +776,11 @@ static int __init etm_hp_setup(void) return 0; } - /* failed dyn state - remove others */ - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); - return ret; } static void etm_hp_clear(void) { - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { cpuhp_remove_state_nocalls(hp_online); hp_online = 0; @@ -891,6 +846,8 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) desc.pdata = pdata; desc.dev = dev; desc.groups = coresight_etm_groups; + desc.cpu = drvdata->cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) return PTR_ERR(drvdata->csdev); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d565a73f0042e..14bb31bd6a0b9 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -56,10 +56,14 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot"); #define PARAM_PM_SAVE_NEVER 1 /* never save any state */ #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */ +/* + * Save option for ETM4. ETE, sysreg ETM4s and ACPI boots ignore this option and + * will always save. + */ static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE; module_param(pm_save_enable, int, 0444); MODULE_PARM_DESC(pm_save_enable, - "Save/restore state on power down: 1 = never, 2 = self-hosted"); + "Save/restore state on power down: 1 = never, 2 = self-hosted. MMIO and DT only."); static struct etmv4_drvdata *etmdrvdata[NR_CPUS]; static void etm4_set_default_config(struct etmv4_config *config); @@ -227,13 +231,6 @@ static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, CS_UNLOCK(csa->base); } -static int etm4_cpu_id(struct coresight_device *csdev) -{ - struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - - return drvdata->cpu; -} - void etm4_release_trace_id(struct etmv4_drvdata *drvdata) { coresight_trace_id_put_cpu_id(drvdata->cpu); @@ -241,6 +238,7 @@ void etm4_release_trace_id(struct etmv4_drvdata *drvdata) struct etm4_enable_arg { struct etmv4_drvdata *drvdata; + struct coresight_path *path; int rc; }; @@ -628,8 +626,12 @@ static void etm4_enable_sysfs_smp_call(void *info) arg->rc = etm4_enable_hw(arg->drvdata); /* The tracer didn't start */ - if (arg->rc) + if (arg->rc) { coresight_set_mode(csdev, CS_MODE_DISABLED); + return; + } + + csdev->path = arg->path; } /* @@ -897,9 +899,13 @@ static int etm4_enable_perf(struct coresight_device *csdev, out: /* Failed to start tracer; roll back to DISABLED mode */ - if (ret) + if (ret) { coresight_set_mode(csdev, CS_MODE_DISABLED); - return ret; + return ret; + } + + csdev->path = path; + return 0; } static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -929,6 +935,7 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa * ensures that register writes occur when cpu is powered. */ arg.drvdata = drvdata; + arg.path = path; ret = smp_call_function_single(drvdata->cpu, etm4_enable_sysfs_smp_call, &arg, 1); if (!ret) @@ -1070,6 +1077,7 @@ static void etm4_disable_sysfs_smp_call(void *info) etm4_disable_hw(drvdata); + drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); } @@ -1099,6 +1107,7 @@ static int etm4_disable_perf(struct coresight_device *csdev, /* TRCVICTLR::SSSTATUS, bit[9] */ filters->ssstatus = (control & BIT(9)); + drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); /* @@ -1113,13 +1122,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - /* - * Taking hotplug lock here protects from clocks getting disabled - * with tracing being left on (crash scenario) if user disable occurs - * after cpu online mask indicates the cpu is offline but before the - * DYING hotplug callback is serviced by the ETM driver. - */ - cpus_read_lock(); raw_spin_lock(&drvdata->spinlock); /* @@ -1133,8 +1135,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) cscfg_csdev_disable_active_config(csdev); - cpus_read_unlock(); - /* * we only release trace IDs when resetting sysfs. * This permits sysfs users to read the trace ID after the trace @@ -1201,18 +1201,12 @@ static void etm4_pause_perf(struct coresight_device *csdev) } static const struct coresight_ops_source etm4_source_ops = { - .cpu_id = etm4_cpu_id, .enable = etm4_enable, .disable = etm4_disable, .resume_perf = etm4_resume_perf, .pause_perf = etm4_pause_perf, }; -static const struct coresight_ops etm4_cs_ops = { - .trace_id = coresight_etm_get_trace_id, - .source_ops = &etm4_source_ops, -}; - static bool cpu_supports_sysreg_trace(void) { u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); @@ -1839,44 +1833,19 @@ static int etm4_online_cpu(unsigned int cpu) return 0; } -static int etm4_starting_cpu(unsigned int cpu) +static inline bool etm4_pm_save_needed(struct etmv4_drvdata *drvdata) { - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (!etmdrvdata[cpu]->os_unlock) - etm4_os_unlock(etmdrvdata[cpu]); - - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_enable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - -static int etm4_dying_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_disable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; + return !!drvdata->save_state; } -static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) +static int etm4_cpu_save(struct coresight_device *csdev) { int i, ret = 0; + struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct etmv4_save_state *state; - struct coresight_device *csdev = drvdata->csdev; struct csdev_access *csa; struct device *etm_dev; - if (WARN_ON(!csdev)) - return -ENODEV; - etm_dev = &csdev->dev; csa = &csdev->access; @@ -1979,7 +1948,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); + state->trcvmidcctlr1 = etm4x_read32(csa, TRCVMIDCCTLR1); state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); @@ -2008,31 +1977,13 @@ out: return ret; } -static int etm4_cpu_save(struct etmv4_drvdata *drvdata) -{ - int ret = 0; - - if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED) - return 0; - - /* - * Save and restore the ETM Trace registers only if - * the ETM is active. - */ - if (coresight_get_mode(drvdata->csdev)) - ret = __etm4_cpu_save(drvdata); - return ret; -} - -static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) +static void etm4_cpu_restore(struct coresight_device *csdev) { int i; + struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct etmv4_save_state *state = drvdata->save_state; struct csdev_access *csa = &drvdata->csdev->access; - if (WARN_ON(!drvdata->csdev)) - return; - etm4_cs_unlock(drvdata, csa); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); @@ -2102,7 +2053,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, state->trcvmidcctlr1, TRCVMIDCCTLR1); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); @@ -2125,47 +2076,16 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4_cs_lock(drvdata, csa); } -static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) -{ - if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED) - return; - - if (coresight_get_mode(drvdata->csdev)) - __etm4_cpu_restore(drvdata); -} - -static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, - void *v) -{ - struct etmv4_drvdata *drvdata; - unsigned int cpu = smp_processor_id(); - - if (!etmdrvdata[cpu]) - return NOTIFY_OK; - - drvdata = etmdrvdata[cpu]; - - if (WARN_ON_ONCE(drvdata->cpu != cpu)) - return NOTIFY_BAD; - - switch (cmd) { - case CPU_PM_ENTER: - if (etm4_cpu_save(drvdata)) - return NOTIFY_BAD; - break; - case CPU_PM_EXIT: - case CPU_PM_ENTER_FAILED: - etm4_cpu_restore(drvdata); - break; - default: - return NOTIFY_DONE; - } - - return NOTIFY_OK; -} +static const struct coresight_ops etm4_cs_ops = { + .trace_id = coresight_etm_get_trace_id, + .source_ops = &etm4_source_ops, +}; -static struct notifier_block etm4_cpu_pm_nb = { - .notifier_call = etm4_cpu_pm_notify, +static const struct coresight_ops etm4_cs_pm_ops = { + .trace_id = coresight_etm_get_trace_id, + .source_ops = &etm4_source_ops, + .pm_save_disable = etm4_cpu_save, + .pm_restore_enable = etm4_cpu_restore, }; /* Setup PM. Deals with error conditions and counts */ @@ -2173,17 +2093,6 @@ static int __init etm4_pm_setup(void) { int ret; - ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb); - if (ret) - return ret; - - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight4:starting", - etm4_starting_cpu, etm4_dying_cpu); - - if (ret) - goto unregister_notifier; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight4:online", etm4_online_cpu, NULL); @@ -2194,24 +2103,28 @@ static int __init etm4_pm_setup(void) return 0; } - /* failed dyn state - remove others */ - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); - -unregister_notifier: - cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); return ret; } static void etm4_pm_clear(void) { - cpu_pm_unregister_notifier(&etm4_cpu_pm_nb); - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { cpuhp_remove_state_nocalls(hp_online); hp_online = 0; } } +static bool etm4x_always_pm_save(struct device *dev, struct csdev_access *csa) +{ + /* + * Only IO mem ETM devices will benefit from skipping PM save and only + * DT has the option to control it, not ACPI. Otherwise system register + * based ETMs and ETEs will always lose context on CPU power down, so + * always save. + */ + return !csa->io_mem || is_acpi_device_node(dev_fwnode(dev)); +} + static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) { int ret; @@ -2221,6 +2134,7 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) struct coresight_desc desc = { 0 }; u8 major, minor; char *type_name; + bool pm_save; if (!drvdata) return -EINVAL; @@ -2248,6 +2162,21 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) etm4_set_default(&drvdata->config); + if (etm4x_always_pm_save(dev, init_arg->csa)) + pm_save = true; + else if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) + pm_save = coresight_loses_context_with_cpu(dev); + else + pm_save = pm_save_enable != PARAM_PM_SAVE_NEVER; + + if (pm_save) { + drvdata->save_state = devm_kmalloc(dev, + sizeof(struct etmv4_save_state), + GFP_KERNEL); + if (!drvdata->save_state) + return -ENOMEM; + } + pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) return PTR_ERR(pdata); @@ -2256,10 +2185,12 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; - desc.ops = &etm4_cs_ops; + desc.ops = etm4_pm_save_needed(drvdata) ? &etm4_cs_pm_ops : &etm4_cs_ops; desc.pdata = pdata; desc.dev = dev; desc.groups = coresight_etmv4_groups; + desc.cpu = drvdata->cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) return PTR_ERR(drvdata->csdev); @@ -2305,17 +2236,6 @@ static int etm4_probe(struct device *dev) if (ret) return ret; - if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE) - pm_save_enable = coresight_loses_context_with_cpu(dev) ? - PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER; - - if (pm_save_enable != PARAM_PM_SAVE_NEVER) { - drvdata->save_state = devm_kmalloc(dev, - sizeof(struct etmv4_save_state), GFP_KERNEL); - if (!drvdata->save_state) - return -ENOMEM; - } - raw_spin_lock_init(&drvdata->spinlock); drvdata->cpu = coresight_get_cpu(dev); diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c index e337b6e2bf327..93c2d075cad66 100644 --- a/drivers/hwtracing/coresight/coresight-platform.c +++ b/drivers/hwtracing/coresight/coresight-platform.c @@ -45,9 +45,8 @@ coresight_add_out_conn(struct device *dev, } } - pdata->nr_outconns++; pdata->out_conns = - devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns, + devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns + 1, sizeof(*pdata->out_conns), GFP_KERNEL); if (!pdata->out_conns) return ERR_PTR(-ENOMEM); @@ -63,7 +62,8 @@ coresight_add_out_conn(struct device *dev, * used right away. */ *conn = *new_conn; - pdata->out_conns[pdata->nr_outconns - 1] = conn; + pdata->out_conns[pdata->nr_outconns] = conn; + pdata->nr_outconns++; return conn; } EXPORT_SYMBOL_GPL(coresight_add_out_conn); @@ -86,13 +86,13 @@ int coresight_add_in_conn(struct coresight_connection *out_conn) return 0; } - pdata->nr_inconns++; pdata->in_conns = - devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns, + devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns + 1, sizeof(*pdata->in_conns), GFP_KERNEL); if (!pdata->in_conns) return -ENOMEM; - pdata->in_conns[pdata->nr_inconns - 1] = out_conn; + pdata->in_conns[pdata->nr_inconns] = out_conn; + pdata->nr_inconns++; return 0; } EXPORT_SYMBOL_GPL(coresight_add_in_conn); diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 1ea882dffd703..dddac946659fc 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -153,7 +153,7 @@ int coresight_make_links(struct coresight_device *orig, void coresight_remove_links(struct coresight_device *orig, struct coresight_connection *conn); u32 coresight_get_sink_id(struct coresight_device *csdev); -void coresight_path_assign_trace_id(struct coresight_path *path, +int coresight_path_assign_trace_id(struct coresight_path *path, enum cs_mode mode); #if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM3X) @@ -248,6 +248,12 @@ void coresight_add_helper(struct coresight_device *csdev, void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev); struct coresight_device *coresight_get_percpu_sink(int cpu); +struct coresight_device *coresight_get_source(struct coresight_path *path); +struct coresight_device *coresight_get_percpu_source_ref(int cpu); +void coresight_put_percpu_source_ref(struct coresight_device *csdev); +int coresight_enable_source(struct coresight_device *csdev, + struct perf_event *event, enum cs_mode mode, + struct coresight_path *path); void coresight_disable_source(struct coresight_device *csdev, void *data); void coresight_pause_source(struct coresight_device *csdev); int coresight_resume_source(struct coresight_device *csdev); diff --git a/drivers/hwtracing/coresight/coresight-syscfg.c b/drivers/hwtracing/coresight/coresight-syscfg.c index d7f5037953d6b..2bfdd7b45e49c 100644 --- a/drivers/hwtracing/coresight/coresight-syscfg.c +++ b/drivers/hwtracing/coresight/coresight-syscfg.c @@ -953,39 +953,41 @@ int cscfg_config_sysfs_activate(struct cscfg_config_desc *config_desc, bool acti unsigned long cfg_hash; int err = 0; - mutex_lock(&cscfg_mutex); + guard(mutex)(&cscfg_mutex); cfg_hash = (unsigned long)config_desc->event_ea->var; if (activate) { /* cannot be a current active value to activate this */ - if (cscfg_mgr->sysfs_active_config) { - err = -EBUSY; - goto exit_unlock; - } - err = _cscfg_activate_config(cfg_hash); - if (!err) + if (cscfg_mgr->sysfs_active_config) + return -EBUSY; + + scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) { + err = _cscfg_activate_config(cfg_hash); + if (err) + return err; + cscfg_mgr->sysfs_active_config = cfg_hash; + } } else { - /* disable if matching current value */ - if (cscfg_mgr->sysfs_active_config == cfg_hash) { + if (cscfg_mgr->sysfs_active_config != cfg_hash) + return -EINVAL; + + scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) { + /* disable if matching current value */ _cscfg_deactivate_config(cfg_hash); cscfg_mgr->sysfs_active_config = 0; - } else - err = -EINVAL; + } } -exit_unlock: - mutex_unlock(&cscfg_mutex); - return err; + return 0; } /* set the sysfs preset value */ void cscfg_config_sysfs_set_preset(int preset) { - mutex_lock(&cscfg_mutex); + guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock); cscfg_mgr->sysfs_active_preset = preset; - mutex_unlock(&cscfg_mutex); } /* @@ -994,10 +996,9 @@ void cscfg_config_sysfs_set_preset(int preset) */ void cscfg_config_sysfs_get_active_cfg(unsigned long *cfg_hash, int *preset) { - mutex_lock(&cscfg_mutex); + guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock); *preset = cscfg_mgr->sysfs_active_preset; *cfg_hash = cscfg_mgr->sysfs_active_config; - mutex_unlock(&cscfg_mutex); } EXPORT_SYMBOL_GPL(cscfg_config_sysfs_get_active_cfg); @@ -1201,6 +1202,7 @@ static int cscfg_create_device(void) INIT_LIST_HEAD(&cscfg_mgr->load_order_list); atomic_set(&cscfg_mgr->sys_active_cnt, 0); cscfg_mgr->load_state = CSCFG_NONE; + raw_spin_lock_init(&cscfg_mgr->sysfs_store_lock); /* setup the device */ dev = cscfg_device(); diff --git a/drivers/hwtracing/coresight/coresight-syscfg.h b/drivers/hwtracing/coresight/coresight-syscfg.h index 66e2db890d820..658e93c3705f1 100644 --- a/drivers/hwtracing/coresight/coresight-syscfg.h +++ b/drivers/hwtracing/coresight/coresight-syscfg.h @@ -42,6 +42,7 @@ enum cscfg_load_ops { * @sysfs_active_config:Active config hash used if CoreSight controlled from sysfs. * @sysfs_active_preset:Active preset index used if CoreSight controlled from sysfs. * @load_state: A multi-stage load/unload operation is in progress. + * @sysfs_store_lock: Exclusive access sysfs stored variables. */ struct cscfg_manager { struct device dev; @@ -54,6 +55,7 @@ struct cscfg_manager { u32 sysfs_active_config; int sysfs_active_preset; enum cscfg_load_ops load_state; + raw_spinlock_t sysfs_store_lock; }; /* get reference to dev in cscfg_manager */ diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index d2a6ed8bcc74d..4b010f8bc4c08 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -5,26 +5,12 @@ */ #include <linux/device.h> -#include <linux/idr.h> #include <linux/kernel.h> #include <linux/property.h> #include "coresight-priv.h" #include "coresight-trace-id.h" -/* - * Use IDR to map the hash of the source's device name - * to the pointer of path for the source. The idr is for - * the sources which aren't associated with CPU. - */ -static DEFINE_IDR(path_idr); - -/* - * When operating Coresight drivers from the sysFS interface, only a single - * path can exist from a tracer (associated to a CPU) to a sink. - */ -static DEFINE_PER_CPU(struct coresight_path *, tracer_path); - ssize_t coresight_simple_show_pair(struct device *_dev, struct device_attribute *attr, char *buf) { @@ -53,6 +39,26 @@ ssize_t coresight_simple_show32(struct device *_dev, } EXPORT_SYMBOL_GPL(coresight_simple_show32); +static void coresight_source_get_refcnt(struct coresight_device *csdev) +{ + /* + * There could be multiple applications driving the software + * source. So keep the refcount for each such user when the + * source is already enabled. + * + * No need to increment the reference counter for other source + * types, as multiple enables are the same as a single enable. + */ + if (coresight_is_software_source(csdev)) + csdev->refcnt++; +} + +static void coresight_source_put_refcnt(struct coresight_device *csdev) +{ + if (coresight_is_software_source(csdev)) + csdev->refcnt--; +} + static int coresight_enable_source_sysfs(struct coresight_device *csdev, enum cs_mode mode, struct coresight_path *path) @@ -66,19 +72,19 @@ static int coresight_enable_source_sysfs(struct coresight_device *csdev, */ lockdep_assert_held(&coresight_mutex); if (coresight_get_mode(csdev) != CS_MODE_SYSFS) { - ret = source_ops(csdev)->enable(csdev, NULL, mode, path); + ret = coresight_enable_source(csdev, NULL, mode, path); if (ret) return ret; } - csdev->refcnt++; + coresight_source_get_refcnt(csdev); return 0; } /** - * coresight_disable_source_sysfs - Drop the reference count by 1 and disable - * the device if there are no users left. + * coresight_disable_source_sysfs - Drop the reference count by 1 for software + * sources. Disable the device if there are no users left. * * @csdev: The coresight device to disable * @data: Opaque data to pass on to the disable function of the source device. @@ -93,7 +99,7 @@ static bool coresight_disable_source_sysfs(struct coresight_device *csdev, if (coresight_get_mode(csdev) != CS_MODE_SYSFS) return false; - csdev->refcnt--; + coresight_source_put_refcnt(csdev); if (csdev->refcnt == 0) { coresight_disable_source(csdev, data); return true; @@ -162,18 +168,17 @@ static int coresight_validate_source_sysfs(struct coresight_device *csdev, return -EINVAL; } + if (coresight_is_percpu_source(csdev) && !cpu_online(csdev->cpu)) + return -ENODEV; + return 0; } int coresight_enable_sysfs(struct coresight_device *csdev) { - int cpu, ret = 0; + int ret = 0; struct coresight_device *sink; struct coresight_path *path; - enum coresight_dev_subtype_source subtype; - u32 hash; - - subtype = csdev->subtype.source_subtype; mutex_lock(&coresight_mutex); @@ -188,13 +193,7 @@ int coresight_enable_sysfs(struct coresight_device *csdev) * doesn't hold coresight_mutex. */ if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { - /* - * There could be multiple applications driving the software - * source. So keep the refcount for each such user when the - * source is already enabled. - */ - if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE) - csdev->refcnt++; + coresight_source_get_refcnt(csdev); goto out; } @@ -211,8 +210,8 @@ int coresight_enable_sysfs(struct coresight_device *csdev) goto out; } - coresight_path_assign_trace_id(path, CS_MODE_SYSFS); - if (!IS_VALID_CS_TRACE_ID(path->trace_id)) + ret = coresight_path_assign_trace_id(path, CS_MODE_SYSFS); + if (ret) goto err_path; ret = coresight_enable_path(path, CS_MODE_SYSFS); @@ -223,35 +222,6 @@ int coresight_enable_sysfs(struct coresight_device *csdev) if (ret) goto err_source; - switch (subtype) { - case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: - /* - * When working from sysFS it is important to keep track - * of the paths that were created so that they can be - * undone in 'coresight_disable()'. Since there can only - * be a single session per tracer (when working from sysFS) - * a per-cpu variable will do just fine. - */ - cpu = source_ops(csdev)->cpu_id(csdev); - per_cpu(tracer_path, cpu) = path; - break; - case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: - case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: - case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: - /* - * Use the hash of source's device name as ID - * and map the ID to the pointer of the path. - */ - hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); - ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL); - if (ret) - goto err_source; - break; - default: - /* We can't be here */ - break; - } - out: mutex_unlock(&coresight_mutex); return ret; @@ -267,9 +237,8 @@ EXPORT_SYMBOL_GPL(coresight_enable_sysfs); void coresight_disable_sysfs(struct coresight_device *csdev) { - int cpu, ret; - struct coresight_path *path = NULL; - u32 hash; + struct coresight_path *path; + int ret; mutex_lock(&coresight_mutex); @@ -277,32 +246,15 @@ void coresight_disable_sysfs(struct coresight_device *csdev) if (ret) goto out; + /* + * coresight_disable_source_sysfs() clears the 'csdev->path' pointer + * when disabling the source. Retrieve the path pointer here. + */ + path = csdev->path; + if (!coresight_disable_source_sysfs(csdev, NULL)) goto out; - switch (csdev->subtype.source_subtype) { - case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: - cpu = source_ops(csdev)->cpu_id(csdev); - path = per_cpu(tracer_path, cpu); - per_cpu(tracer_path, cpu) = NULL; - break; - case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: - case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: - case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: - hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); - /* Find the path by the hash. */ - path = idr_find(&path_idr, hash); - if (path == NULL) { - pr_err("Path is not found for %s\n", dev_name(&csdev->dev)); - goto out; - } - idr_remove(&path_idr, hash); - break; - default: - /* We can't be here */ - break; - } - coresight_disable_path(path); coresight_release_path(path); @@ -360,6 +312,13 @@ static ssize_t enable_source_store(struct device *dev, if (ret) return ret; + /* + * CoreSight hotplug callbacks in core layer control a activated path + * from its source to sink. Taking hotplug lock here protects a race + * condition with hotplug callbacks. + */ + guard(cpus_read_lock)(); + if (val) { ret = coresight_enable_sysfs(csdev); if (ret) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 4dc1defe27a5f..361a433e6f0c5 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -154,7 +154,7 @@ tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr) for (i = 0; i < tmc_pages->nr_pages; i++) { page_start = tmc_pages->daddrs[i]; if (addr >= page_start && addr < (page_start + PAGE_SIZE)) - return i * PAGE_SIZE + (addr - page_start); + return (long)i * PAGE_SIZE + (addr - page_start); } return -EINVAL; @@ -1379,7 +1379,7 @@ alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); /* Use the minimum limit if the required size is smaller */ - size = nr_pages << PAGE_SHIFT; + size = (ssize_t)nr_pages << PAGE_SHIFT; size = max_t(ssize_t, size, TMC_ETR_PERF_MIN_BUF_SIZE); /* diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 1511f8eb95afb..c7cbca45f2deb 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -117,6 +117,20 @@ static int trbe_errata_cpucaps[] = { #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256 /* + * struct trbe_save_state: Register values representing TRBE state + * @trblimitr - Trace Buffer Limit Address Register value + * @trbbaser - Trace Buffer Base Register value + * @trbptr - Trace Buffer Write Pointer Register value + * @trbsr - Trace Buffer Status Register value + */ +struct trbe_save_state { + u64 trblimitr; + u64 trbbaser; + u64 trbptr; + u64 trbsr; +}; + +/* * struct trbe_cpudata: TRBE instance specific data * @trbe_flag - TRBE dirty/access flag support * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1. @@ -134,6 +148,7 @@ struct trbe_cpudata { enum cs_mode mode; struct trbe_buf *buf; struct trbe_drvdata *drvdata; + struct trbe_save_state save_state; DECLARE_BITMAP(errata, TRBE_ERRATA_MAX); }; @@ -1189,6 +1204,46 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) return IRQ_HANDLED; } +static int arm_trbe_save(struct coresight_device *csdev) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_save_state *state = &cpudata->save_state; + + state->trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + /* Disable the unit, ensure the writes to memory are complete */ + if (state->trblimitr & TRBLIMITR_EL1_E) + trbe_drain_and_disable_local(cpudata); + + state->trbbaser = read_sysreg_s(SYS_TRBBASER_EL1); + state->trbptr = read_sysreg_s(SYS_TRBPTR_EL1); + state->trbsr = read_sysreg_s(SYS_TRBSR_EL1); + return 0; +} + +static void arm_trbe_restore(struct coresight_device *csdev) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_save_state *state = &cpudata->save_state; + + write_sysreg_s(state->trbbaser, SYS_TRBBASER_EL1); + write_sysreg_s(state->trbptr, SYS_TRBPTR_EL1); + write_sysreg_s(state->trbsr, SYS_TRBSR_EL1); + + if (!(state->trblimitr & TRBLIMITR_EL1_E)) { + write_sysreg_s(state->trblimitr, SYS_TRBLIMITR_EL1); + } else { + /* + * The section K5.5 Context switching, Arm ARM (ARM DDI 0487 + * L.a), S_PKLXF requires a Context synchronization event to + * guarantee the Trace Buffer Unit will observe the new values + * of the system registers. + */ + isb(); + set_trbe_enabled(cpudata, state->trblimitr); + } +} + static const struct coresight_ops_sink arm_trbe_sink_ops = { .enable = arm_trbe_enable, .disable = arm_trbe_disable, @@ -1198,7 +1253,9 @@ static const struct coresight_ops_sink arm_trbe_sink_ops = { }; static const struct coresight_ops arm_trbe_cs_ops = { - .sink_ops = &arm_trbe_sink_ops, + .pm_save_disable = arm_trbe_save, + .pm_restore_enable = arm_trbe_restore, + .sink_ops = &arm_trbe_sink_ops, }; static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1289,6 +1346,8 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp desc.ops = &arm_trbe_cs_ops; desc.groups = arm_trbe_groups; desc.dev = dev; + desc.cpu = cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; trbe_csdev = coresight_register(&desc); if (IS_ERR(trbe_csdev)) goto cpu_clear; diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 2131febebee93..add0579cad884 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -141,6 +141,8 @@ struct csdev_access { .base = (_addr), \ }) +#define CORESIGHT_DESC_CPU_BOUND BIT(0) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -153,6 +155,8 @@ struct csdev_access { * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. * @access: Describe access to the device + * @flags: The descritpion flags. + * @cpu: The CPU this component is affined to. */ struct coresight_desc { enum coresight_dev_type type; @@ -163,6 +167,8 @@ struct coresight_desc { const struct attribute_group **groups; const char *name; struct csdev_access access; + u32 flags; + int cpu; }; /** @@ -251,6 +257,7 @@ struct coresight_trace_id_map { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. + * @path: Activated path pointer (only used for source device). * @mode: The device mode, i.e sysFS, Perf or disabled. This is actually * an 'enum cs_mode' but stored in an atomic type. Access is always * through atomic APIs, ensuring SMP-safe synchronisation between @@ -260,6 +267,7 @@ struct coresight_trace_id_map { * device's spinlock when the coresight_mutex held and mode == * CS_MODE_SYSFS. Otherwise it must be accessed from inside the * spinlock. + * @cpu: The CPU this component is affined to (-1 for not CPU bound). * @orphan: true if the component has connections that haven't been linked. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be @@ -284,8 +292,10 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; + struct coresight_path *path; atomic_t mode; int refcnt; + int cpu; bool orphan; /* sink specific fields */ bool sysfs_sink_activated; @@ -334,9 +344,9 @@ struct coresight_path { }; enum cs_mode { - CS_MODE_DISABLED, - CS_MODE_SYSFS, - CS_MODE_PERF, + CS_MODE_DISABLED = 0, + CS_MODE_SYSFS = BIT(0), + CS_MODE_PERF = BIT(1), }; #define coresight_ops(csdev) csdev->ops @@ -387,15 +397,12 @@ struct coresight_ops_link { /** * struct coresight_ops_source - basic operations for a source * Operations available for sources. - * @cpu_id: returns the value of the CPU number this component - * is associated to. * @enable: enables tracing for a source. * @disable: disables tracing for a source. * @resume_perf: resumes tracing for a source in perf session. * @pause_perf: pauses tracing for a source in perf session. */ struct coresight_ops_source { - int (*cpu_id)(struct coresight_device *csdev); int (*enable)(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode, struct coresight_path *path); void (*disable)(struct coresight_device *csdev, @@ -433,6 +440,8 @@ struct coresight_ops_panic { struct coresight_ops { int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode, struct coresight_device *sink); + int (*pm_save_disable)(struct coresight_device *csdev); + void (*pm_restore_enable)(struct coresight_device *csdev); const struct coresight_ops_sink *sink_ops; const struct coresight_ops_link *link_ops; const struct coresight_ops_source *source_ops; @@ -602,6 +611,12 @@ static inline bool coresight_is_percpu_source(struct coresight_device *csdev) (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC); } +static inline bool coresight_is_software_source(struct coresight_device *csdev) +{ + return csdev && coresight_is_device_source(csdev) && + (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE); +} + static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) { return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) && diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 22ba327ec2278..0fb3a2a62eb00 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -180,7 +180,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_STARTING, CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, @@ -200,6 +199,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_AFFINITY_ONLINE, CPUHP_AP_BLK_MQ_ONLINE, CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, + CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE, |
