aboutsummaryrefslogtreecommitdiffstats
diff options
authorMark Brown <broonie@kernel.org>2026-05-29 23:00:38 +0100
committerMark Brown <broonie@kernel.org>2026-05-29 23:00:38 +0100
commit1aef57bf376607540d77304cc65b44996746cbaf (patch)
tree23cc26befc9e65aa47eade8207f136a4942f75d5
parent5438eb458830693c5f8481aa36097245a727e2c2 (diff)
parentfa09f08ede3db3050ae16ae1ed92c902d0cada23 (diff)
downloadlinux-next-history-1aef57bf376607540d77304cc65b44996746cbaf.tar.gz
Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git
-rw-r--r--drivers/hwtracing/coresight/coresight-catu.c2
-rw-r--r--drivers/hwtracing/coresight/coresight-core.c574
-rw-r--r--drivers/hwtracing/coresight/coresight-cti-core.c9
-rw-r--r--drivers/hwtracing/coresight/coresight-cti-platform.c1
-rw-r--r--drivers/hwtracing/coresight/coresight-etb10.c6
-rw-r--r--drivers/hwtracing/coresight/coresight-etm-perf.c287
-rw-r--r--drivers/hwtracing/coresight/coresight-etm3x-core.c73
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x-core.c214
-rw-r--r--drivers/hwtracing/coresight/coresight-platform.c12
-rw-r--r--drivers/hwtracing/coresight/coresight-priv.h8
-rw-r--r--drivers/hwtracing/coresight/coresight-syscfg.c38
-rw-r--r--drivers/hwtracing/coresight/coresight-syscfg.h2
-rw-r--r--drivers/hwtracing/coresight/coresight-sysfs.c135
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etr.c4
-rw-r--r--drivers/hwtracing/coresight/coresight-trbe.c61
-rw-r--r--include/linux/coresight.h27
-rw-r--r--include/linux/cpuhotplug.h2
17 files changed, 936 insertions, 519 deletions
diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c
index ce71dcddfca25..43abe13995cf3 100644
--- a/drivers/hwtracing/coresight/coresight-catu.c
+++ b/drivers/hwtracing/coresight/coresight-catu.c
@@ -514,7 +514,7 @@ static int __catu_probe(struct device *dev, struct resource *res)
int ret = 0;
u32 dma_mask;
struct catu_drvdata *drvdata;
- struct coresight_desc catu_desc;
+ struct coresight_desc catu_desc = { 0 };
struct coresight_platform_data *pdata = NULL;
void __iomem *base;
diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c
index 46f247f73cf64..f7b1308a759c3 100644
--- a/drivers/hwtracing/coresight/coresight-core.c
+++ b/drivers/hwtracing/coresight/coresight-core.c
@@ -6,6 +6,7 @@
#include <linux/acpi.h>
#include <linux/bitfield.h>
#include <linux/build_bug.h>
+#include <linux/cpu_pm.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/types.h>
@@ -35,6 +36,10 @@
DEFINE_MUTEX(coresight_mutex);
static DEFINE_PER_CPU(struct coresight_device *, csdev_sink);
+static DEFINE_RAW_SPINLOCK(coresight_dev_lock);
+static DEFINE_PER_CPU(struct coresight_device *, csdev_source);
+static DEFINE_PER_CPU(bool, percpu_pm_failed);
+
/**
* struct coresight_node - elements of a path, from source to sink
* @csdev: Address of an element.
@@ -58,6 +63,24 @@ static LIST_HEAD(coresight_dev_idx_list);
static const struct cti_assoc_op *cti_assoc_ops;
+static struct coresight_node *
+coresight_path_first_node(struct coresight_path *path)
+{
+ if (list_empty(&path->path_list))
+ return NULL;
+
+ return list_first_entry(&path->path_list, struct coresight_node, link);
+}
+
+static struct coresight_node *
+coresight_path_last_node(struct coresight_path *path)
+{
+ if (list_empty(&path->path_list))
+ return NULL;
+
+ return list_last_entry(&path->path_list, struct coresight_node, link);
+}
+
void coresight_set_cti_ops(const struct cti_assoc_op *cti_op)
{
cti_assoc_ops = cti_op;
@@ -82,14 +105,89 @@ struct coresight_device *coresight_get_percpu_sink(int cpu)
}
EXPORT_SYMBOL_GPL(coresight_get_percpu_sink);
-static struct coresight_device *coresight_get_source(struct coresight_path *path)
+static void coresight_set_percpu_source(struct coresight_device *csdev)
+{
+ if (!csdev || !coresight_is_percpu_source(csdev))
+ return;
+
+ guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+ /* Expect no device to be set yet */
+ WARN_ON(per_cpu(csdev_source, csdev->cpu));
+ per_cpu(csdev_source, csdev->cpu) = csdev;
+}
+
+static void coresight_clear_percpu_source(struct coresight_device *csdev)
+{
+ if (!csdev || !coresight_is_percpu_source(csdev))
+ return;
+
+ /* Clear percpu_pm_failed */
+ per_cpu(percpu_pm_failed, csdev->cpu) = false;
+
+ guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+ /* The per-CPU pointer should contain the same csdev */
+ WARN_ON(per_cpu(csdev_source, csdev->cpu) != csdev);
+ per_cpu(csdev_source, csdev->cpu) = NULL;
+}
+
+struct coresight_device *coresight_get_percpu_source_ref(int cpu)
+{
+ struct coresight_device *csdev;
+
+ if (WARN_ON(cpu < 0))
+ return NULL;
+
+ guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+ csdev = per_cpu(csdev_source, cpu);
+ if (!csdev)
+ return NULL;
+
+ /*
+ * Holding a reference to the csdev->dev ensures that the
+ * coresight_device is live for the caller. The path building
+ * logic can safely either build a path to the sink or fail
+ * if the device is being unregistered (if there was a race).
+ * The caller can skip the "source" device, if no path could
+ * be built.
+ */
+ get_device(&csdev->dev);
+
+ return csdev;
+}
+
+void coresight_put_percpu_source_ref(struct coresight_device *csdev)
+{
+ if (!csdev || !coresight_is_percpu_source(csdev))
+ return;
+
+ guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+ /*
+ * TODO: coresight_device_release() is invoked to release resources when
+ * the device's refcount reaches zero. It then calls free_percpu(),
+ * which acquires pcpu_lock — a sleepable lock when PREEMPT_RT is
+ * enabled. Since the raw spinlock coresight_dev_lock is held, this can
+ * lead to a potential "scheduling while atomic" issue.
+ */
+ put_device(&csdev->dev);
+}
+
+struct coresight_device *coresight_get_source(struct coresight_path *path)
{
struct coresight_device *csdev;
+ struct coresight_node *nd;
if (!path)
return NULL;
- csdev = list_first_entry(&path->path_list, struct coresight_node, link)->csdev;
+ nd = coresight_path_first_node(path);
+ if (!nd)
+ return NULL;
+
+ csdev = nd->csdev;
if (!coresight_is_device_source(csdev))
return NULL;
@@ -385,19 +483,47 @@ static void coresight_disable_helpers(struct coresight_device *csdev,
}
/*
- * Helper function to call source_ops(csdev)->disable and also disable the
- * helpers.
- *
- * There is an imbalance between coresight_enable_path() and
- * coresight_disable_path(). Enabling also enables the source's helpers as part
- * of the path, but disabling always skips the first item in the path (which is
- * the source), so sources and their helpers don't get disabled as part of that
- * function and we need the extra step here.
+ * coresight_enable_source() and coresight_disable_source() only enable and
+ * disable the source, but do nothing for the associated helpers, which are
+ * controlled as part of the path.
*/
+int coresight_enable_source(struct coresight_device *csdev,
+ struct perf_event *event, enum cs_mode mode,
+ struct coresight_path *path)
+{
+ int ret;
+
+ if (!coresight_is_device_source(csdev))
+ return -EINVAL;
+
+ ret = source_ops(csdev)->enable(csdev, event, mode, path);
+ if (ret)
+ return ret;
+
+ /*
+ * Update the path pointer until after the source is enabled to avoid
+ * races where multiple paths attempt to enable the same source.
+ *
+ * Do not set the path pointer here for per-CPU sources; set it locally
+ * on the CPU instead. Otherwise, there is a window where the path is
+ * enabled but the pointer is not yet set, causing CPU PM notifiers to
+ * miss PM operations due to reading a NULL pointer.
+ */
+ if (!coresight_is_percpu_source(csdev))
+ csdev->path = path;
+
+ return 0;
+}
+
void coresight_disable_source(struct coresight_device *csdev, void *data)
{
+ if (!coresight_is_device_source(csdev))
+ return;
+
+ if (!coresight_is_percpu_source(csdev))
+ csdev->path = NULL;
+
source_ops(csdev)->disable(csdev, data);
- coresight_disable_helpers(csdev, NULL);
}
EXPORT_SYMBOL_GPL(coresight_disable_source);
@@ -424,20 +550,42 @@ int coresight_resume_source(struct coresight_device *csdev)
EXPORT_SYMBOL_GPL(coresight_resume_source);
/*
- * coresight_disable_path_from : Disable components in the given path beyond
- * @nd in the list. If @nd is NULL, all the components, except the SOURCE are
- * disabled.
+ * Callers must fetch nodes from the path and pass @from and @to to the path
+ * enable/disable functions. Walk the path from @from to locate @to. If @to
+ * is found, it indicates @from and @to are in order. Otherwise, they are out
+ * of order.
*/
-static void coresight_disable_path_from(struct coresight_path *path,
- struct coresight_node *nd)
+static bool coresight_path_nodes_in_order(struct coresight_path *path,
+ struct coresight_node *from,
+ struct coresight_node *to)
+{
+ struct coresight_node *nd;
+
+ if (WARN_ON_ONCE(!from || !to))
+ return false;
+
+ nd = from;
+ list_for_each_entry_from(nd, &path->path_list, link) {
+ if (nd == to)
+ return true;
+ }
+
+ return false;
+}
+
+static void coresight_disable_path_from_to(struct coresight_path *path,
+ struct coresight_node *from,
+ struct coresight_node *to)
{
u32 type;
struct coresight_device *csdev, *parent, *child;
+ struct coresight_node *nd;
- if (!nd)
- nd = list_first_entry(&path->path_list, struct coresight_node, link);
+ if (!coresight_path_nodes_in_order(path, from, to))
+ return;
- list_for_each_entry_continue(nd, &path->path_list, link) {
+ nd = from;
+ list_for_each_entry_from(nd, &path->path_list, link) {
csdev = nd->csdev;
type = csdev->type;
@@ -457,12 +605,6 @@ static void coresight_disable_path_from(struct coresight_path *path,
coresight_disable_sink(csdev);
break;
case CORESIGHT_DEV_TYPE_SOURCE:
- /*
- * We skip the first node in the path assuming that it
- * is the source. So we don't expect a source device in
- * the middle of a path.
- */
- WARN_ON(1);
break;
case CORESIGHT_DEV_TYPE_LINK:
parent = list_prev_entry(nd, link)->csdev;
@@ -476,12 +618,18 @@ static void coresight_disable_path_from(struct coresight_path *path,
/* Disable all helpers adjacent along the path last */
coresight_disable_helpers(csdev, path);
+
+ /* Iterate up to and including @to */
+ if (nd == to)
+ break;
}
}
void coresight_disable_path(struct coresight_path *path)
{
- coresight_disable_path_from(path, NULL);
+ coresight_disable_path_from_to(path,
+ coresight_path_first_node(path),
+ coresight_path_last_node(path));
}
EXPORT_SYMBOL_GPL(coresight_disable_path);
@@ -499,22 +647,36 @@ static int coresight_enable_helpers(struct coresight_device *csdev,
ret = coresight_enable_helper(helper, mode, path);
if (ret)
- return ret;
+ goto err;
}
return 0;
+
+err:
+ while (i--) {
+ helper = csdev->pdata->out_conns[i]->dest_dev;
+ if (helper && coresight_is_helper(helper))
+ coresight_disable_helper(helper, path);
+ }
+
+ return ret;
}
-int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
+static int coresight_enable_path_from_to(struct coresight_path *path,
+ enum cs_mode mode,
+ struct coresight_node *from,
+ struct coresight_node *to)
{
int ret = 0;
u32 type;
struct coresight_node *nd;
struct coresight_device *csdev, *parent, *child;
- struct coresight_device *source;
- source = coresight_get_source(path);
- list_for_each_entry_reverse(nd, &path->path_list, link) {
+ if (!coresight_path_nodes_in_order(path, from, to))
+ return -EINVAL;
+
+ nd = to;
+ list_for_each_entry_from_reverse(nd, &path->path_list, link) {
csdev = nd->csdev;
type = csdev->type;
@@ -553,7 +715,8 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
case CORESIGHT_DEV_TYPE_LINK:
parent = list_prev_entry(nd, link)->csdev;
child = list_next_entry(nd, link)->csdev;
- ret = coresight_enable_link(csdev, parent, child, source);
+ ret = coresight_enable_link(csdev, parent, child,
+ coresight_get_source(path));
if (ret)
goto err_disable_helpers;
break;
@@ -561,6 +724,10 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
ret = -EINVAL;
goto err_disable_helpers;
}
+
+ /* Iterate down to and including @from */
+ if (nd == from)
+ break;
}
out:
@@ -568,18 +735,36 @@ out:
err_disable_helpers:
coresight_disable_helpers(csdev, path);
err_disable_path:
- coresight_disable_path_from(path, nd);
+ /* No device is actually enabled */
+ if (nd == to)
+ goto out;
+
+ /* Fetch the previous node, the last successfully enabled one */
+ nd = list_next_entry(nd, link);
+ coresight_disable_path_from_to(path, nd, to);
goto out;
}
+int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
+{
+ return coresight_enable_path_from_to(path, mode,
+ coresight_path_first_node(path),
+ coresight_path_last_node(path));
+}
+
struct coresight_device *coresight_get_sink(struct coresight_path *path)
{
struct coresight_device *csdev;
+ struct coresight_node *nd;
if (!path)
return NULL;
- csdev = list_last_entry(&path->path_list, struct coresight_node, link)->csdev;
+ nd = coresight_path_last_node(path);
+ if (!nd)
+ return NULL;
+
+ csdev = nd->csdev;
if (csdev->type != CORESIGHT_DEV_TYPE_SINK &&
csdev->type != CORESIGHT_DEV_TYPE_LINKSINK)
return NULL;
@@ -642,15 +827,29 @@ struct coresight_device *coresight_get_sink_by_id(u32 id)
*/
static bool coresight_get_ref(struct coresight_device *csdev)
{
- struct device *dev = csdev->dev.parent;
+ struct device *dev = &csdev->dev;
+ struct device *parent = csdev->dev.parent;
+ struct device_driver *drv;
- /* Make sure the driver can't be removed */
- if (!try_module_get(dev->driver->owner))
- return false;
- /* Make sure the device can't go away */
+ /* Make sure csdev can't go away */
get_device(dev);
- pm_runtime_get_sync(dev);
+
+ /* Make sure parent device can't go away */
+ get_device(parent);
+
+ /* Make sure the driver can't be removed */
+ drv = parent->driver;
+ if (!drv || !try_module_get(drv->owner))
+ goto err_module;
+
+ /* Make sure the device is powered on */
+ pm_runtime_get_sync(parent);
return true;
+
+err_module:
+ put_device(parent);
+ put_device(dev);
+ return false;
}
/**
@@ -661,11 +860,15 @@ static bool coresight_get_ref(struct coresight_device *csdev)
*/
static void coresight_put_ref(struct coresight_device *csdev)
{
- struct device *dev = csdev->dev.parent;
+ struct device *dev = &csdev->dev;
+ struct device *parent = csdev->dev.parent;
+ struct device_driver *drv = parent->driver;
- pm_runtime_put(dev);
+ pm_runtime_put(parent);
+ if (drv)
+ module_put(drv->owner);
+ put_device(parent);
put_device(dev);
- module_put(dev->driver->owner);
}
/*
@@ -739,8 +942,8 @@ static int coresight_get_trace_id(struct coresight_device *csdev,
* Call this after creating the path and before enabling it. This leaves
* the trace ID set on the path, or it remains 0 if it couldn't be assigned.
*/
-void coresight_path_assign_trace_id(struct coresight_path *path,
- enum cs_mode mode)
+int coresight_path_assign_trace_id(struct coresight_path *path,
+ enum cs_mode mode)
{
struct coresight_device *sink = coresight_get_sink(path);
struct coresight_node *nd;
@@ -750,15 +953,18 @@ void coresight_path_assign_trace_id(struct coresight_path *path,
/* Assign a trace ID to the path for the first device that wants to do it */
trace_id = coresight_get_trace_id(nd->csdev, mode, sink);
- /*
- * 0 in this context is that it didn't want to assign so keep searching.
- * Non 0 is either success or fail.
- */
- if (trace_id != 0) {
- path->trace_id = trace_id;
- return;
- }
+ /* 0 means the device has no ID assignment, so keep searching */
+ if (trace_id == 0)
+ continue;
+
+ if (!IS_VALID_CS_TRACE_ID(trace_id))
+ return -EINVAL;
+
+ path->trace_id = trace_id;
+ return 0;
}
+
+ return -EINVAL;
}
/**
@@ -787,7 +993,7 @@ static int _coresight_build_path(struct coresight_device *csdev,
goto out;
if (coresight_is_percpu_source(csdev) && coresight_is_percpu_sink(sink) &&
- sink == per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev))) {
+ sink == per_cpu(csdev_sink, csdev->cpu)) {
if (_coresight_build_path(sink, source, sink, path) == 0) {
found = true;
goto out;
@@ -1014,7 +1220,7 @@ coresight_find_default_sink(struct coresight_device *csdev)
/* look for a default sink if we have not found for this device */
if (!csdev->def_sink) {
if (coresight_is_percpu_source(csdev))
- csdev->def_sink = per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev));
+ csdev->def_sink = per_cpu(csdev_sink, csdev->cpu);
if (!csdev->def_sink)
csdev->def_sink = coresight_find_sink(csdev, &depth);
}
@@ -1322,32 +1528,56 @@ void coresight_release_platform_data(struct device *dev,
devm_kfree(dev, pdata);
}
-struct coresight_device *coresight_register(struct coresight_desc *desc)
+static struct coresight_device *
+coresight_init_device(struct coresight_desc *desc)
{
- int ret;
struct coresight_device *csdev;
- bool registered = false;
csdev = kzalloc_obj(*csdev);
- if (!csdev) {
- ret = -ENOMEM;
- goto err_out;
- }
+ if (!csdev)
+ return ERR_PTR(-ENOMEM);
csdev->pdata = desc->pdata;
-
csdev->type = desc->type;
csdev->subtype = desc->subtype;
csdev->ops = desc->ops;
csdev->access = desc->access;
csdev->orphan = true;
+ if (desc->flags & CORESIGHT_DESC_CPU_BOUND) {
+ csdev->cpu = desc->cpu;
+ } else {
+ /* A per-CPU source or sink must set CPU_BOUND flag */
+ if (coresight_is_percpu_source(csdev) ||
+ coresight_is_percpu_sink(csdev)) {
+ kfree(csdev);
+ return ERR_PTR(-EINVAL);
+ }
+
+ csdev->cpu = -1;
+ }
+
csdev->dev.type = &coresight_dev_type[desc->type];
csdev->dev.groups = desc->groups;
csdev->dev.parent = desc->dev;
csdev->dev.release = coresight_device_release;
csdev->dev.bus = &coresight_bustype;
+ return csdev;
+}
+
+struct coresight_device *coresight_register(struct coresight_desc *desc)
+{
+ int ret;
+ struct coresight_device *csdev;
+ bool registered = false;
+
+ csdev = coresight_init_device(desc);
+ if (IS_ERR(csdev)) {
+ ret = PTR_ERR(csdev);
+ goto err_out;
+ }
+
if (csdev->type == CORESIGHT_DEV_TYPE_SINK ||
csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) {
raw_spin_lock_init(&csdev->perf_sink_id_map.lock);
@@ -1398,6 +1628,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
if (ret)
goto out_unlock;
+ coresight_set_percpu_source(csdev);
mutex_unlock(&coresight_mutex);
if (cti_assoc_ops && cti_assoc_ops->add)
@@ -1427,6 +1658,7 @@ void coresight_unregister(struct coresight_device *csdev)
cti_assoc_ops->remove(csdev);
mutex_lock(&coresight_mutex);
+ coresight_clear_percpu_source(csdev);
etm_perf_del_symlink_sink(csdev);
coresight_remove_conns(csdev);
coresight_clear_default_sink(csdev);
@@ -1615,6 +1847,211 @@ static void coresight_release_device_list(void)
}
}
+static struct coresight_path *coresight_cpu_get_active_path(enum cs_mode mode)
+{
+ struct coresight_device *source;
+ bool is_active = false;
+
+ source = coresight_get_percpu_source_ref(smp_processor_id());
+ if (!source)
+ return NULL;
+
+ if (coresight_get_mode(source) & mode)
+ is_active = true;
+
+ coresight_put_percpu_source_ref(source);
+
+ /*
+ * It is expected to run in atomic context or with the CPU lock held for
+ * sysfs mode, so it cannot be preempted to disable the path. Here
+ * returns the active path pointer without concern that its state may
+ * change. Since the build path has taken a reference on the component,
+ * the path can be safely used by the caller.
+ */
+ return is_active ? source->path : NULL;
+}
+
+/* Return: 1 if PM is required, 0 if skip, or a negative error */
+static int coresight_pm_is_needed(struct coresight_path *path)
+{
+ struct coresight_device *source, *sink;
+
+ if (this_cpu_read(percpu_pm_failed))
+ return -EIO;
+
+ if (!path)
+ return 0;
+
+ source = coresight_get_source(path);
+ sink = coresight_get_sink(path);
+ if (!source || !sink)
+ return 0;
+
+ /* pm_save_disable() and pm_restore_enable() must be paired */
+ if (coresight_ops(source)->pm_save_disable &&
+ coresight_ops(source)->pm_restore_enable)
+ return 1;
+
+ /*
+ * It is not permitted that the source has no callbacks while the sink
+ * does, as the sink cannot be disabled without disabling the source,
+ * which may lead to lockups. Fix this by enabling self-hosted PM
+ * mode for ETM (see etm4_probe()).
+ */
+ if (coresight_ops(sink)->pm_save_disable &&
+ coresight_ops(sink)->pm_restore_enable) {
+ pr_warn_once("coresight PM failed: source has no PM callbacks; "
+ "cannot safely control sink\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int coresight_pm_device_save(struct coresight_device *csdev)
+{
+ if (!csdev || !coresight_ops(csdev)->pm_save_disable)
+ return 0;
+
+ return coresight_ops(csdev)->pm_save_disable(csdev);
+}
+
+static void coresight_pm_device_restore(struct coresight_device *csdev)
+{
+ if (!csdev || !coresight_ops(csdev)->pm_restore_enable)
+ return;
+
+ coresight_ops(csdev)->pm_restore_enable(csdev);
+}
+
+static int coresight_pm_save(struct coresight_path *path)
+{
+ struct coresight_device *source = coresight_get_source(path);
+ struct coresight_node *from, *to;
+ int ret;
+
+ ret = coresight_pm_device_save(source);
+ if (ret)
+ return ret;
+
+ from = coresight_path_first_node(path);
+ /* Disable up to the node before sink */
+ to = list_prev_entry(coresight_path_last_node(path), link);
+ coresight_disable_path_from_to(path, from, to);
+
+ /*
+ * Save the sink. Most sinks do not implement a save callback to avoid
+ * latency from memory copying. We assume the sink's save and restore
+ * always succeed.
+ */
+ coresight_pm_device_save(coresight_get_sink(path));
+ return 0;
+}
+
+static void coresight_pm_restore(struct coresight_path *path)
+{
+ struct coresight_device *source = coresight_get_source(path);
+ struct coresight_device *sink = coresight_get_sink(path);
+ struct coresight_node *from, *to;
+ int ret;
+
+ coresight_pm_device_restore(sink);
+
+ from = coresight_path_first_node(path);
+ /* Enable up to the node before sink */
+ to = list_prev_entry(coresight_path_last_node(path), link);
+ ret = coresight_enable_path_from_to(path, coresight_get_mode(source),
+ from, to);
+ if (ret)
+ goto path_failed;
+
+ coresight_pm_device_restore(source);
+ return;
+
+path_failed:
+ coresight_pm_device_save(sink);
+
+ pr_err("Failed in coresight PM restore on CPU%d: %d\n",
+ smp_processor_id(), ret);
+
+ /*
+ * Once PM fails on a CPU, set percpu_pm_failed and leave it set until
+ * reboot. This prevents repeated partial transitions during idle
+ * entry and exit.
+ */
+ this_cpu_write(percpu_pm_failed, true);
+}
+
+static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
+ void *v)
+{
+ struct coresight_path *path =
+ coresight_cpu_get_active_path(CS_MODE_SYSFS | CS_MODE_PERF);
+ int ret;
+
+ ret = coresight_pm_is_needed(path);
+ if (ret <= 0)
+ return ret ? NOTIFY_BAD : NOTIFY_DONE;
+
+ switch (cmd) {
+ case CPU_PM_ENTER:
+ if (coresight_pm_save(path))
+ return NOTIFY_BAD;
+ break;
+ case CPU_PM_EXIT:
+ case CPU_PM_ENTER_FAILED:
+ coresight_pm_restore(path);
+ break;
+ default:
+ return NOTIFY_DONE;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block coresight_cpu_pm_nb = {
+ .notifier_call = coresight_cpu_pm_notify,
+};
+
+static int coresight_dying_cpu(unsigned int cpu)
+{
+ struct coresight_path *path;
+
+ /*
+ * The perf event layer will disable PMU events in the CPU
+ * hotplug. Here only handles SYSFS case.
+ */
+ path = coresight_cpu_get_active_path(CS_MODE_SYSFS);
+ if (!path)
+ return 0;
+
+ coresight_disable_sysfs(coresight_get_source(path));
+ return 0;
+}
+
+static int __init coresight_pm_setup(void)
+{
+ int ret;
+
+ ret = cpu_pm_register_notifier(&coresight_cpu_pm_nb);
+ if (ret)
+ return ret;
+
+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE,
+ "arm/coresight-core:dying",
+ NULL, coresight_dying_cpu);
+ if (ret)
+ cpu_pm_unregister_notifier(&coresight_cpu_pm_nb);
+
+ return ret;
+}
+
+static void coresight_pm_cleanup(void)
+{
+ cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE);
+ cpu_pm_unregister_notifier(&coresight_cpu_pm_nb);
+}
+
const struct bus_type coresight_bustype = {
.name = "coresight",
};
@@ -1669,9 +2106,15 @@ static int __init coresight_init(void)
/* initialise the coresight syscfg API */
ret = cscfg_init();
+ if (ret)
+ goto exit_notifier;
+
+ ret = coresight_pm_setup();
if (!ret)
return 0;
+ cscfg_exit();
+exit_notifier:
atomic_notifier_chain_unregister(&panic_notifier_list,
&coresight_notifier);
exit_perf:
@@ -1683,6 +2126,7 @@ exit_bus_unregister:
static void __exit coresight_exit(void)
{
+ coresight_pm_cleanup();
cscfg_exit();
atomic_notifier_chain_unregister(&panic_notifier_list,
&coresight_notifier);
@@ -1728,10 +2172,10 @@ int coresight_etm_get_trace_id(struct coresight_device *csdev, enum cs_mode mode
{
int cpu, trace_id;
- if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE || !source_ops(csdev)->cpu_id)
+ if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE)
return -EINVAL;
- cpu = source_ops(csdev)->cpu_id(csdev);
+ cpu = csdev->cpu;
switch (mode) {
case CS_MODE_SYSFS:
trace_id = coresight_trace_id_get_cpu_id(cpu);
diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c
index 2f4c9362709a9..b2c9a4db13b4e 100644
--- a/drivers/hwtracing/coresight/coresight-cti-core.c
+++ b/drivers/hwtracing/coresight/coresight-cti-core.c
@@ -659,7 +659,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
void __iomem *base;
struct device *dev = &adev->dev;
struct cti_drvdata *drvdata = NULL;
- struct coresight_desc cti_desc;
+ struct coresight_desc cti_desc = { 0 };
struct coresight_platform_data *pdata = NULL;
struct resource *res = &adev->res;
@@ -702,11 +702,14 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
* eCPU ID. System CTIs will have the name cti_sys<I> where I is an
* index allocated by order of discovery.
*/
- if (drvdata->ctidev.cpu >= 0)
+ if (drvdata->ctidev.cpu >= 0) {
+ cti_desc.cpu = drvdata->ctidev.cpu;
+ cti_desc.flags = CORESIGHT_DESC_CPU_BOUND;
cti_desc.name = devm_kasprintf(dev, GFP_KERNEL, "cti_cpu%d",
drvdata->ctidev.cpu);
- else
+ } else {
cti_desc.name = coresight_alloc_device_name("cti_sys", dev);
+ }
if (!cti_desc.name)
return -ENOMEM;
diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c
index 4eff96f48594e..d6d5388705c3e 100644
--- a/drivers/hwtracing/coresight/coresight-cti-platform.c
+++ b/drivers/hwtracing/coresight/coresight-cti-platform.c
@@ -329,6 +329,7 @@ static int cti_plat_process_filter_sigs(struct cti_drvdata *drvdata,
if (!tg)
return -ENOMEM;
+ tg->nr_sigs = nr_filter_sigs;
err = cti_plat_read_trig_group(tg, fwnode, CTI_DT_FILTER_OUT_SIGS);
if (!err)
drvdata->config.trig_out_filter |= tg->used_mask;
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index b952a1d47f12f..a827f76b8144a 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -83,7 +83,7 @@ struct etb_drvdata {
struct coresight_device *csdev;
struct miscdevice miscdev;
raw_spinlock_t spinlock;
- local_t reading;
+ atomic_t reading;
pid_t pid;
u8 *buf;
u32 buffer_depth;
@@ -601,7 +601,7 @@ static int etb_open(struct inode *inode, struct file *file)
struct etb_drvdata *drvdata = container_of(file->private_data,
struct etb_drvdata, miscdev);
- if (local_cmpxchg(&drvdata->reading, 0, 1))
+ if (atomic_cmpxchg(&drvdata->reading, 0, 1))
return -EBUSY;
dev_dbg(&drvdata->csdev->dev, "%s: successfully opened\n", __func__);
@@ -639,7 +639,7 @@ static int etb_release(struct inode *inode, struct file *file)
{
struct etb_drvdata *drvdata = container_of(file->private_data,
struct etb_drvdata, miscdev);
- local_set(&drvdata->reading, 0);
+ atomic_set(&drvdata->reading, 0);
dev_dbg(&drvdata->csdev->dev, "%s: released\n", __func__);
return 0;
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index f85dedf89a3f9..09b21a711a876 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -49,7 +49,6 @@ struct etm_ctxt {
};
static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt);
-static DEFINE_PER_CPU(struct coresight_device *, csdev_src);
GEN_PMU_FORMAT_ATTR(cycacc);
GEN_PMU_FORMAT_ATTR(timestamp);
@@ -315,6 +314,115 @@ static bool sinks_compatible(struct coresight_device *a,
(sink_ops(a) == sink_ops(b));
}
+/*
+ * This helper is used for fetching the path pointer via the ctxt.
+ *
+ * Perf event callbacks run on the same CPU in atomic context, but AUX pause
+ * and resume may run in NMI context and preempt other callbacks. Since the
+ * event stop callback clears ctxt->event_data before the data is released,
+ * AUX pause/resume will either observe a NULL pointer and stop fetching the
+ * path pointer, or safely access event_data and the path, as the data has
+ * not yet been freed.
+ */
+static struct coresight_path *etm_event_get_ctxt_path(struct etm_ctxt *ctxt)
+{
+ struct etm_event_data *event_data;
+ struct coresight_path *path;
+
+ if (!ctxt)
+ return NULL;
+
+ event_data = READ_ONCE(ctxt->event_data);
+ if (!event_data)
+ return NULL;
+
+ path = etm_event_cpu_path(event_data, smp_processor_id());
+ if (!path)
+ return NULL;
+
+ return path;
+}
+
+static struct coresight_path *
+etm_event_build_path(struct perf_event *event, int cpu,
+ struct coresight_device *user_sink,
+ struct coresight_device *match_sink)
+{
+ struct coresight_path *path = NULL;
+ struct coresight_device *source, *sink;
+ int ret;
+
+ source = coresight_get_percpu_source_ref(cpu);
+
+ /*
+ * If there is no ETM associated with this CPU or ever we try to trace
+ * on this CPU, we handle it accordingly.
+ */
+ if (!source)
+ return NULL;
+
+ /*
+ * If AUX pause feature is enabled but the ETM driver does not
+ * support the operations, skip for this source.
+ */
+ if (event->attr.aux_start_paused &&
+ (!source_ops(source)->pause_perf ||
+ !source_ops(source)->resume_perf)) {
+ dev_err_once(&source->dev, "AUX pause is not supported.\n");
+ goto out;
+ }
+
+ /* If sink has been specified by user, directly use it */
+ if (user_sink) {
+ sink = user_sink;
+ } else {
+ /*
+ * No sink provided - look for a default sink for all the ETMs,
+ * where this event can be scheduled.
+ *
+ * We allocate the sink specific buffers only once for this
+ * event. If the ETMs have different default sink devices, we
+ * can only use a single "type" of sink as the event can carry
+ * only one sink specific buffer. Thus we have to make sure
+ * that the sinks are of the same type and driven by the same
+ * driver, as the one we allocate the buffer for. We don't
+ * trace on a CPU if the sink is not compatible.
+ */
+
+ /* Find the default sink for this ETM */
+ sink = coresight_find_default_sink(source);
+ if (!sink)
+ goto out;
+
+ /* Check if this sink compatible with the last sink */
+ if (match_sink && !sinks_compatible(match_sink, sink))
+ goto out;
+ }
+
+ /*
+ * Building a path doesn't enable it, it simply builds a
+ * list of devices from source to sink that can be
+ * referenced later when the path is actually needed.
+ */
+ path = coresight_build_path(source, sink);
+ if (IS_ERR(path))
+ goto out;
+
+ /* ensure we can allocate a trace ID for this CPU */
+ ret = coresight_path_assign_trace_id(path, CS_MODE_PERF);
+ if (ret) {
+ coresight_release_path(path);
+ path = NULL;
+ goto out;
+ }
+
+ coresight_trace_id_perf_start(&sink->perf_sink_id_map);
+
+out:
+ coresight_put_percpu_source_ref(source);
+ return IS_ERR_OR_NULL(path) ? NULL : path;
+}
+
static void *etm_setup_aux(struct perf_event *event, void **pages,
int nr_pages, bool overwrite)
{
@@ -322,7 +430,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages,
int cpu = event->cpu;
cpumask_t *mask;
struct coresight_device *sink = NULL;
- struct coresight_device *user_sink = NULL, *last_sink = NULL;
+ struct coresight_device *user_sink = NULL;
struct etm_event_data *event_data = NULL;
event_data = alloc_event_data(cpu);
@@ -354,80 +462,25 @@ static void *etm_setup_aux(struct perf_event *event, void **pages,
*/
for_each_cpu(cpu, mask) {
struct coresight_path *path;
- struct coresight_device *csdev;
-
- csdev = per_cpu(csdev_src, cpu);
- /*
- * If there is no ETM associated with this CPU clear it from
- * the mask and continue with the rest. If ever we try to trace
- * on this CPU, we handle it accordingly.
- */
- if (!csdev) {
- cpumask_clear_cpu(cpu, mask);
- continue;
- }
- /*
- * If AUX pause feature is enabled but the ETM driver does not
- * support the operations, clear this CPU from the mask and
- * continue to next one.
- */
- if (event->attr.aux_start_paused &&
- (!source_ops(csdev)->pause_perf || !source_ops(csdev)->resume_perf)) {
- dev_err_once(&csdev->dev, "AUX pause is not supported.\n");
+ path = etm_event_build_path(event, cpu, user_sink, sink);
+ if (!path) {
+ /*
+ * Failed to create a path for the CPU, clear it from
+ * the mask and continue to next one.
+ */
cpumask_clear_cpu(cpu, mask);
continue;
}
/*
- * No sink provided - look for a default sink for all the ETMs,
- * where this event can be scheduled.
- * We allocate the sink specific buffers only once for this
- * event. If the ETMs have different default sink devices, we
- * can only use a single "type" of sink as the event can carry
- * only one sink specific buffer. Thus we have to make sure
- * that the sinks are of the same type and driven by the same
- * driver, as the one we allocate the buffer for. As such
- * we choose the first sink and check if the remaining ETMs
- * have a compatible default sink. We don't trace on a CPU
- * if the sink is not compatible.
+ * The first found sink is saved here and passed to
+ * etm_event_build_path() to check whether the remaining ETMs
+ * have a compatible default sink.
*/
- if (!user_sink) {
- /* Find the default sink for this ETM */
- sink = coresight_find_default_sink(csdev);
- if (!sink) {
- cpumask_clear_cpu(cpu, mask);
- continue;
- }
+ if (!user_sink && !sink)
+ sink = coresight_get_sink(path);
- /* Check if this sink compatible with the last sink */
- if (last_sink && !sinks_compatible(last_sink, sink)) {
- cpumask_clear_cpu(cpu, mask);
- continue;
- }
- last_sink = sink;
- }
-
- /*
- * Building a path doesn't enable it, it simply builds a
- * list of devices from source to sink that can be
- * referenced later when the path is actually needed.
- */
- path = coresight_build_path(csdev, sink);
- if (IS_ERR(path)) {
- cpumask_clear_cpu(cpu, mask);
- continue;
- }
-
- /* ensure we can allocate a trace ID for this CPU */
- coresight_path_assign_trace_id(path, CS_MODE_PERF);
- if (!IS_VALID_CS_TRACE_ID(path->trace_id)) {
- cpumask_clear_cpu(cpu, mask);
- coresight_release_path(path);
- continue;
- }
-
- coresight_trace_id_perf_start(&sink->perf_sink_id_map);
*etm_event_cpu_path_ptr(event_data, cpu) = path;
}
@@ -464,13 +517,23 @@ err:
goto out;
}
-static int etm_event_resume(struct coresight_device *csdev,
- struct etm_ctxt *ctxt)
+static int etm_event_resume(struct coresight_path *path)
{
- if (!ctxt->event_data)
+ struct coresight_device *source;
+ int ret;
+
+ if (!path)
return 0;
- return coresight_resume_source(csdev);
+ source = coresight_get_source(path);
+ if (!source)
+ return 0;
+
+ ret = coresight_resume_source(source);
+ if (ret < 0)
+ dev_err(&source->dev, "Failed to resume ETM event.\n");
+
+ return ret;
}
static void etm_event_start(struct perf_event *event, int flags)
@@ -479,23 +542,19 @@ static void etm_event_start(struct perf_event *event, int flags)
struct etm_event_data *event_data;
struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt);
struct perf_output_handle *handle = &ctxt->handle;
- struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
+ struct coresight_device *source, *sink;
struct coresight_path *path;
u64 hw_id;
- if (!csdev)
- goto fail;
-
if (flags & PERF_EF_RESUME) {
- if (etm_event_resume(csdev, ctxt) < 0) {
- dev_err(&csdev->dev, "Failed to resume ETM event.\n");
+ path = etm_event_get_ctxt_path(ctxt);
+ if (etm_event_resume(path) < 0)
goto fail;
- }
return;
}
/* Have we messed up our tracking ? */
- if (WARN_ON(ctxt->event_data))
+ if (WARN_ON(READ_ONCE(ctxt->event_data)))
goto fail;
/*
@@ -523,9 +582,10 @@ static void etm_event_start(struct perf_event *event, int flags)
path = etm_event_cpu_path(event_data, cpu);
path->handle = handle;
- /* We need a sink, no need to continue without one */
+ /* We need source and sink, no need to continue if any is not set */
+ source = coresight_get_source(path);
sink = coresight_get_sink(path);
- if (WARN_ON_ONCE(!sink))
+ if (WARN_ON_ONCE(!source || !sink))
goto fail_end_stop;
/* Nothing will happen without a path */
@@ -533,7 +593,7 @@ static void etm_event_start(struct perf_event *event, int flags)
goto fail_end_stop;
/* Finally enable the tracer */
- if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF, path))
+ if (coresight_enable_source(source, event, CS_MODE_PERF, path))
goto fail_disable_path;
/*
@@ -557,7 +617,7 @@ out:
/* Tell the perf core the event is alive */
event->hw.state = 0;
/* Save the event_data for this ETM */
- ctxt->event_data = event_data;
+ WRITE_ONCE(ctxt->event_data, event_data);
return;
fail_disable_path:
@@ -577,27 +637,26 @@ fail:
return;
}
-static void etm_event_pause(struct perf_event *event,
- struct coresight_device *csdev,
+static void etm_event_pause(struct coresight_path *path,
+ struct perf_event *event,
struct etm_ctxt *ctxt)
{
- int cpu = smp_processor_id();
- struct coresight_device *sink;
struct perf_output_handle *handle = &ctxt->handle;
- struct coresight_path *path;
+ struct coresight_device *source, *sink;
+ struct etm_event_data *event_data;
unsigned long size;
- if (!ctxt->event_data)
+ if (!path)
return;
- /* Stop tracer */
- coresight_pause_source(csdev);
-
- path = etm_event_cpu_path(ctxt->event_data, cpu);
+ source = coresight_get_source(path);
sink = coresight_get_sink(path);
- if (WARN_ON_ONCE(!sink))
+ if (WARN_ON_ONCE(!source || !sink))
return;
+ /* Stop tracer */
+ coresight_pause_source(source);
+
/*
* The per CPU sink has own interrupt handling, it might have
* race condition with updating buffer on AUX trace pause if
@@ -613,8 +672,9 @@ static void etm_event_pause(struct perf_event *event,
if (!sink_ops(sink)->update_buffer)
return;
+ event_data = READ_ONCE(ctxt->event_data);
size = sink_ops(sink)->update_buffer(sink, handle,
- ctxt->event_data->snk_config);
+ event_data->snk_config);
if (READ_ONCE(handle->event)) {
if (!size)
return;
@@ -630,14 +690,14 @@ static void etm_event_stop(struct perf_event *event, int mode)
{
int cpu = smp_processor_id();
unsigned long size;
- struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
+ struct coresight_device *source, *sink;
struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt);
struct perf_output_handle *handle = &ctxt->handle;
+ struct coresight_path *path = etm_event_get_ctxt_path(ctxt);
struct etm_event_data *event_data;
- struct coresight_path *path;
if (mode & PERF_EF_PAUSE)
- return etm_event_pause(event, csdev, ctxt);
+ return etm_event_pause(path, event, ctxt);
/*
* If we still have access to the event_data via handle,
@@ -647,9 +707,9 @@ static void etm_event_stop(struct perf_event *event, int mode)
WARN_ON(perf_get_aux(handle) != ctxt->event_data))
return;
- event_data = ctxt->event_data;
+ event_data = READ_ONCE(ctxt->event_data);
/* Clear the event_data as this ETM is stopping the trace. */
- ctxt->event_data = NULL;
+ WRITE_ONCE(ctxt->event_data, NULL);
if (event->hw.state == PERF_HES_STOPPED)
return;
@@ -671,19 +731,13 @@ static void etm_event_stop(struct perf_event *event, int mode)
return;
}
- if (!csdev)
- return;
-
- path = etm_event_cpu_path(event_data, cpu);
- if (!path)
- return;
-
+ source = coresight_get_source(path);
sink = coresight_get_sink(path);
- if (!sink)
+ if (!source || !sink)
return;
/* stop tracer */
- coresight_disable_source(csdev, event);
+ coresight_disable_source(source, event);
/* tell the core */
event->hw.state = PERF_HES_STOPPED;
@@ -824,7 +878,7 @@ static void etm_addr_filters_sync(struct perf_event *event)
int etm_perf_symlink(struct coresight_device *csdev, bool link)
{
char entry[sizeof("cpu9999999")];
- int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev);
+ int ret = 0, cpu = csdev->cpu;
struct device *pmu_dev = etm_pmu.dev;
struct device *cs_dev = &csdev->dev;
@@ -833,17 +887,12 @@ int etm_perf_symlink(struct coresight_device *csdev, bool link)
if (!etm_perf_up)
return -EPROBE_DEFER;
- if (link) {
+ if (link)
ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry);
- if (ret)
- return ret;
- per_cpu(csdev_src, cpu) = csdev;
- } else {
+ else
sysfs_remove_link(&pmu_dev->kobj, entry);
- per_cpu(csdev_src, cpu) = NULL;
- }
- return 0;
+ return ret;
}
EXPORT_SYMBOL_GPL(etm_perf_symlink);
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c
index a547a6d2e0bde..862ad0786699c 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c
@@ -441,6 +441,7 @@ done:
struct etm_enable_arg {
struct etm_drvdata *drvdata;
+ struct coresight_path *path;
int rc;
};
@@ -462,15 +463,12 @@ static void etm_enable_sysfs_smp_call(void *info)
arg->rc = etm_enable_hw(arg->drvdata);
/* The tracer didn't start */
- if (arg->rc)
+ if (arg->rc) {
coresight_set_mode(csdev, CS_MODE_DISABLED);
-}
-
-static int etm_cpu_id(struct coresight_device *csdev)
-{
- struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
+ return;
+ }
- return drvdata->cpu;
+ csdev->path = arg->path;
}
void etm_release_trace_id(struct etm_drvdata *drvdata)
@@ -499,10 +497,13 @@ static int etm_enable_perf(struct coresight_device *csdev,
ret = etm_enable_hw(drvdata);
/* Failed to start tracer; roll back to DISABLED mode */
- if (ret)
+ if (ret) {
coresight_set_mode(csdev, CS_MODE_DISABLED);
+ return ret;
+ }
- return ret;
+ csdev->path = path;
+ return 0;
}
static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path)
@@ -521,6 +522,7 @@ static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_pat
*/
if (cpu_online(drvdata->cpu)) {
arg.drvdata = drvdata;
+ arg.path = path;
ret = smp_call_function_single(drvdata->cpu,
etm_enable_sysfs_smp_call, &arg, 1);
if (!ret)
@@ -590,6 +592,7 @@ static void etm_disable_sysfs_smp_call(void *info)
etm_disable_hw(drvdata);
+ drvdata->csdev->path = NULL;
coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
}
@@ -614,6 +617,7 @@ static void etm_disable_perf(struct coresight_device *csdev)
CS_LOCK(drvdata->csa.base);
+ drvdata->csdev->path = NULL;
coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
/*
@@ -627,13 +631,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev)
{
struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
- /*
- * Taking hotplug lock here protects from clocks getting disabled
- * with tracing being left on (crash scenario) if user disable occurs
- * after cpu online mask indicates the cpu is offline but before the
- * DYING hotplug callback is serviced by the ETM driver.
- */
- cpus_read_lock();
spin_lock(&drvdata->spinlock);
/*
@@ -644,7 +641,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev)
drvdata, 1);
spin_unlock(&drvdata->spinlock);
- cpus_read_unlock();
/*
* we only release trace IDs when resetting sysfs.
@@ -684,7 +680,6 @@ static void etm_disable(struct coresight_device *csdev,
}
static const struct coresight_ops_source etm_source_ops = {
- .cpu_id = etm_cpu_id,
.enable = etm_enable,
.disable = etm_disable,
};
@@ -704,35 +699,6 @@ static int etm_online_cpu(unsigned int cpu)
return 0;
}
-static int etm_starting_cpu(unsigned int cpu)
-{
- if (!etmdrvdata[cpu])
- return 0;
-
- spin_lock(&etmdrvdata[cpu]->spinlock);
- if (!etmdrvdata[cpu]->os_unlock) {
- etm_os_unlock(etmdrvdata[cpu]);
- etmdrvdata[cpu]->os_unlock = true;
- }
-
- if (coresight_get_mode(etmdrvdata[cpu]->csdev))
- etm_enable_hw(etmdrvdata[cpu]);
- spin_unlock(&etmdrvdata[cpu]->spinlock);
- return 0;
-}
-
-static int etm_dying_cpu(unsigned int cpu)
-{
- if (!etmdrvdata[cpu])
- return 0;
-
- spin_lock(&etmdrvdata[cpu]->spinlock);
- if (coresight_get_mode(etmdrvdata[cpu]->csdev))
- etm_disable_hw(etmdrvdata[cpu]);
- spin_unlock(&etmdrvdata[cpu]->spinlock);
- return 0;
-}
-
static bool etm_arch_supported(u8 arch)
{
switch (arch) {
@@ -800,13 +766,6 @@ static int __init etm_hp_setup(void)
{
int ret;
- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
- "arm/coresight:starting",
- etm_starting_cpu, etm_dying_cpu);
-
- if (ret)
- return ret;
-
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"arm/coresight:online",
etm_online_cpu, NULL);
@@ -817,15 +776,11 @@ static int __init etm_hp_setup(void)
return 0;
}
- /* failed dyn state - remove others */
- cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
-
return ret;
}
static void etm_hp_clear(void)
{
- cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
if (hp_online) {
cpuhp_remove_state_nocalls(hp_online);
hp_online = 0;
@@ -891,6 +846,8 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id)
desc.pdata = pdata;
desc.dev = dev;
desc.groups = coresight_etm_groups;
+ desc.cpu = drvdata->cpu;
+ desc.flags = CORESIGHT_DESC_CPU_BOUND;
drvdata->csdev = coresight_register(&desc);
if (IS_ERR(drvdata->csdev))
return PTR_ERR(drvdata->csdev);
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c
index d565a73f0042e..14bb31bd6a0b9 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c
@@ -56,10 +56,14 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
#define PARAM_PM_SAVE_NEVER 1 /* never save any state */
#define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
+/*
+ * Save option for ETM4. ETE, sysreg ETM4s and ACPI boots ignore this option and
+ * will always save.
+ */
static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
module_param(pm_save_enable, int, 0444);
MODULE_PARM_DESC(pm_save_enable,
- "Save/restore state on power down: 1 = never, 2 = self-hosted");
+ "Save/restore state on power down: 1 = never, 2 = self-hosted. MMIO and DT only.");
static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
static void etm4_set_default_config(struct etmv4_config *config);
@@ -227,13 +231,6 @@ static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,
CS_UNLOCK(csa->base);
}
-static int etm4_cpu_id(struct coresight_device *csdev)
-{
- struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
-
- return drvdata->cpu;
-}
-
void etm4_release_trace_id(struct etmv4_drvdata *drvdata)
{
coresight_trace_id_put_cpu_id(drvdata->cpu);
@@ -241,6 +238,7 @@ void etm4_release_trace_id(struct etmv4_drvdata *drvdata)
struct etm4_enable_arg {
struct etmv4_drvdata *drvdata;
+ struct coresight_path *path;
int rc;
};
@@ -628,8 +626,12 @@ static void etm4_enable_sysfs_smp_call(void *info)
arg->rc = etm4_enable_hw(arg->drvdata);
/* The tracer didn't start */
- if (arg->rc)
+ if (arg->rc) {
coresight_set_mode(csdev, CS_MODE_DISABLED);
+ return;
+ }
+
+ csdev->path = arg->path;
}
/*
@@ -897,9 +899,13 @@ static int etm4_enable_perf(struct coresight_device *csdev,
out:
/* Failed to start tracer; roll back to DISABLED mode */
- if (ret)
+ if (ret) {
coresight_set_mode(csdev, CS_MODE_DISABLED);
- return ret;
+ return ret;
+ }
+
+ csdev->path = path;
+ return 0;
}
static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path)
@@ -929,6 +935,7 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa
* ensures that register writes occur when cpu is powered.
*/
arg.drvdata = drvdata;
+ arg.path = path;
ret = smp_call_function_single(drvdata->cpu,
etm4_enable_sysfs_smp_call, &arg, 1);
if (!ret)
@@ -1070,6 +1077,7 @@ static void etm4_disable_sysfs_smp_call(void *info)
etm4_disable_hw(drvdata);
+ drvdata->csdev->path = NULL;
coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
}
@@ -1099,6 +1107,7 @@ static int etm4_disable_perf(struct coresight_device *csdev,
/* TRCVICTLR::SSSTATUS, bit[9] */
filters->ssstatus = (control & BIT(9));
+ drvdata->csdev->path = NULL;
coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
/*
@@ -1113,13 +1122,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)
{
struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
- /*
- * Taking hotplug lock here protects from clocks getting disabled
- * with tracing being left on (crash scenario) if user disable occurs
- * after cpu online mask indicates the cpu is offline but before the
- * DYING hotplug callback is serviced by the ETM driver.
- */
- cpus_read_lock();
raw_spin_lock(&drvdata->spinlock);
/*
@@ -1133,8 +1135,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)
cscfg_csdev_disable_active_config(csdev);
- cpus_read_unlock();
-
/*
* we only release trace IDs when resetting sysfs.
* This permits sysfs users to read the trace ID after the trace
@@ -1201,18 +1201,12 @@ static void etm4_pause_perf(struct coresight_device *csdev)
}
static const struct coresight_ops_source etm4_source_ops = {
- .cpu_id = etm4_cpu_id,
.enable = etm4_enable,
.disable = etm4_disable,
.resume_perf = etm4_resume_perf,
.pause_perf = etm4_pause_perf,
};
-static const struct coresight_ops etm4_cs_ops = {
- .trace_id = coresight_etm_get_trace_id,
- .source_ops = &etm4_source_ops,
-};
-
static bool cpu_supports_sysreg_trace(void)
{
u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
@@ -1839,44 +1833,19 @@ static int etm4_online_cpu(unsigned int cpu)
return 0;
}
-static int etm4_starting_cpu(unsigned int cpu)
+static inline bool etm4_pm_save_needed(struct etmv4_drvdata *drvdata)
{
- if (!etmdrvdata[cpu])
- return 0;
-
- raw_spin_lock(&etmdrvdata[cpu]->spinlock);
- if (!etmdrvdata[cpu]->os_unlock)
- etm4_os_unlock(etmdrvdata[cpu]);
-
- if (coresight_get_mode(etmdrvdata[cpu]->csdev))
- etm4_enable_hw(etmdrvdata[cpu]);
- raw_spin_unlock(&etmdrvdata[cpu]->spinlock);
- return 0;
-}
-
-static int etm4_dying_cpu(unsigned int cpu)
-{
- if (!etmdrvdata[cpu])
- return 0;
-
- raw_spin_lock(&etmdrvdata[cpu]->spinlock);
- if (coresight_get_mode(etmdrvdata[cpu]->csdev))
- etm4_disable_hw(etmdrvdata[cpu]);
- raw_spin_unlock(&etmdrvdata[cpu]->spinlock);
- return 0;
+ return !!drvdata->save_state;
}
-static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
+static int etm4_cpu_save(struct coresight_device *csdev)
{
int i, ret = 0;
+ struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
struct etmv4_save_state *state;
- struct coresight_device *csdev = drvdata->csdev;
struct csdev_access *csa;
struct device *etm_dev;
- if (WARN_ON(!csdev))
- return -ENODEV;
-
etm_dev = &csdev->dev;
csa = &csdev->access;
@@ -1979,7 +1948,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);
if (drvdata->numvmidc > 4)
- state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);
+ state->trcvmidcctlr1 = etm4x_read32(csa, TRCVMIDCCTLR1);
state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);
@@ -2008,31 +1977,13 @@ out:
return ret;
}
-static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
-{
- int ret = 0;
-
- if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED)
- return 0;
-
- /*
- * Save and restore the ETM Trace registers only if
- * the ETM is active.
- */
- if (coresight_get_mode(drvdata->csdev))
- ret = __etm4_cpu_save(drvdata);
- return ret;
-}
-
-static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
+static void etm4_cpu_restore(struct coresight_device *csdev)
{
int i;
+ struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
struct etmv4_save_state *state = drvdata->save_state;
struct csdev_access *csa = &drvdata->csdev->access;
- if (WARN_ON(!drvdata->csdev))
- return;
-
etm4_cs_unlock(drvdata, csa);
etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
@@ -2102,7 +2053,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);
if (drvdata->numvmidc > 4)
- etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);
+ etm4x_relaxed_write32(csa, state->trcvmidcctlr1, TRCVMIDCCTLR1);
etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
@@ -2125,47 +2076,16 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
etm4_cs_lock(drvdata, csa);
}
-static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
-{
- if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED)
- return;
-
- if (coresight_get_mode(drvdata->csdev))
- __etm4_cpu_restore(drvdata);
-}
-
-static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
- void *v)
-{
- struct etmv4_drvdata *drvdata;
- unsigned int cpu = smp_processor_id();
-
- if (!etmdrvdata[cpu])
- return NOTIFY_OK;
-
- drvdata = etmdrvdata[cpu];
-
- if (WARN_ON_ONCE(drvdata->cpu != cpu))
- return NOTIFY_BAD;
-
- switch (cmd) {
- case CPU_PM_ENTER:
- if (etm4_cpu_save(drvdata))
- return NOTIFY_BAD;
- break;
- case CPU_PM_EXIT:
- case CPU_PM_ENTER_FAILED:
- etm4_cpu_restore(drvdata);
- break;
- default:
- return NOTIFY_DONE;
- }
-
- return NOTIFY_OK;
-}
+static const struct coresight_ops etm4_cs_ops = {
+ .trace_id = coresight_etm_get_trace_id,
+ .source_ops = &etm4_source_ops,
+};
-static struct notifier_block etm4_cpu_pm_nb = {
- .notifier_call = etm4_cpu_pm_notify,
+static const struct coresight_ops etm4_cs_pm_ops = {
+ .trace_id = coresight_etm_get_trace_id,
+ .source_ops = &etm4_source_ops,
+ .pm_save_disable = etm4_cpu_save,
+ .pm_restore_enable = etm4_cpu_restore,
};
/* Setup PM. Deals with error conditions and counts */
@@ -2173,17 +2093,6 @@ static int __init etm4_pm_setup(void)
{
int ret;
- ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
- if (ret)
- return ret;
-
- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
- "arm/coresight4:starting",
- etm4_starting_cpu, etm4_dying_cpu);
-
- if (ret)
- goto unregister_notifier;
-
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"arm/coresight4:online",
etm4_online_cpu, NULL);
@@ -2194,24 +2103,28 @@ static int __init etm4_pm_setup(void)
return 0;
}
- /* failed dyn state - remove others */
- cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
-
-unregister_notifier:
- cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
return ret;
}
static void etm4_pm_clear(void)
{
- cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
- cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
if (hp_online) {
cpuhp_remove_state_nocalls(hp_online);
hp_online = 0;
}
}
+static bool etm4x_always_pm_save(struct device *dev, struct csdev_access *csa)
+{
+ /*
+ * Only IO mem ETM devices will benefit from skipping PM save and only
+ * DT has the option to control it, not ACPI. Otherwise system register
+ * based ETMs and ETEs will always lose context on CPU power down, so
+ * always save.
+ */
+ return !csa->io_mem || is_acpi_device_node(dev_fwnode(dev));
+}
+
static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
{
int ret;
@@ -2221,6 +2134,7 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
struct coresight_desc desc = { 0 };
u8 major, minor;
char *type_name;
+ bool pm_save;
if (!drvdata)
return -EINVAL;
@@ -2248,6 +2162,21 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
etm4_set_default(&drvdata->config);
+ if (etm4x_always_pm_save(dev, init_arg->csa))
+ pm_save = true;
+ else if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
+ pm_save = coresight_loses_context_with_cpu(dev);
+ else
+ pm_save = pm_save_enable != PARAM_PM_SAVE_NEVER;
+
+ if (pm_save) {
+ drvdata->save_state = devm_kmalloc(dev,
+ sizeof(struct etmv4_save_state),
+ GFP_KERNEL);
+ if (!drvdata->save_state)
+ return -ENOMEM;
+ }
+
pdata = coresight_get_platform_data(dev);
if (IS_ERR(pdata))
return PTR_ERR(pdata);
@@ -2256,10 +2185,12 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
desc.type = CORESIGHT_DEV_TYPE_SOURCE;
desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
- desc.ops = &etm4_cs_ops;
+ desc.ops = etm4_pm_save_needed(drvdata) ? &etm4_cs_pm_ops : &etm4_cs_ops;
desc.pdata = pdata;
desc.dev = dev;
desc.groups = coresight_etmv4_groups;
+ desc.cpu = drvdata->cpu;
+ desc.flags = CORESIGHT_DESC_CPU_BOUND;
drvdata->csdev = coresight_register(&desc);
if (IS_ERR(drvdata->csdev))
return PTR_ERR(drvdata->csdev);
@@ -2305,17 +2236,6 @@ static int etm4_probe(struct device *dev)
if (ret)
return ret;
- if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
- pm_save_enable = coresight_loses_context_with_cpu(dev) ?
- PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;
-
- if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
- drvdata->save_state = devm_kmalloc(dev,
- sizeof(struct etmv4_save_state), GFP_KERNEL);
- if (!drvdata->save_state)
- return -ENOMEM;
- }
-
raw_spin_lock_init(&drvdata->spinlock);
drvdata->cpu = coresight_get_cpu(dev);
diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c
index e337b6e2bf327..93c2d075cad66 100644
--- a/drivers/hwtracing/coresight/coresight-platform.c
+++ b/drivers/hwtracing/coresight/coresight-platform.c
@@ -45,9 +45,8 @@ coresight_add_out_conn(struct device *dev,
}
}
- pdata->nr_outconns++;
pdata->out_conns =
- devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns,
+ devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns + 1,
sizeof(*pdata->out_conns), GFP_KERNEL);
if (!pdata->out_conns)
return ERR_PTR(-ENOMEM);
@@ -63,7 +62,8 @@ coresight_add_out_conn(struct device *dev,
* used right away.
*/
*conn = *new_conn;
- pdata->out_conns[pdata->nr_outconns - 1] = conn;
+ pdata->out_conns[pdata->nr_outconns] = conn;
+ pdata->nr_outconns++;
return conn;
}
EXPORT_SYMBOL_GPL(coresight_add_out_conn);
@@ -86,13 +86,13 @@ int coresight_add_in_conn(struct coresight_connection *out_conn)
return 0;
}
- pdata->nr_inconns++;
pdata->in_conns =
- devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns,
+ devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns + 1,
sizeof(*pdata->in_conns), GFP_KERNEL);
if (!pdata->in_conns)
return -ENOMEM;
- pdata->in_conns[pdata->nr_inconns - 1] = out_conn;
+ pdata->in_conns[pdata->nr_inconns] = out_conn;
+ pdata->nr_inconns++;
return 0;
}
EXPORT_SYMBOL_GPL(coresight_add_in_conn);
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 1ea882dffd703..dddac946659fc 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -153,7 +153,7 @@ int coresight_make_links(struct coresight_device *orig,
void coresight_remove_links(struct coresight_device *orig,
struct coresight_connection *conn);
u32 coresight_get_sink_id(struct coresight_device *csdev);
-void coresight_path_assign_trace_id(struct coresight_path *path,
+int coresight_path_assign_trace_id(struct coresight_path *path,
enum cs_mode mode);
#if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM3X)
@@ -248,6 +248,12 @@ void coresight_add_helper(struct coresight_device *csdev,
void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev);
struct coresight_device *coresight_get_percpu_sink(int cpu);
+struct coresight_device *coresight_get_source(struct coresight_path *path);
+struct coresight_device *coresight_get_percpu_source_ref(int cpu);
+void coresight_put_percpu_source_ref(struct coresight_device *csdev);
+int coresight_enable_source(struct coresight_device *csdev,
+ struct perf_event *event, enum cs_mode mode,
+ struct coresight_path *path);
void coresight_disable_source(struct coresight_device *csdev, void *data);
void coresight_pause_source(struct coresight_device *csdev);
int coresight_resume_source(struct coresight_device *csdev);
diff --git a/drivers/hwtracing/coresight/coresight-syscfg.c b/drivers/hwtracing/coresight/coresight-syscfg.c
index d7f5037953d6b..2bfdd7b45e49c 100644
--- a/drivers/hwtracing/coresight/coresight-syscfg.c
+++ b/drivers/hwtracing/coresight/coresight-syscfg.c
@@ -953,39 +953,41 @@ int cscfg_config_sysfs_activate(struct cscfg_config_desc *config_desc, bool acti
unsigned long cfg_hash;
int err = 0;
- mutex_lock(&cscfg_mutex);
+ guard(mutex)(&cscfg_mutex);
cfg_hash = (unsigned long)config_desc->event_ea->var;
if (activate) {
/* cannot be a current active value to activate this */
- if (cscfg_mgr->sysfs_active_config) {
- err = -EBUSY;
- goto exit_unlock;
- }
- err = _cscfg_activate_config(cfg_hash);
- if (!err)
+ if (cscfg_mgr->sysfs_active_config)
+ return -EBUSY;
+
+ scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) {
+ err = _cscfg_activate_config(cfg_hash);
+ if (err)
+ return err;
+
cscfg_mgr->sysfs_active_config = cfg_hash;
+ }
} else {
- /* disable if matching current value */
- if (cscfg_mgr->sysfs_active_config == cfg_hash) {
+ if (cscfg_mgr->sysfs_active_config != cfg_hash)
+ return -EINVAL;
+
+ scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) {
+ /* disable if matching current value */
_cscfg_deactivate_config(cfg_hash);
cscfg_mgr->sysfs_active_config = 0;
- } else
- err = -EINVAL;
+ }
}
-exit_unlock:
- mutex_unlock(&cscfg_mutex);
- return err;
+ return 0;
}
/* set the sysfs preset value */
void cscfg_config_sysfs_set_preset(int preset)
{
- mutex_lock(&cscfg_mutex);
+ guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock);
cscfg_mgr->sysfs_active_preset = preset;
- mutex_unlock(&cscfg_mutex);
}
/*
@@ -994,10 +996,9 @@ void cscfg_config_sysfs_set_preset(int preset)
*/
void cscfg_config_sysfs_get_active_cfg(unsigned long *cfg_hash, int *preset)
{
- mutex_lock(&cscfg_mutex);
+ guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock);
*preset = cscfg_mgr->sysfs_active_preset;
*cfg_hash = cscfg_mgr->sysfs_active_config;
- mutex_unlock(&cscfg_mutex);
}
EXPORT_SYMBOL_GPL(cscfg_config_sysfs_get_active_cfg);
@@ -1201,6 +1202,7 @@ static int cscfg_create_device(void)
INIT_LIST_HEAD(&cscfg_mgr->load_order_list);
atomic_set(&cscfg_mgr->sys_active_cnt, 0);
cscfg_mgr->load_state = CSCFG_NONE;
+ raw_spin_lock_init(&cscfg_mgr->sysfs_store_lock);
/* setup the device */
dev = cscfg_device();
diff --git a/drivers/hwtracing/coresight/coresight-syscfg.h b/drivers/hwtracing/coresight/coresight-syscfg.h
index 66e2db890d820..658e93c3705f1 100644
--- a/drivers/hwtracing/coresight/coresight-syscfg.h
+++ b/drivers/hwtracing/coresight/coresight-syscfg.h
@@ -42,6 +42,7 @@ enum cscfg_load_ops {
* @sysfs_active_config:Active config hash used if CoreSight controlled from sysfs.
* @sysfs_active_preset:Active preset index used if CoreSight controlled from sysfs.
* @load_state: A multi-stage load/unload operation is in progress.
+ * @sysfs_store_lock: Exclusive access sysfs stored variables.
*/
struct cscfg_manager {
struct device dev;
@@ -54,6 +55,7 @@ struct cscfg_manager {
u32 sysfs_active_config;
int sysfs_active_preset;
enum cscfg_load_ops load_state;
+ raw_spinlock_t sysfs_store_lock;
};
/* get reference to dev in cscfg_manager */
diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c
index d2a6ed8bcc74d..4b010f8bc4c08 100644
--- a/drivers/hwtracing/coresight/coresight-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-sysfs.c
@@ -5,26 +5,12 @@
*/
#include <linux/device.h>
-#include <linux/idr.h>
#include <linux/kernel.h>
#include <linux/property.h>
#include "coresight-priv.h"
#include "coresight-trace-id.h"
-/*
- * Use IDR to map the hash of the source's device name
- * to the pointer of path for the source. The idr is for
- * the sources which aren't associated with CPU.
- */
-static DEFINE_IDR(path_idr);
-
-/*
- * When operating Coresight drivers from the sysFS interface, only a single
- * path can exist from a tracer (associated to a CPU) to a sink.
- */
-static DEFINE_PER_CPU(struct coresight_path *, tracer_path);
-
ssize_t coresight_simple_show_pair(struct device *_dev,
struct device_attribute *attr, char *buf)
{
@@ -53,6 +39,26 @@ ssize_t coresight_simple_show32(struct device *_dev,
}
EXPORT_SYMBOL_GPL(coresight_simple_show32);
+static void coresight_source_get_refcnt(struct coresight_device *csdev)
+{
+ /*
+ * There could be multiple applications driving the software
+ * source. So keep the refcount for each such user when the
+ * source is already enabled.
+ *
+ * No need to increment the reference counter for other source
+ * types, as multiple enables are the same as a single enable.
+ */
+ if (coresight_is_software_source(csdev))
+ csdev->refcnt++;
+}
+
+static void coresight_source_put_refcnt(struct coresight_device *csdev)
+{
+ if (coresight_is_software_source(csdev))
+ csdev->refcnt--;
+}
+
static int coresight_enable_source_sysfs(struct coresight_device *csdev,
enum cs_mode mode,
struct coresight_path *path)
@@ -66,19 +72,19 @@ static int coresight_enable_source_sysfs(struct coresight_device *csdev,
*/
lockdep_assert_held(&coresight_mutex);
if (coresight_get_mode(csdev) != CS_MODE_SYSFS) {
- ret = source_ops(csdev)->enable(csdev, NULL, mode, path);
+ ret = coresight_enable_source(csdev, NULL, mode, path);
if (ret)
return ret;
}
- csdev->refcnt++;
+ coresight_source_get_refcnt(csdev);
return 0;
}
/**
- * coresight_disable_source_sysfs - Drop the reference count by 1 and disable
- * the device if there are no users left.
+ * coresight_disable_source_sysfs - Drop the reference count by 1 for software
+ * sources. Disable the device if there are no users left.
*
* @csdev: The coresight device to disable
* @data: Opaque data to pass on to the disable function of the source device.
@@ -93,7 +99,7 @@ static bool coresight_disable_source_sysfs(struct coresight_device *csdev,
if (coresight_get_mode(csdev) != CS_MODE_SYSFS)
return false;
- csdev->refcnt--;
+ coresight_source_put_refcnt(csdev);
if (csdev->refcnt == 0) {
coresight_disable_source(csdev, data);
return true;
@@ -162,18 +168,17 @@ static int coresight_validate_source_sysfs(struct coresight_device *csdev,
return -EINVAL;
}
+ if (coresight_is_percpu_source(csdev) && !cpu_online(csdev->cpu))
+ return -ENODEV;
+
return 0;
}
int coresight_enable_sysfs(struct coresight_device *csdev)
{
- int cpu, ret = 0;
+ int ret = 0;
struct coresight_device *sink;
struct coresight_path *path;
- enum coresight_dev_subtype_source subtype;
- u32 hash;
-
- subtype = csdev->subtype.source_subtype;
mutex_lock(&coresight_mutex);
@@ -188,13 +193,7 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
* doesn't hold coresight_mutex.
*/
if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
- /*
- * There could be multiple applications driving the software
- * source. So keep the refcount for each such user when the
- * source is already enabled.
- */
- if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE)
- csdev->refcnt++;
+ coresight_source_get_refcnt(csdev);
goto out;
}
@@ -211,8 +210,8 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
goto out;
}
- coresight_path_assign_trace_id(path, CS_MODE_SYSFS);
- if (!IS_VALID_CS_TRACE_ID(path->trace_id))
+ ret = coresight_path_assign_trace_id(path, CS_MODE_SYSFS);
+ if (ret)
goto err_path;
ret = coresight_enable_path(path, CS_MODE_SYSFS);
@@ -223,35 +222,6 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
if (ret)
goto err_source;
- switch (subtype) {
- case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC:
- /*
- * When working from sysFS it is important to keep track
- * of the paths that were created so that they can be
- * undone in 'coresight_disable()'. Since there can only
- * be a single session per tracer (when working from sysFS)
- * a per-cpu variable will do just fine.
- */
- cpu = source_ops(csdev)->cpu_id(csdev);
- per_cpu(tracer_path, cpu) = path;
- break;
- case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
- case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
- case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
- /*
- * Use the hash of source's device name as ID
- * and map the ID to the pointer of the path.
- */
- hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev)));
- ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL);
- if (ret)
- goto err_source;
- break;
- default:
- /* We can't be here */
- break;
- }
-
out:
mutex_unlock(&coresight_mutex);
return ret;
@@ -267,9 +237,8 @@ EXPORT_SYMBOL_GPL(coresight_enable_sysfs);
void coresight_disable_sysfs(struct coresight_device *csdev)
{
- int cpu, ret;
- struct coresight_path *path = NULL;
- u32 hash;
+ struct coresight_path *path;
+ int ret;
mutex_lock(&coresight_mutex);
@@ -277,32 +246,15 @@ void coresight_disable_sysfs(struct coresight_device *csdev)
if (ret)
goto out;
+ /*
+ * coresight_disable_source_sysfs() clears the 'csdev->path' pointer
+ * when disabling the source. Retrieve the path pointer here.
+ */
+ path = csdev->path;
+
if (!coresight_disable_source_sysfs(csdev, NULL))
goto out;
- switch (csdev->subtype.source_subtype) {
- case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC:
- cpu = source_ops(csdev)->cpu_id(csdev);
- path = per_cpu(tracer_path, cpu);
- per_cpu(tracer_path, cpu) = NULL;
- break;
- case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
- case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
- case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
- hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev)));
- /* Find the path by the hash. */
- path = idr_find(&path_idr, hash);
- if (path == NULL) {
- pr_err("Path is not found for %s\n", dev_name(&csdev->dev));
- goto out;
- }
- idr_remove(&path_idr, hash);
- break;
- default:
- /* We can't be here */
- break;
- }
-
coresight_disable_path(path);
coresight_release_path(path);
@@ -360,6 +312,13 @@ static ssize_t enable_source_store(struct device *dev,
if (ret)
return ret;
+ /*
+ * CoreSight hotplug callbacks in core layer control a activated path
+ * from its source to sink. Taking hotplug lock here protects a race
+ * condition with hotplug callbacks.
+ */
+ guard(cpus_read_lock)();
+
if (val) {
ret = coresight_enable_sysfs(csdev);
if (ret)
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 4dc1defe27a5f..361a433e6f0c5 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -154,7 +154,7 @@ tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
for (i = 0; i < tmc_pages->nr_pages; i++) {
page_start = tmc_pages->daddrs[i];
if (addr >= page_start && addr < (page_start + PAGE_SIZE))
- return i * PAGE_SIZE + (addr - page_start);
+ return (long)i * PAGE_SIZE + (addr - page_start);
}
return -EINVAL;
@@ -1379,7 +1379,7 @@ alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
/* Use the minimum limit if the required size is smaller */
- size = nr_pages << PAGE_SHIFT;
+ size = (ssize_t)nr_pages << PAGE_SHIFT;
size = max_t(ssize_t, size, TMC_ETR_PERF_MIN_BUF_SIZE);
/*
diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c
index 1511f8eb95afb..c7cbca45f2deb 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.c
+++ b/drivers/hwtracing/coresight/coresight-trbe.c
@@ -117,6 +117,20 @@ static int trbe_errata_cpucaps[] = {
#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
/*
+ * struct trbe_save_state: Register values representing TRBE state
+ * @trblimitr - Trace Buffer Limit Address Register value
+ * @trbbaser - Trace Buffer Base Register value
+ * @trbptr - Trace Buffer Write Pointer Register value
+ * @trbsr - Trace Buffer Status Register value
+ */
+struct trbe_save_state {
+ u64 trblimitr;
+ u64 trbbaser;
+ u64 trbptr;
+ u64 trbsr;
+};
+
+/*
* struct trbe_cpudata: TRBE instance specific data
* @trbe_flag - TRBE dirty/access flag support
* @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
@@ -134,6 +148,7 @@ struct trbe_cpudata {
enum cs_mode mode;
struct trbe_buf *buf;
struct trbe_drvdata *drvdata;
+ struct trbe_save_state save_state;
DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
};
@@ -1189,6 +1204,46 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
return IRQ_HANDLED;
}
+static int arm_trbe_save(struct coresight_device *csdev)
+{
+ struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
+ struct trbe_save_state *state = &cpudata->save_state;
+
+ state->trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
+
+ /* Disable the unit, ensure the writes to memory are complete */
+ if (state->trblimitr & TRBLIMITR_EL1_E)
+ trbe_drain_and_disable_local(cpudata);
+
+ state->trbbaser = read_sysreg_s(SYS_TRBBASER_EL1);
+ state->trbptr = read_sysreg_s(SYS_TRBPTR_EL1);
+ state->trbsr = read_sysreg_s(SYS_TRBSR_EL1);
+ return 0;
+}
+
+static void arm_trbe_restore(struct coresight_device *csdev)
+{
+ struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
+ struct trbe_save_state *state = &cpudata->save_state;
+
+ write_sysreg_s(state->trbbaser, SYS_TRBBASER_EL1);
+ write_sysreg_s(state->trbptr, SYS_TRBPTR_EL1);
+ write_sysreg_s(state->trbsr, SYS_TRBSR_EL1);
+
+ if (!(state->trblimitr & TRBLIMITR_EL1_E)) {
+ write_sysreg_s(state->trblimitr, SYS_TRBLIMITR_EL1);
+ } else {
+ /*
+ * The section K5.5 Context switching, Arm ARM (ARM DDI 0487
+ * L.a), S_PKLXF requires a Context synchronization event to
+ * guarantee the Trace Buffer Unit will observe the new values
+ * of the system registers.
+ */
+ isb();
+ set_trbe_enabled(cpudata, state->trblimitr);
+ }
+}
+
static const struct coresight_ops_sink arm_trbe_sink_ops = {
.enable = arm_trbe_enable,
.disable = arm_trbe_disable,
@@ -1198,7 +1253,9 @@ static const struct coresight_ops_sink arm_trbe_sink_ops = {
};
static const struct coresight_ops arm_trbe_cs_ops = {
- .sink_ops = &arm_trbe_sink_ops,
+ .pm_save_disable = arm_trbe_save,
+ .pm_restore_enable = arm_trbe_restore,
+ .sink_ops = &arm_trbe_sink_ops,
};
static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
@@ -1289,6 +1346,8 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp
desc.ops = &arm_trbe_cs_ops;
desc.groups = arm_trbe_groups;
desc.dev = dev;
+ desc.cpu = cpu;
+ desc.flags = CORESIGHT_DESC_CPU_BOUND;
trbe_csdev = coresight_register(&desc);
if (IS_ERR(trbe_csdev))
goto cpu_clear;
diff --git a/include/linux/coresight.h b/include/linux/coresight.h
index 2131febebee93..add0579cad884 100644
--- a/include/linux/coresight.h
+++ b/include/linux/coresight.h
@@ -141,6 +141,8 @@ struct csdev_access {
.base = (_addr), \
})
+#define CORESIGHT_DESC_CPU_BOUND BIT(0)
+
/**
* struct coresight_desc - description of a component required from drivers
* @type: as defined by @coresight_dev_type.
@@ -153,6 +155,8 @@ struct csdev_access {
* in the component's sysfs sub-directory.
* @name: name for the coresight device, also shown under sysfs.
* @access: Describe access to the device
+ * @flags: The descritpion flags.
+ * @cpu: The CPU this component is affined to.
*/
struct coresight_desc {
enum coresight_dev_type type;
@@ -163,6 +167,8 @@ struct coresight_desc {
const struct attribute_group **groups;
const char *name;
struct csdev_access access;
+ u32 flags;
+ int cpu;
};
/**
@@ -251,6 +257,7 @@ struct coresight_trace_id_map {
* by @coresight_ops.
* @access: Device i/o access abstraction for this device.
* @dev: The device entity associated to this component.
+ * @path: Activated path pointer (only used for source device).
* @mode: The device mode, i.e sysFS, Perf or disabled. This is actually
* an 'enum cs_mode' but stored in an atomic type. Access is always
* through atomic APIs, ensuring SMP-safe synchronisation between
@@ -260,6 +267,7 @@ struct coresight_trace_id_map {
* device's spinlock when the coresight_mutex held and mode ==
* CS_MODE_SYSFS. Otherwise it must be accessed from inside the
* spinlock.
+ * @cpu: The CPU this component is affined to (-1 for not CPU bound).
* @orphan: true if the component has connections that haven't been linked.
* @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs
* by writing a 1 to the 'enable_sink' file. A sink can be
@@ -284,8 +292,10 @@ struct coresight_device {
const struct coresight_ops *ops;
struct csdev_access access;
struct device dev;
+ struct coresight_path *path;
atomic_t mode;
int refcnt;
+ int cpu;
bool orphan;
/* sink specific fields */
bool sysfs_sink_activated;
@@ -334,9 +344,9 @@ struct coresight_path {
};
enum cs_mode {
- CS_MODE_DISABLED,
- CS_MODE_SYSFS,
- CS_MODE_PERF,
+ CS_MODE_DISABLED = 0,
+ CS_MODE_SYSFS = BIT(0),
+ CS_MODE_PERF = BIT(1),
};
#define coresight_ops(csdev) csdev->ops
@@ -387,15 +397,12 @@ struct coresight_ops_link {
/**
* struct coresight_ops_source - basic operations for a source
* Operations available for sources.
- * @cpu_id: returns the value of the CPU number this component
- * is associated to.
* @enable: enables tracing for a source.
* @disable: disables tracing for a source.
* @resume_perf: resumes tracing for a source in perf session.
* @pause_perf: pauses tracing for a source in perf session.
*/
struct coresight_ops_source {
- int (*cpu_id)(struct coresight_device *csdev);
int (*enable)(struct coresight_device *csdev, struct perf_event *event,
enum cs_mode mode, struct coresight_path *path);
void (*disable)(struct coresight_device *csdev,
@@ -433,6 +440,8 @@ struct coresight_ops_panic {
struct coresight_ops {
int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode,
struct coresight_device *sink);
+ int (*pm_save_disable)(struct coresight_device *csdev);
+ void (*pm_restore_enable)(struct coresight_device *csdev);
const struct coresight_ops_sink *sink_ops;
const struct coresight_ops_link *link_ops;
const struct coresight_ops_source *source_ops;
@@ -602,6 +611,12 @@ static inline bool coresight_is_percpu_source(struct coresight_device *csdev)
(csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC);
}
+static inline bool coresight_is_software_source(struct coresight_device *csdev)
+{
+ return csdev && coresight_is_device_source(csdev) &&
+ (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE);
+}
+
static inline bool coresight_is_percpu_sink(struct coresight_device *csdev)
{
return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) &&
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 22ba327ec2278..0fb3a2a62eb00 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -180,7 +180,6 @@ enum cpuhp_state {
CPUHP_AP_DUMMY_TIMER_STARTING,
CPUHP_AP_ARM_XEN_STARTING,
CPUHP_AP_ARM_XEN_RUNSTATE_STARTING,
- CPUHP_AP_ARM_CORESIGHT_STARTING,
CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
CPUHP_AP_ARM64_ISNDEP_STARTING,
CPUHP_AP_SMPCFD_DYING,
@@ -200,6 +199,7 @@ enum cpuhp_state {
CPUHP_AP_IRQ_AFFINITY_ONLINE,
CPUHP_AP_BLK_MQ_ONLINE,
CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,
+ CPUHP_AP_ARM_CORESIGHT_ONLINE,
CPUHP_AP_X86_INTEL_EPB_ONLINE,
CPUHP_AP_PERF_ONLINE,
CPUHP_AP_PERF_X86_ONLINE,