Merge branch 'rdma-optional-stats' into next

Mark Zhang  says:

====================

This is supplementary part of kernel series [1], which provides an
extension to the rdma statistics tool that allows to set or list
optional counters dynamically, using netlink.

Thanks

[1] https://www.spinics.net/lists/linux-rdma/msg106283.html

====================

Signed-off-by: David Ahern <dsahern@kernel.org>

man/man8/rdma-statistic.8

@@ -58,6 +58,28 @@ rdma-statistic \- RDMA statistic counter configuration
 .RI "[ " COUNTER-ID " ]"
 .RI "[ " OBJECT-ID " ]"
 
+.ti -8
+.B rdma statistic
+.B mode
+.B "[" supported "]"
+.B link
+.RI "[ " DEV/PORT_INDEX " ]"
+
+.ti -8
+.B rdma statistic
+.B set
+.B link
+.RI "[ " DEV/PORT_INDEX " ]"
+.B optional-counters
+.RI "[ " OPTIONAL-COUNTERS " ]"
+
+.ti -8
+.B rdma statistic
+.B unset
+.B link
+.RI "[ " DEV/PORT_INDEX " ]"
+.B optional-counters
+
 .ti -8
 .IR COUNTER_SCOPE " := "
 .RB "{ " link " | " dev " }"
@@ -100,6 +122,17 @@ When unbound the statistics of this object are no longer available in this count
 - specifies the id of the counter to be bound.
 If this argument is omitted then a new counter will be allocated.
 
+.SS rdma statistic mode - Display the enabled optional counters for each link.
+
+.SS rdma statistic mode supported - Display the supported optional counters for each link.
+
+.SS rdma statistic set - Enable a set of optional counters for a specific device/port.
+
+.I "OPTIONAL-COUNTERS"
+- specifies the name of the optional counters to enable. Optional counters that are not specified will be disabled. Note that optional counters are driver-specific.
+
+.SS rdma statistic unset - Disable all optional counters for a specific device/port.
+
 .SH "EXAMPLES"
 .PP
 rdma statistic show
@@ -186,6 +219,26 @@ rdma statistic show mr mrn 6
 .RS 4
 Dump a specific MR statistics with mrn 6. Dumps nothing if does not exists.
 .RE
+.PP
+rdma statistic mode link mlx5_2/1
+.RS 4
+Display the optional counters that was enabled on mlx5_2/1.
+.RE
+.PP
+rdma statistic mode supported link mlx5_2/1
+.RS 4
+Display the optional counters that mlx5_2/1 supports.
+.RE
+.PP
+rdma statistic set link mlx5_2/1 optional-counters cc_rx_ce_pkts,cc_rx_cnp_pkts
+.RS 4
+Enable the cc_rx_ce_pkts,cc_rx_cnp_pkts counters on device mlx5_2 port 1.
+.RE
+.PP
+rdma statistic unset link mlx5_2/1 optional-counters
+.RS 4
+Disable all the optional counters on device mlx5_2 port 1.
+.RE
 
 .SH SEE ALSO
 .BR rdma (8),
@@ -198,3 +251,5 @@ Dump a specific MR statistics with mrn 6. Dumps nothing if does not exists.
 Mark Zhang <markz@mellanox.com>
 .br
 Erez Alfasi <ereza@mellanox.com>
+.br
+Neta Ostrovsky <netao@nvidia.com>

rdma/include/uapi/rdma/rdma_netlink.h

@@ -297,6 +297,8 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_RES_SRQ_GET, /* can dump */
 
+	RDMA_NLDEV_CMD_STAT_GET_STATUS,
+
 	RDMA_NLDEV_NUM_OPS
 };
 
@@ -549,6 +551,9 @@ enum rdma_nldev_attr {
 
 	RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK,	/* u8 */
 
+	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX,	/* u32 */
+	RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC, /* u8 */
+
 	/*
 	 * Always the end
 	 */

rdma/stat.c

@@ -20,6 +20,10 @@ static int stat_help(struct rd *rd)
 	pr_out("       %s statistic OBJECT unbind COUNTER_SCOPE [DEV/PORT_INDEX] [COUNTER-ID]\n", rd->filename);
 	pr_out("       %s statistic show\n", rd->filename);
 	pr_out("       %s statistic show link [ DEV/PORT_INDEX ]\n", rd->filename);
+	pr_out("       %s statistic mode [ supported ]\n", rd->filename);
+	pr_out("       %s statistic mode [ supported ] link [ DEV/PORT_INDEX ]\n", rd->filename);
+	pr_out("       %s statistic set link [ DEV/PORT_INDEX ] optional-counters [ OPTIONAL-COUNTERS ]\n", rd->filename);
+	pr_out("       %s statistic unset link [ DEV/PORT_INDEX ] optional-counters\n", rd->filename);
 	pr_out("where  OBJECT: = { qp }\n");
 	pr_out("       CRITERIA : = { type }\n");
 	pr_out("       COUNTER_SCOPE: = { link | dev }\n");
@@ -37,6 +41,12 @@ static int stat_help(struct rd *rd)
 	pr_out("       %s statistic qp unbind link mlx5_2/1 cntn 4 lqpn 178\n", rd->filename);
 	pr_out("       %s statistic show\n", rd->filename);
 	pr_out("       %s statistic show link mlx5_2/1\n", rd->filename);
+	pr_out("       %s statistic mode\n", rd->filename);
+	pr_out("       %s statistic mode link mlx5_2/1\n", rd->filename);
+	pr_out("       %s statistic mode supported\n", rd->filename);
+	pr_out("       %s statistic mode supported link mlx5_2/1\n", rd->filename);
+	pr_out("       %s statistic set link mlx5_2/1 optional-counters cc_rx_ce_pkts,cc_rx_cnp_pkts\n", rd->filename);
+	pr_out("       %s statistic unset link mlx5_2/1 optional-counters\n", rd->filename);
 
 	return 0;
 }
@@ -493,6 +503,30 @@ static int stat_qp_set(struct rd *rd)
 	return rd_exec_cmd(rd, cmds, "parameter");
 }
 
+static int stat_get_arg_str(struct rd *rd, const char *arg, char **value, bool allow_empty)
+{
+	int len = 0;
+
+	if (strcmpx(rd_argv(rd), arg) != 0) {
+		pr_err("Unknown parameter '%s'.\n", rd_argv(rd));
+		return -EINVAL;
+	}
+
+	rd_arg_inc(rd);
+	if (!rd_no_arg(rd)) {
+		*value = strdup(rd_argv(rd));
+		len = strlen(*value);
+		rd_arg_inc(rd);
+	}
+
+	if ((allow_empty && len) || (!allow_empty && !len)) {
+		stat_help(rd);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int stat_get_arg(struct rd *rd, const char *arg)
 {
 	int value = 0;
@@ -715,6 +749,310 @@ static int stat_qp(struct rd *rd)
 	return rd_exec_cmd(rd, cmds, "parameter");
 }
 
+static int do_stat_mode_parse_cb(const struct nlmsghdr *nlh, void *data,
+				 bool supported)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+	struct nlattr *nla_entry;
+	const char *dev, *name;
+	struct rd *rd = data;
+	int enabled, err = 0;
+	bool isfirst = true;
+	uint32_t port;
+
+	mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+	if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+	    !tb[RDMA_NLDEV_ATTR_PORT_INDEX] ||
+	    !tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS])
+		return MNL_CB_ERROR;
+
+	dev = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+	port = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
+
+	mnl_attr_for_each_nested(nla_entry,
+				 tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
+		struct nlattr *cnt[RDMA_NLDEV_ATTR_MAX] = {};
+
+		err  = mnl_attr_parse_nested(nla_entry, rd_attr_cb, cnt);
+		if ((err != MNL_CB_OK) ||
+		    (!cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME]))
+			return -EINVAL;
+
+		if (!cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC])
+			continue;
+
+		enabled = mnl_attr_get_u8(cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC]);
+		name = mnl_attr_get_str(cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME]);
+		if (supported || enabled) {
+			if (isfirst) {
+				open_json_object(NULL);
+				print_color_string(PRINT_ANY, COLOR_NONE,
+						   "ifname", "link %s/", dev);
+				print_color_uint(PRINT_ANY, COLOR_NONE, "port",
+						 "%u ", port);
+				if (supported)
+					open_json_array(PRINT_ANY,
+						"supported optional-counters");
+				else
+					open_json_array(PRINT_ANY,
+							"optional-counters");
+				print_color_string(PRINT_FP, COLOR_NONE, NULL,
+						   " ", NULL);
+				isfirst = false;
+			} else {
+				print_color_string(PRINT_FP, COLOR_NONE, NULL,
+						   ",", NULL);
+			}
+			if (rd->pretty_output && !rd->json_output)
+				newline_indent(rd);
+
+			print_color_string(PRINT_ANY, COLOR_NONE, NULL, "%s",
+					   name);
+		}
+	}
+
+	if (!isfirst) {
+		close_json_array(PRINT_JSON, NULL);
+		newline(rd);
+	}
+
+	return 0;
+}
+
+static int stat_mode_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+	return do_stat_mode_parse_cb(nlh, data, false);
+}
+
+static int stat_mode_parse_cb_supported(const struct nlmsghdr *nlh, void *data)
+{
+	return do_stat_mode_parse_cb(nlh, data, true);
+}
+
+static int stat_one_link_get_status_req(struct rd *rd, uint32_t *seq)
+{
+	int flags = NLM_F_REQUEST | NLM_F_ACK;
+
+	rd_prepare_msg(rd, RDMA_NLDEV_CMD_STAT_GET_STATUS, seq,  flags);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_DEV_INDEX, rd->dev_idx);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_PORT_INDEX, rd->port_idx);
+
+	return rd_send_msg(rd);
+}
+
+static int stat_one_link_get_mode(struct rd *rd)
+{
+	uint32_t seq;
+	int err;
+
+	if (!rd->port_idx)
+		return 0;
+
+	err = stat_one_link_get_status_req(rd, &seq);
+	if (err)
+		return err;
+
+	return rd_recv_msg(rd, stat_mode_parse_cb, rd, seq);
+}
+
+static int stat_one_link_get_mode_supported(struct rd *rd)
+{
+	uint32_t seq;
+	int err;
+
+	if (!rd->port_idx)
+		return 0;
+
+	err = stat_one_link_get_status_req(rd, &seq);
+	if (err)
+		return err;
+
+	return rd_recv_msg(rd, stat_mode_parse_cb_supported, rd, seq);
+}
+
+static int stat_link_get_mode(struct rd *rd)
+{
+	return rd_exec_link(rd, stat_one_link_get_mode, false);
+}
+
+static int stat_link_get_mode_supported(struct rd *rd)
+{
+	return rd_exec_link(rd, stat_one_link_get_mode_supported, false);
+}
+
+static int stat_mode_supported(struct rd *rd)
+{
+	const struct rd_cmd cmds[] = {
+		{ NULL,		stat_link_get_mode_supported },
+		{ "link",	stat_link_get_mode_supported },
+		{ "help",	stat_help },
+		{ 0 },
+	};
+	return rd_exec_cmd(rd, cmds, "parameter");
+}
+
+static int stat_mode(struct rd *rd)
+{
+	const struct rd_cmd cmds[] = {
+		{ NULL,		stat_link_get_mode },
+		{ "link",	stat_link_get_mode },
+		{ "show",	stat_link_get_mode },
+		{ "supported",	stat_mode_supported },
+		{ "help",	stat_help },
+		{ 0 },
+	};
+
+	return rd_exec_cmd(rd, cmds, "parameter");
+}
+
+static int stat_one_set_link_opcounters(const struct nlmsghdr *nlh, void *data)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+	struct nlattr *nla_entry, *tb_set;
+	int ret, flags = NLM_F_REQUEST | NLM_F_ACK;
+	char *opcnt, *opcnts;
+	struct rd *rd = data;
+	uint32_t seq;
+	bool found;
+
+	mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+	if (!tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS])
+		return MNL_CB_ERROR;
+
+	if (rd_no_arg(rd)) {
+		stat_help(rd);
+		return -EINVAL;
+	}
+
+	ret = stat_get_arg_str(rd, "optional-counters", &opcnts, false);
+	if (ret)
+		return ret;
+
+	rd_prepare_msg(rd, RDMA_NLDEV_CMD_STAT_SET, &seq, flags);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_DEV_INDEX,
+			 rd->dev_idx);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_PORT_INDEX,
+			 rd->port_idx);
+
+	tb_set = mnl_attr_nest_start(rd->nlh, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
+
+	opcnt = strtok(opcnts, ",");
+	while (opcnt) {
+		found = false;
+		mnl_attr_for_each_nested(nla_entry,
+					 tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
+			struct nlattr *cnt[RDMA_NLDEV_ATTR_MAX] = {}, *nm, *id;
+
+			if (mnl_attr_parse_nested(nla_entry, rd_attr_cb,
+						  cnt) != MNL_CB_OK)
+				return -EINVAL;
+
+			nm = cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME];
+			id = cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX];
+			if (!nm || ! id)
+				return -EINVAL;
+
+			if (!cnt[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC])
+				continue;
+
+			if (strcmp(opcnt, mnl_attr_get_str(nm)) == 0) {
+				mnl_attr_put_u32(rd->nlh,
+						 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX,
+						 mnl_attr_get_u32(id));
+				found = true;
+			}
+		}
+
+		if (!found)
+			return -EINVAL;
+
+		opcnt = strtok(NULL, ",");
+	}
+	mnl_attr_nest_end(rd->nlh, tb_set);
+
+	return rd_sendrecv_msg(rd, seq);
+}
+
+static int stat_one_set_link(struct rd *rd)
+{
+	uint32_t seq;
+	int err;
+
+	if (!rd->port_idx)
+		return 0;
+
+	err = stat_one_link_get_status_req(rd, &seq);
+	if (err)
+		return err;
+
+	return rd_recv_msg(rd, stat_one_set_link_opcounters, rd, seq);
+}
+
+static int stat_set_link(struct rd *rd)
+{
+	return rd_exec_link(rd, stat_one_set_link, true);
+}
+
+static int stat_set(struct rd *rd)
+{
+	const struct rd_cmd cmds[] = {
+		{ NULL,		stat_help },
+		{ "link",	stat_set_link },
+		{ "help",	stat_help },
+		{ 0 },
+	};
+	return rd_exec_cmd(rd, cmds, "parameter");
+}
+
+static int stat_one_unset_link_opcounters(struct rd *rd)
+{
+	int ret, flags = NLM_F_REQUEST | NLM_F_ACK;
+	struct nlattr *tbl;
+	uint32_t seq;
+	char *opcnts;
+
+	if (rd_no_arg(rd)) {
+		stat_help(rd);
+		return -EINVAL;
+	}
+
+	ret = stat_get_arg_str(rd, "optional-counters", &opcnts, true);
+	if (ret)
+		return ret;
+
+	rd_prepare_msg(rd, RDMA_NLDEV_CMD_STAT_SET, &seq, flags);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_DEV_INDEX,
+			 rd->dev_idx);
+	mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_PORT_INDEX,
+			 rd->port_idx);
+
+	tbl = mnl_attr_nest_start(rd->nlh, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
+	mnl_attr_nest_end(rd->nlh, tbl);
+
+	return rd_sendrecv_msg(rd, seq);
+}
+
+static int stat_one_unset_link(struct rd *rd)
+{
+	return stat_one_unset_link_opcounters(rd);
+}
+
+static int stat_unset_link(struct rd *rd)
+{
+	return rd_exec_link(rd, stat_one_unset_link, true);
+}
+
+static int stat_unset(struct rd *rd)
+{
+	const struct rd_cmd cmds[] = {
+		{ NULL,		stat_help },
+		{ "link",	stat_unset_link },
+		{ "help",	stat_help },
+		{ 0 },
+	};
+	return rd_exec_cmd(rd, cmds, "parameter");
+}
+
 static int stat_show_parse_cb(const struct nlmsghdr *nlh, void *data)
 {
 	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
@@ -786,6 +1124,9 @@ int cmd_stat(struct rd *rd)
 		{ "help",	stat_help },
 		{ "qp",		stat_qp },
 		{ "mr",		stat_mr },
+		{ "mode",	stat_mode },
+		{ "set",	stat_set },
+		{ "unset",	stat_unset },
 		{ 0 }
 	};