Merge branch 'devlink-rate-support' into next
Dmytro Linkin says:
====================
Series implements devlink rate commands, which are:
- Dump particular or all rate objects (JSON or non-JSON)
- Add/Delete node rate object
- Set tx rate share/max values for rate object
- Set/Unset parent rate object for other rate object
Examples:
Display all rate objects:
# devlink port function rate show
pci/0000:03:00.0/1 type leaf parent some_group
pci/0000:03:00.0/2 type leaf tx_share 12Mbit
pci/0000:03:00.0/some_group type node tx_share 1Gbps tx_max 5Gbps
Display leaf rate object bound to the 1st devlink port of the
pci/0000:03:00.0 device:
# devlink port function rate show pci/0000:03:00.0/1
pci/0000:03:00.0/1 type leaf
Display node rate object with name some_group of the pci/0000:03:00.0
device:
# devlink port function rate show pci/0000:03:00.0/some_group
pci/0000:03:00.0/some_group type node
Display leaf rate object rate values using IEC units:
# devlink -i port function rate show pci/0000:03:00.0/2
pci/0000:03:00.0/2 type leaf 11718Kibit
Display pci/0000:03:00.0/2 leaf rate object as pretty JSON output:
# devlink -jp port function rate show pci/0000:03:00.0/2
{
"rate": {
"pci/0000:03:00.0/2": {
"type": "leaf",
"tx_share": 1500000
}
}
}
Create node rate object with name "1st_group" on pci/0000:03:00.0 device:
# devlink port function rate add pci/0000:03:00.0/1st_group
Create node rate object with specified parameters:
# devlink port function rate add pci/0000:03:00.0/2nd_group \
tx_share 10Mbit tx_max 30Mbit parent 1st_group
Set parameters to the specified leaf rate object:
# devlink port function rate set pci/0000:03:00.0/1 \
tx_share 2Mbit tx_max 10Mbit
Set leaf's parent to "1st_group":
# devlink port function rate set pci/0000:03:00.0/1 parent 1st_group
Unset leaf's parent:
# devlink port function rate set pci/0000:03:00.0/1 noparent
Delete node rate object:
# devlink port function rate del pci/0000:03:00.0/2nd_group
Rate values can be specified in bits or bytes per second (bit|bps), with
any SI (k, m, g, t) or IEC (ki, mi, gi, ti) prefix. Bare number means
bits per second. Units also printed in "show" command output, but not
necessarily the same which were specified with "set" or "add" command.
-i/--iec switch force output in IEC units. JSON output always print
values as bytes per sec.
====================
Signed-off-by: David Ahern <dsahern@kernel.org>
This commit is contained in:
David Ahern
commit
d8b3b9d32d
|
|
@ -71,6 +71,8 @@ static bool g_indent_newline;
|
|||
#define INDENT_STR_MAXLEN 32
|
||||
static char g_indent_str[INDENT_STR_MAXLEN + 1] = "";
|
||||
|
||||
static bool use_iec = false;
|
||||
|
||||
static void __attribute__((format(printf, 1, 2)))
|
||||
pr_err(const char *fmt, ...)
|
||||
{
|
||||
|
|
@ -287,6 +289,11 @@ static void ifname_map_free(struct ifname_map *ifname_map)
|
|||
#define DL_OPT_PORT_SFNUMBER BIT(44)
|
||||
#define DL_OPT_PORT_FUNCTION_STATE BIT(45)
|
||||
#define DL_OPT_PORT_CONTROLLER BIT(46)
|
||||
#define DL_OPT_PORT_FN_RATE_TYPE BIT(47)
|
||||
#define DL_OPT_PORT_FN_RATE_TX_SHARE BIT(48)
|
||||
#define DL_OPT_PORT_FN_RATE_TX_MAX BIT(49)
|
||||
#define DL_OPT_PORT_FN_RATE_NODE_NAME BIT(50)
|
||||
#define DL_OPT_PORT_FN_RATE_PARENT BIT(51)
|
||||
|
||||
struct dl_opts {
|
||||
uint64_t present; /* flags of present items */
|
||||
|
|
@ -342,6 +349,11 @@ struct dl_opts {
|
|||
uint16_t port_flavour;
|
||||
uint16_t port_pfnumber;
|
||||
uint8_t port_fn_state;
|
||||
uint16_t rate_type;
|
||||
uint64_t rate_tx_share;
|
||||
uint64_t rate_tx_max;
|
||||
char *rate_node_name;
|
||||
const char *rate_parent_node;
|
||||
};
|
||||
|
||||
struct dl {
|
||||
|
|
@ -917,6 +929,19 @@ static int strtobool(const char *str, bool *p_val)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int ident_str_validate(char *str, unsigned int expected)
|
||||
{
|
||||
if (!str)
|
||||
return -EINVAL;
|
||||
|
||||
if (get_str_char_count(str, '/') != expected) {
|
||||
pr_err("Wrong identification string format.\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __dl_argv_handle(char *str, char **p_bus_name, char **p_dev_name)
|
||||
{
|
||||
int err;
|
||||
|
|
@ -932,15 +957,12 @@ static int __dl_argv_handle(char *str, char **p_bus_name, char **p_dev_name)
|
|||
static int dl_argv_handle(struct dl *dl, char **p_bus_name, char **p_dev_name)
|
||||
{
|
||||
char *str = dl_argv_next(dl);
|
||||
int err;
|
||||
|
||||
if (!str) {
|
||||
err = ident_str_validate(str, 1);
|
||||
if (err) {
|
||||
pr_err("Devlink identification (\"bus_name/dev_name\") expected\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
if (get_str_char_count(str, '/') != 1) {
|
||||
pr_err("Wrong devlink identification string format.\n");
|
||||
pr_err("Expected \"bus_name/dev_name\".\n");
|
||||
return -EINVAL;
|
||||
return err;
|
||||
}
|
||||
return __dl_argv_handle(str, p_bus_name, p_dev_name);
|
||||
}
|
||||
|
|
@ -1052,44 +1074,103 @@ static int dl_argv_handle_both(struct dl *dl, char **p_bus_name,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int __dl_argv_handle_region(char *str, char **p_bus_name,
|
||||
char **p_dev_name, char **p_region)
|
||||
static int __dl_argv_handle_name(char *str, char **p_bus_name,
|
||||
char **p_dev_name, char **p_name)
|
||||
{
|
||||
char *handlestr;
|
||||
int err;
|
||||
|
||||
err = str_split_by_char(str, &handlestr, p_region, '/');
|
||||
if (err) {
|
||||
pr_err("Region identification \"%s\" is invalid\n", str);
|
||||
err = str_split_by_char(str, &handlestr, p_name, '/');
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
err = str_split_by_char(handlestr, p_bus_name, p_dev_name, '/');
|
||||
if (err) {
|
||||
pr_err("Region identification \"%s\" is invalid\n", str);
|
||||
return err;
|
||||
}
|
||||
return 0;
|
||||
|
||||
return str_split_by_char(handlestr, p_bus_name, p_dev_name, '/');
|
||||
}
|
||||
|
||||
static int dl_argv_handle_region(struct dl *dl, char **p_bus_name,
|
||||
char **p_dev_name, char **p_region)
|
||||
char **p_dev_name, char **p_region)
|
||||
{
|
||||
char *str = dl_argv_next(dl);
|
||||
unsigned int slash_count;
|
||||
int err;
|
||||
|
||||
if (!str) {
|
||||
err = ident_str_validate(str, 2);
|
||||
if (err) {
|
||||
pr_err("Expected \"bus_name/dev_name/region\" identification.\n");
|
||||
return err;
|
||||
}
|
||||
|
||||
err = __dl_argv_handle_name(str, p_bus_name, p_dev_name, p_region);
|
||||
if (err)
|
||||
pr_err("Region identification \"%s\" is invalid\n", str);
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
static int dl_argv_handle_rate_node(struct dl *dl, char **p_bus_name,
|
||||
char **p_dev_name, char **p_node)
|
||||
{
|
||||
char *str = dl_argv_next(dl);
|
||||
int err;
|
||||
|
||||
err = ident_str_validate(str, 2);
|
||||
if (err) {
|
||||
pr_err("Expected \"bus_name/dev_name/node\" identification.\n");
|
||||
return err;
|
||||
}
|
||||
|
||||
err = __dl_argv_handle_name(str, p_bus_name, p_dev_name, p_node);
|
||||
if (err) {
|
||||
pr_err("Node identification \"%s\" is invalid\n", str);
|
||||
return err;
|
||||
}
|
||||
|
||||
if (!**p_node || strspn(*p_node, "0123456789") == strlen(*p_node)) {
|
||||
err = -EINVAL;
|
||||
pr_err("Node name cannot be a devlink port index or empty.\n");
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int dl_argv_handle_rate(struct dl *dl, char **p_bus_name,
|
||||
char **p_dev_name, uint32_t *p_port_index,
|
||||
char **p_node_name, uint64_t *p_handle_bit)
|
||||
{
|
||||
char *str = dl_argv_next(dl);
|
||||
char *identifier;
|
||||
int err;
|
||||
|
||||
err = ident_str_validate(str, 2);
|
||||
if (err) {
|
||||
pr_err("Expected \"bus_name/dev_name/node\" or "
|
||||
"\"bus_name/dev_name/port_index\" identification.\n");
|
||||
return err;
|
||||
}
|
||||
|
||||
err = __dl_argv_handle_name(str, p_bus_name, p_dev_name, &identifier);
|
||||
if (err) {
|
||||
pr_err("Identification \"%s\" is invalid\n", str);
|
||||
return err;
|
||||
}
|
||||
|
||||
if (!*identifier) {
|
||||
pr_err("Identifier cannot be empty");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
slash_count = get_str_char_count(str, '/');
|
||||
if (slash_count != 2) {
|
||||
pr_err("Wrong region identification string format.\n");
|
||||
pr_err("Expected \"bus_name/dev_name/region\" identification.\n"".\n");
|
||||
return -EINVAL;
|
||||
if (strspn(identifier, "0123456789") == strlen(identifier)) {
|
||||
err = strtouint32_t(identifier, p_port_index);
|
||||
if (err) {
|
||||
pr_err("Port index \"%s\" is not a number"
|
||||
" or not within range\n", identifier);
|
||||
return err;
|
||||
}
|
||||
*p_handle_bit = DL_OPT_HANDLEP;
|
||||
} else {
|
||||
*p_handle_bit = DL_OPT_PORT_FN_RATE_NODE_NAME;
|
||||
*p_node_name = identifier;
|
||||
}
|
||||
|
||||
return __dl_argv_handle_region(str, p_bus_name, p_dev_name, p_region);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int dl_argv_uint64_t(struct dl *dl, uint64_t *p_val)
|
||||
|
|
@ -1401,6 +1482,36 @@ static int port_fn_state_parse(const char *statestr, uint8_t *state)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int port_fn_rate_type_get(const char *typestr, uint16_t *type)
|
||||
{
|
||||
if (!strcmp(typestr, "leaf"))
|
||||
*type = DEVLINK_RATE_TYPE_LEAF;
|
||||
else if (!strcmp(typestr, "node"))
|
||||
*type = DEVLINK_RATE_TYPE_NODE;
|
||||
else
|
||||
return -EINVAL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int port_fn_rate_value_get(struct dl *dl, uint64_t *rate)
|
||||
{
|
||||
const char *ratestr;
|
||||
__u64 rate64;
|
||||
int err;
|
||||
|
||||
err = dl_argv_str(dl, &ratestr);
|
||||
if (err)
|
||||
return err;
|
||||
err = get_rate64(&rate64, ratestr);
|
||||
if (err) {
|
||||
pr_err("Invalid rate value: \"%s\"\n", ratestr);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
*rate = rate64;
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct dl_args_metadata {
|
||||
uint64_t o_flag;
|
||||
char err_msg[DL_ARGS_REQUIRED_MAX_ERR_LEN];
|
||||
|
|
@ -1473,6 +1584,19 @@ static int dl_argv_parse(struct dl *dl, uint64_t o_required,
|
|||
return err;
|
||||
o_required &= ~(DL_OPT_HANDLE | DL_OPT_HANDLEP) | handle_bit;
|
||||
o_found |= handle_bit;
|
||||
} else if (o_required & DL_OPT_HANDLEP &&
|
||||
o_required & DL_OPT_PORT_FN_RATE_NODE_NAME) {
|
||||
uint64_t handle_bit;
|
||||
|
||||
err = dl_argv_handle_rate(dl, &opts->bus_name, &opts->dev_name,
|
||||
&opts->port_index,
|
||||
&opts->rate_node_name,
|
||||
&handle_bit);
|
||||
if (err)
|
||||
return err;
|
||||
o_required &= ~(DL_OPT_HANDLEP | DL_OPT_PORT_FN_RATE_NODE_NAME) |
|
||||
handle_bit;
|
||||
o_found |= handle_bit;
|
||||
} else if (o_required & DL_OPT_HANDLE) {
|
||||
err = dl_argv_handle(dl, &opts->bus_name, &opts->dev_name);
|
||||
if (err)
|
||||
|
|
@ -1491,6 +1615,13 @@ static int dl_argv_parse(struct dl *dl, uint64_t o_required,
|
|||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_HANDLE_REGION;
|
||||
} else if (o_required & DL_OPT_PORT_FN_RATE_NODE_NAME) {
|
||||
err = dl_argv_handle_rate_node(dl, &opts->bus_name,
|
||||
&opts->dev_name,
|
||||
&opts->rate_node_name);
|
||||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_FN_RATE_NODE_NAME;
|
||||
}
|
||||
|
||||
while (dl_argc(dl)) {
|
||||
|
|
@ -1894,6 +2025,44 @@ static int dl_argv_parse(struct dl *dl, uint64_t o_required,
|
|||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_CONTROLLER;
|
||||
} else if (dl_argv_match(dl, "type") &&
|
||||
(o_all & DL_OPT_PORT_FN_RATE_TYPE)) {
|
||||
const char *typestr;
|
||||
|
||||
dl_arg_inc(dl);
|
||||
err = dl_argv_str(dl, &typestr);
|
||||
if (err)
|
||||
return err;
|
||||
err = port_fn_rate_type_get(typestr, &opts->rate_type);
|
||||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_FN_RATE_TYPE;
|
||||
} else if (dl_argv_match(dl, "tx_share") &&
|
||||
(o_all & DL_OPT_PORT_FN_RATE_TX_SHARE)) {
|
||||
dl_arg_inc(dl);
|
||||
err = port_fn_rate_value_get(dl, &opts->rate_tx_share);
|
||||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_FN_RATE_TX_SHARE;
|
||||
} else if (dl_argv_match(dl, "tx_max") &&
|
||||
(o_all & DL_OPT_PORT_FN_RATE_TX_MAX)) {
|
||||
dl_arg_inc(dl);
|
||||
err = port_fn_rate_value_get(dl, &opts->rate_tx_max);
|
||||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_FN_RATE_TX_MAX;
|
||||
} else if (dl_argv_match(dl, "parent") &&
|
||||
(o_all & DL_OPT_PORT_FN_RATE_PARENT)) {
|
||||
dl_arg_inc(dl);
|
||||
err = dl_argv_str(dl, &opts->rate_parent_node);
|
||||
if (err)
|
||||
return err;
|
||||
o_found |= DL_OPT_PORT_FN_RATE_PARENT;
|
||||
} else if (dl_argv_match(dl, "noparent") &&
|
||||
(o_all & DL_OPT_PORT_FN_RATE_PARENT)) {
|
||||
dl_arg_inc(dl);
|
||||
opts->rate_parent_node = "";
|
||||
o_found |= DL_OPT_PORT_FN_RATE_PARENT;
|
||||
} else {
|
||||
pr_err("Unknown option \"%s\"\n", dl_argv(dl));
|
||||
return -EINVAL;
|
||||
|
|
@ -1966,6 +2135,11 @@ static void dl_opts_put(struct nlmsghdr *nlh, struct dl *dl)
|
|||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_DEV_NAME, opts->dev_name);
|
||||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_REGION_NAME,
|
||||
opts->region_name);
|
||||
} else if (opts->present & DL_OPT_PORT_FN_RATE_NODE_NAME) {
|
||||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_BUS_NAME, opts->bus_name);
|
||||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_DEV_NAME, opts->dev_name);
|
||||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_RATE_NODE_NAME,
|
||||
opts->rate_node_name);
|
||||
}
|
||||
if (opts->present & DL_OPT_PORT_TYPE)
|
||||
mnl_attr_put_u16(nlh, DEVLINK_ATTR_PORT_TYPE,
|
||||
|
|
@ -2090,6 +2264,18 @@ static void dl_opts_put(struct nlmsghdr *nlh, struct dl *dl)
|
|||
if (opts->present & DL_OPT_PORT_CONTROLLER)
|
||||
mnl_attr_put_u32(nlh, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
|
||||
opts->port_controller);
|
||||
if (opts->present & DL_OPT_PORT_FN_RATE_TYPE)
|
||||
mnl_attr_put_u16(nlh, DEVLINK_ATTR_RATE_TYPE,
|
||||
opts->rate_type);
|
||||
if (opts->present & DL_OPT_PORT_FN_RATE_TX_SHARE)
|
||||
mnl_attr_put_u64(nlh, DEVLINK_ATTR_RATE_TX_SHARE,
|
||||
opts->rate_tx_share);
|
||||
if (opts->present & DL_OPT_PORT_FN_RATE_TX_MAX)
|
||||
mnl_attr_put_u64(nlh, DEVLINK_ATTR_RATE_TX_MAX,
|
||||
opts->rate_tx_max);
|
||||
if (opts->present & DL_OPT_PORT_FN_RATE_PARENT)
|
||||
mnl_attr_put_strz(nlh, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
|
||||
opts->rate_parent_node);
|
||||
}
|
||||
|
||||
static int dl_argv_parse_put(struct nlmsghdr *nlh, struct dl *dl,
|
||||
|
|
@ -3803,6 +3989,7 @@ static void cmd_port_help(void)
|
|||
pr_err(" devlink port split DEV/PORT_INDEX count COUNT\n");
|
||||
pr_err(" devlink port unsplit DEV/PORT_INDEX\n");
|
||||
pr_err(" devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state STATE ]\n");
|
||||
pr_err(" devlink port function rate { help | show | add | del | set }\n");
|
||||
pr_err(" devlink port param set DEV/PORT_INDEX name PARAMETER value VALUE cmode { permanent | driverinit | runtime }\n");
|
||||
pr_err(" devlink port param show [DEV/PORT_INDEX name PARAMETER]\n");
|
||||
pr_err(" devlink port health show [ DEV/PORT_INDEX reporter REPORTER_NAME ]\n");
|
||||
|
|
@ -4097,6 +4284,7 @@ static int cmd_port_param_show(struct dl *dl)
|
|||
static void cmd_port_function_help(void)
|
||||
{
|
||||
pr_err("Usage: devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state STATE ]\n");
|
||||
pr_err(" devlink port function rate { help | show | add | del | set }\n");
|
||||
}
|
||||
|
||||
static int cmd_port_function_set(struct dl *dl)
|
||||
|
|
@ -4318,6 +4506,280 @@ static int cmd_port_param(struct dl *dl)
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
static void
|
||||
pr_out_port_rate_handle_start(struct dl *dl, struct nlattr **tb, bool try_nice)
|
||||
{
|
||||
const char *bus_name;
|
||||
const char *dev_name;
|
||||
const char *node_name;
|
||||
static char buf[64];
|
||||
|
||||
bus_name = mnl_attr_get_str(tb[DEVLINK_ATTR_BUS_NAME]);
|
||||
dev_name = mnl_attr_get_str(tb[DEVLINK_ATTR_DEV_NAME]);
|
||||
node_name = mnl_attr_get_str(tb[DEVLINK_ATTR_RATE_NODE_NAME]);
|
||||
sprintf(buf, "%s/%s/%s", bus_name, dev_name, node_name);
|
||||
if (dl->json_output)
|
||||
open_json_object(buf);
|
||||
else
|
||||
pr_out("%s:", buf);
|
||||
}
|
||||
|
||||
static char *port_rate_type_name(uint16_t type)
|
||||
{
|
||||
switch (type) {
|
||||
case DEVLINK_RATE_TYPE_LEAF:
|
||||
return "leaf";
|
||||
case DEVLINK_RATE_TYPE_NODE:
|
||||
return "node";
|
||||
default:
|
||||
return "<unknown type>";
|
||||
}
|
||||
}
|
||||
|
||||
static void pr_out_port_fn_rate(struct dl *dl, struct nlattr **tb)
|
||||
{
|
||||
|
||||
if (!tb[DEVLINK_ATTR_RATE_NODE_NAME])
|
||||
pr_out_port_handle_start(dl, tb, false);
|
||||
else
|
||||
pr_out_port_rate_handle_start(dl, tb, false);
|
||||
check_indent_newline(dl);
|
||||
|
||||
if (tb[DEVLINK_ATTR_RATE_TYPE]) {
|
||||
uint16_t type =
|
||||
mnl_attr_get_u16(tb[DEVLINK_ATTR_RATE_TYPE]);
|
||||
|
||||
print_string(PRINT_ANY, "type", "type %s",
|
||||
port_rate_type_name(type));
|
||||
}
|
||||
if (tb[DEVLINK_ATTR_RATE_TX_SHARE]) {
|
||||
uint64_t rate =
|
||||
mnl_attr_get_u64(tb[DEVLINK_ATTR_RATE_TX_SHARE]);
|
||||
|
||||
if (rate)
|
||||
print_rate(use_iec, PRINT_ANY, "tx_share",
|
||||
" tx_share %s", rate);
|
||||
}
|
||||
if (tb[DEVLINK_ATTR_RATE_TX_MAX]) {
|
||||
uint64_t rate =
|
||||
mnl_attr_get_u64(tb[DEVLINK_ATTR_RATE_TX_MAX]);
|
||||
|
||||
if (rate)
|
||||
print_rate(use_iec, PRINT_ANY, "tx_max",
|
||||
" tx_max %s", rate);
|
||||
}
|
||||
if (tb[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]) {
|
||||
const char *parent =
|
||||
mnl_attr_get_str(tb[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]);
|
||||
|
||||
print_string(PRINT_ANY, "parent", " parent %s", parent);
|
||||
}
|
||||
|
||||
pr_out_port_handle_end(dl);
|
||||
}
|
||||
|
||||
static int cmd_port_fn_rate_show_cb(const struct nlmsghdr *nlh, void *data)
|
||||
{
|
||||
struct dl *dl = data;
|
||||
struct nlattr *tb[DEVLINK_ATTR_MAX + 1] = {};
|
||||
struct genlmsghdr *genl = mnl_nlmsg_get_payload(nlh);
|
||||
|
||||
mnl_attr_parse(nlh, sizeof(*genl), attr_cb, tb);
|
||||
if ((!tb[DEVLINK_ATTR_BUS_NAME] || !tb[DEVLINK_ATTR_DEV_NAME] ||
|
||||
!tb[DEVLINK_ATTR_PORT_INDEX]) &&
|
||||
!tb[DEVLINK_ATTR_RATE_NODE_NAME]) {
|
||||
return MNL_CB_ERROR;
|
||||
}
|
||||
pr_out_port_fn_rate(dl, tb);
|
||||
return MNL_CB_OK;
|
||||
}
|
||||
|
||||
static void cmd_port_fn_rate_help(void)
|
||||
{
|
||||
pr_err("Usage: devlink port function rate help\n");
|
||||
pr_err(" devlink port function rate show [ DEV/{ PORT_INDEX | NODE_NAME } ]\n");
|
||||
pr_err(" devlink port function rate add DEV/NODE_NAME\n");
|
||||
pr_err(" [ tx_share VAL ][ tx_max VAL ][ { parent NODE_NAME | noparent } ]\n");
|
||||
pr_err(" devlink port function rate del DEV/NODE_NAME\n");
|
||||
pr_err(" devlink port function rate set DEV/{ PORT_INDEX | NODE_NAME }\n");
|
||||
pr_err(" [ tx_share VAL ][ tx_max VAL ][ { parent NODE_NAME | noparent } ]\n\n");
|
||||
pr_err(" VAL - float or integer value in units of bits or bytes per second (bit|bps)\n");
|
||||
pr_err(" and SI (k-, m-, g-, t-) or IEC (ki-, mi-, gi-, ti-) case-insensitive prefix.\n");
|
||||
pr_err(" Bare number, means bits per second, is possible.\n\n");
|
||||
pr_err(" For details refer to devlink-rate(8) man page.\n");
|
||||
}
|
||||
|
||||
static int cmd_port_fn_rate_show(struct dl *dl)
|
||||
{
|
||||
struct nlmsghdr *nlh;
|
||||
uint16_t flags = NLM_F_REQUEST | NLM_F_ACK;
|
||||
int err;
|
||||
|
||||
if (dl_argc(dl) == 0)
|
||||
flags |= NLM_F_DUMP;
|
||||
|
||||
nlh = mnlu_gen_socket_cmd_prepare(&dl->nlg, DEVLINK_CMD_RATE_GET, flags);
|
||||
|
||||
if (dl_argc(dl) > 0) {
|
||||
err = dl_argv_parse_put(nlh, dl, DL_OPT_HANDLEP |
|
||||
DL_OPT_PORT_FN_RATE_NODE_NAME, 0);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
pr_out_section_start(dl, "rate");
|
||||
err = mnlu_gen_socket_sndrcv(&dl->nlg, nlh, cmd_port_fn_rate_show_cb, dl);
|
||||
pr_out_section_end(dl);
|
||||
return err;
|
||||
}
|
||||
|
||||
static int port_fn_check_tx_rates(uint64_t min_rate, uint64_t max_rate)
|
||||
{
|
||||
if (max_rate && min_rate > max_rate) {
|
||||
pr_err("Invalid. Expected tx_share <= tx_max or tx_share == 0.\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int port_fn_get_and_check_tx_rates(struct dl_opts *reply,
|
||||
struct dl_opts *request)
|
||||
{
|
||||
uint64_t min = reply->rate_tx_share;
|
||||
uint64_t max = reply->rate_tx_max;
|
||||
|
||||
if (request->present & DL_OPT_PORT_FN_RATE_TX_SHARE)
|
||||
return port_fn_check_tx_rates(request->rate_tx_share, max);
|
||||
return port_fn_check_tx_rates(min, request->rate_tx_max);
|
||||
}
|
||||
|
||||
static int cmd_port_fn_rate_add(struct dl *dl)
|
||||
{
|
||||
struct nlmsghdr *nlh;
|
||||
int err;
|
||||
|
||||
nlh = mnlu_gen_socket_cmd_prepare(&dl->nlg, DEVLINK_CMD_RATE_NEW,
|
||||
NLM_F_REQUEST | NLM_F_ACK);
|
||||
err = dl_argv_parse_put(nlh, dl, DL_OPT_PORT_FN_RATE_NODE_NAME,
|
||||
DL_OPT_PORT_FN_RATE_TX_SHARE |
|
||||
DL_OPT_PORT_FN_RATE_TX_MAX);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if ((dl->opts.present & DL_OPT_PORT_FN_RATE_TX_SHARE) &&
|
||||
(dl->opts.present & DL_OPT_PORT_FN_RATE_TX_MAX)) {
|
||||
err = port_fn_check_tx_rates(dl->opts.rate_tx_share,
|
||||
dl->opts.rate_tx_max);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
return mnlu_gen_socket_sndrcv(&dl->nlg, nlh, NULL, NULL);
|
||||
}
|
||||
|
||||
static int cmd_port_fn_rate_del(struct dl *dl)
|
||||
{
|
||||
struct nlmsghdr *nlh;
|
||||
int err;
|
||||
|
||||
nlh = mnlu_gen_socket_cmd_prepare(&dl->nlg, DEVLINK_CMD_RATE_DEL,
|
||||
NLM_F_REQUEST | NLM_F_ACK);
|
||||
err = dl_argv_parse_put(nlh, dl, DL_OPT_PORT_FN_RATE_NODE_NAME, 0);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
return mnlu_gen_socket_sndrcv(&dl->nlg, nlh, NULL, NULL);
|
||||
}
|
||||
|
||||
static int port_fn_get_rates_cb(const struct nlmsghdr *nlh, void *data)
|
||||
{
|
||||
struct dl_opts *opts = data;
|
||||
struct nlattr *tb[DEVLINK_ATTR_MAX + 1] = {};
|
||||
struct genlmsghdr *genl = mnl_nlmsg_get_payload(nlh);
|
||||
|
||||
mnl_attr_parse(nlh, sizeof(*genl), attr_cb, tb);
|
||||
if ((!tb[DEVLINK_ATTR_BUS_NAME] || !tb[DEVLINK_ATTR_DEV_NAME] ||
|
||||
!tb[DEVLINK_ATTR_PORT_INDEX]) &&
|
||||
!tb[DEVLINK_ATTR_RATE_NODE_NAME]) {
|
||||
return MNL_CB_ERROR;
|
||||
}
|
||||
|
||||
if (tb[DEVLINK_ATTR_RATE_TX_SHARE])
|
||||
opts->rate_tx_share =
|
||||
mnl_attr_get_u64(tb[DEVLINK_ATTR_RATE_TX_SHARE]);
|
||||
if (tb[DEVLINK_ATTR_RATE_TX_MAX])
|
||||
opts->rate_tx_max =
|
||||
mnl_attr_get_u64(tb[DEVLINK_ATTR_RATE_TX_MAX]);
|
||||
return MNL_CB_OK;
|
||||
}
|
||||
|
||||
static int cmd_port_fn_rate_set(struct dl *dl)
|
||||
{
|
||||
struct dl_opts tmp_opts = {0};
|
||||
struct nlmsghdr *nlh;
|
||||
int err;
|
||||
|
||||
err = dl_argv_parse(dl, DL_OPT_HANDLEP |
|
||||
DL_OPT_PORT_FN_RATE_NODE_NAME,
|
||||
DL_OPT_PORT_FN_RATE_TX_SHARE |
|
||||
DL_OPT_PORT_FN_RATE_TX_MAX |
|
||||
DL_OPT_PORT_FN_RATE_PARENT);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if ((dl->opts.present & DL_OPT_PORT_FN_RATE_TX_SHARE) &&
|
||||
(dl->opts.present & DL_OPT_PORT_FN_RATE_TX_MAX)) {
|
||||
err = port_fn_check_tx_rates(dl->opts.rate_tx_share,
|
||||
dl->opts.rate_tx_max);
|
||||
if (err)
|
||||
return err;
|
||||
} else if (dl->opts.present &
|
||||
(DL_OPT_PORT_FN_RATE_TX_SHARE | DL_OPT_PORT_FN_RATE_TX_MAX)) {
|
||||
nlh = mnlu_gen_socket_cmd_prepare(&dl->nlg, DEVLINK_CMD_RATE_GET,
|
||||
NLM_F_REQUEST | NLM_F_ACK);
|
||||
tmp_opts = dl->opts;
|
||||
dl->opts.present &= ~(DL_OPT_PORT_FN_RATE_TX_SHARE |
|
||||
DL_OPT_PORT_FN_RATE_TX_MAX |
|
||||
DL_OPT_PORT_FN_RATE_PARENT);
|
||||
dl_opts_put(nlh, dl);
|
||||
err = mnlu_gen_socket_sndrcv(&dl->nlg, nlh, port_fn_get_rates_cb,
|
||||
&dl->opts);
|
||||
if (err)
|
||||
return err;
|
||||
err = port_fn_get_and_check_tx_rates(&dl->opts, &tmp_opts);
|
||||
if (err)
|
||||
return err;
|
||||
dl->opts = tmp_opts;
|
||||
}
|
||||
|
||||
nlh = mnlu_gen_socket_cmd_prepare(&dl->nlg, DEVLINK_CMD_RATE_SET,
|
||||
NLM_F_REQUEST | NLM_F_ACK);
|
||||
dl_opts_put(nlh, dl);
|
||||
return mnlu_gen_socket_sndrcv(&dl->nlg, nlh, NULL, NULL);
|
||||
}
|
||||
|
||||
static int cmd_port_function_rate(struct dl *dl)
|
||||
{
|
||||
if (dl_argv_match(dl, "help")) {
|
||||
cmd_port_fn_rate_help();
|
||||
return 0;
|
||||
} else if (dl_argv_match(dl, "show") || dl_no_arg(dl)) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_fn_rate_show(dl);
|
||||
} else if (dl_argv_match(dl, "add")) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_fn_rate_add(dl);
|
||||
} else if (dl_argv_match(dl, "del")) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_fn_rate_del(dl);
|
||||
} else if (dl_argv_match(dl, "set")) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_fn_rate_set(dl);
|
||||
}
|
||||
pr_err("Command \"%s\" not found\n", dl_argv(dl));
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
static int cmd_port_function(struct dl *dl)
|
||||
{
|
||||
if (dl_argv_match(dl, "help") || dl_no_arg(dl)) {
|
||||
|
|
@ -4326,6 +4788,9 @@ static int cmd_port_function(struct dl *dl)
|
|||
} else if (dl_argv_match(dl, "set")) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_function_set(dl);
|
||||
} else if (dl_argv_match(dl, "rate")) {
|
||||
dl_arg_inc(dl);
|
||||
return cmd_port_function_rate(dl);
|
||||
}
|
||||
pr_err("Command \"%s\" not found\n", dl_argv(dl));
|
||||
return -ENOENT;
|
||||
|
|
@ -8622,6 +9087,7 @@ int main(int argc, char **argv)
|
|||
{ "verbose", no_argument, NULL, 'v' },
|
||||
{ "statistics", no_argument, NULL, 's' },
|
||||
{ "Netns", required_argument, NULL, 'N' },
|
||||
{ "iec", no_argument, NULL, 'i' },
|
||||
{ NULL, 0, NULL, 0 }
|
||||
};
|
||||
const char *batch_file = NULL;
|
||||
|
|
@ -8637,7 +9103,7 @@ int main(int argc, char **argv)
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
while ((opt = getopt_long(argc, argv, "Vfb:njpvsN:",
|
||||
while ((opt = getopt_long(argc, argv, "Vfb:njpvsN:i",
|
||||
long_options, NULL)) >= 0) {
|
||||
|
||||
switch (opt) {
|
||||
|
|
@ -8672,6 +9138,9 @@ int main(int argc, char **argv)
|
|||
goto dl_free;
|
||||
}
|
||||
break;
|
||||
case 'i':
|
||||
use_iec = true;
|
||||
break;
|
||||
default:
|
||||
pr_err("Unknown option.\n");
|
||||
help();
|
||||
|
|
|
|||
|
|
@ -74,6 +74,10 @@ devlink-port \- devlink port configuration
|
|||
.BR "state"
|
||||
.RI "STATE }"
|
||||
|
||||
.ti -8
|
||||
.BR "devlink port function rate "
|
||||
.RI "{ " show " | " set " | " add " | " del " | " help " }"
|
||||
|
||||
.ti -8
|
||||
.B devlink dev param set
|
||||
.I DEV/PORT_INDEX
|
||||
|
|
@ -250,6 +254,10 @@ Configuration mode in which the new value is set.
|
|||
Specify parameter name to show.
|
||||
If this argument, as well as port index, are omitted - all parameters supported by devlink device ports are listed.
|
||||
|
||||
.SS devlink port function rate - manage devlink rate objects
|
||||
Is an alias for
|
||||
.BR devlink-rate (8).
|
||||
|
||||
.SH "EXAMPLES"
|
||||
.PP
|
||||
devlink port show
|
||||
|
|
|
|||
|
|
@ -0,0 +1,270 @@
|
|||
.TH DEVLINK\-RATE 8 "12 Mar 2021" "iproute2" "Linux"
|
||||
.SH NAME
|
||||
devlink-rate \- devlink rate management
|
||||
.SH SYNOPSIS
|
||||
.sp
|
||||
.ad l
|
||||
.in +8
|
||||
.ti -8
|
||||
.B devlink
|
||||
.RI "[ " OPTIONS " ]"
|
||||
.B port function rate
|
||||
.RI " { " COMMAND " | "
|
||||
.BR help " }"
|
||||
.sp
|
||||
|
||||
.ti -8
|
||||
.IR OPTIONS " := { "
|
||||
.BR -j [ \fIson "] | " -p [ \fIretty "] | " -i [ \fIec "] }"
|
||||
|
||||
.ti -8
|
||||
.B devlink port function rate show
|
||||
.RI "[ { " DEV/PORT_INDEX " | " DEV/NODE_NAME " } ]"
|
||||
|
||||
.ti -8
|
||||
.B devlink port function rate set
|
||||
.RI "{ " DEV/PORT_INDEX " | " DEV/NODE_NAME " } "
|
||||
.RB [ " tx_share \fIVALUE " ]
|
||||
.RB [ " tx_max \fIVALUE " ]
|
||||
.RB "[ {" " parent \fINODE_NAME " | " noparent " "} ]"
|
||||
|
||||
.ti -8
|
||||
.BI "devlink port function rate add " DEV/NODE_NAME
|
||||
.RB [ " tx_share \fIVALUE " ]
|
||||
.RB [ " tx_max \fIVALUE " ]
|
||||
.RB "[ {" " parent \fINODE_NAME " | " noparent " "} ]"
|
||||
|
||||
.ti -8
|
||||
.BI "devlink port function rate del " DEV/NODE_NAME
|
||||
|
||||
.ti -8
|
||||
.B devlink port function rate help
|
||||
|
||||
.SH "DESCRIPTION"
|
||||
|
||||
.SS devlink port function rate show - display rate objects.
|
||||
Displays specified rate object or, if not specified, all rate objects. Rate
|
||||
object can be presented by one of the two types:
|
||||
.TP 8
|
||||
.B leaf
|
||||
Represents a single devlink port; created/destroyed by the driver and bound to
|
||||
the devlink port. As example, some driver may create leaf rate object for every
|
||||
devlink port associated with VF. Since leaf have 1to1 mapping to it's devlink
|
||||
port, in user space it is referred as corresponding devlink port
|
||||
\fIDEV/PORT_INDEX\fR;
|
||||
.TP 8
|
||||
.B node
|
||||
Represents a group of rate objects; created/deleted by the user (see command
|
||||
below) and bound to the devlink device rather then to the devlink port. In
|
||||
userspace it is referred as \fIDEV/NODE_NAME\fR, where node name can be any,
|
||||
except decimal number, to avoid collisions with leafs.
|
||||
.PP
|
||||
Command output show rate object identifier, it's type and rate values along with
|
||||
parent node name. Rate values printed in SI units which are more suitable to
|
||||
represent specific value. To print values in IEC units \fB-i\fR switch is
|
||||
used. JSON (\fB-j\fR) output always print rate values in bytes per second. Zero
|
||||
rate values means "unlimited" rates and ommited in output, as well as parent
|
||||
node name.
|
||||
|
||||
.SS devlink port function rate set - set rate object parameters.
|
||||
Allows set rate object's parameters. If any parameter specified multiple times
|
||||
the last occurrence is used.
|
||||
.PP
|
||||
.I DEV/PORT_INDEX
|
||||
- specifies devlink leaf rate object.
|
||||
.br
|
||||
.I DEV/NODE_NAME
|
||||
- specifies devlink node rate object.
|
||||
.PP
|
||||
.BI tx_share " VALUE"
|
||||
- specifies minimal tx rate value shared among all rate objects. If rate object
|
||||
is a part of some rate group, then this value shared with rate objects of this
|
||||
rate group.
|
||||
.PP
|
||||
.BI tx_max " VALUE"
|
||||
- specifies maximum tx rate value.
|
||||
.TP 8
|
||||
.I VALUE
|
||||
These parameter accept a floating point number, possibly followed by either a
|
||||
unit (both SI and IEC units supported).
|
||||
.RS
|
||||
.TP
|
||||
bit or a bare number
|
||||
Bits per second
|
||||
.TP
|
||||
kbit
|
||||
Kilobits per second
|
||||
.TP
|
||||
mbit
|
||||
Megabits per second
|
||||
.TP
|
||||
gbit
|
||||
Gigabits per second
|
||||
.TP
|
||||
tbit
|
||||
Terabits per second
|
||||
.TP
|
||||
bps
|
||||
Bytes per second
|
||||
.TP
|
||||
kbps
|
||||
Kilobytes per second
|
||||
.TP
|
||||
mbps
|
||||
Megabytes per second
|
||||
.TP
|
||||
gbps
|
||||
Gigabytes per second
|
||||
.TP
|
||||
tbps
|
||||
Terabytes per second
|
||||
.P
|
||||
To specify in IEC units, replace the SI prefix (k-, m-, g-, t-) with IEC prefix
|
||||
(ki-, mi-, gi- and ti-) respectively. Input is case-insensitive.
|
||||
.RE
|
||||
.PP
|
||||
.BI parent " NODE_NAME \fR| " noparent
|
||||
- set rate object parent to existing node with name \fINODE_NAME\fR or unset
|
||||
parent. Rate limits of the parent node applied to all it's children. Actual
|
||||
behaviour is details of driver's implementation. Setting parent to empty ("")
|
||||
name due to the kernel logic threated as parent unset.
|
||||
|
||||
.SS devlink port function rate add - create node rate object with specified parameters.
|
||||
Creates rate object of type node and sets parameters. Parameters same as for the
|
||||
"set" command.
|
||||
.PP
|
||||
.I DEV/NODE_NAME
|
||||
- specifies the devlink node rate object to create.
|
||||
|
||||
.SS devlink port function rate del - delete node rate object
|
||||
Delete specified devlink node rate object. Node can't be deleted if there is any
|
||||
child, user must explicitly unset the parent.
|
||||
.PP
|
||||
.I DEV/NODE_NAME
|
||||
- specifies devlink node rate object to delete.
|
||||
|
||||
.SS devlink port function rate help - display usage information
|
||||
Display devlink rate usage information
|
||||
|
||||
.SH "EXAMPLES"
|
||||
|
||||
.PP
|
||||
\fB*\fR Display all rate objects:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate show
|
||||
.br
|
||||
pci/0000:03:00.0/1 type leaf parent some_group
|
||||
.br
|
||||
pci/0000:03:00.0/2 type leaf tx_share 12Mbit
|
||||
.br
|
||||
pci/0000:03:00.0/some_group type node tx_share 1Gbps tx_max 5Gbps
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Display leaf rate object bound to the 1st devlink port of the
|
||||
pci/0000:03:00.0 device:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate show pci/0000:03:00.0/1
|
||||
.br
|
||||
pci/0000:03:00.0/1 type leaf
|
||||
.br
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Display leaf rate object rate values using IEC units:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink -i port function rate show pci/0000:03:00.0/2
|
||||
.br
|
||||
pci/0000:03:00.0/2 type leaf 11718Kibit
|
||||
.br
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Display node rate object with name some_group of the pci/0000:03:00.0 device:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate show pci/0000:03:00.0/some_group
|
||||
.br
|
||||
pci/0000:03:00.0/some_group type node
|
||||
.br
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Display pci/0000:03:00.0/2 leaf rate object as pretty JSON output:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink -jp port function rate show pci/0000:03:00.0/2
|
||||
.br
|
||||
{
|
||||
.br
|
||||
"rate": {
|
||||
.br
|
||||
"pci/0000:03:00.0/2": {
|
||||
.br
|
||||
"type": "leaf",
|
||||
.br
|
||||
"tx_share": 1500000
|
||||
.br
|
||||
}
|
||||
.br
|
||||
}
|
||||
.br
|
||||
}
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Create node rate object with name "1st_group" on pci/0000:03:00.0 device:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate add pci/0000:03:00.0/1st_group
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Create node rate object with specified parameters:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate add pci/0000:03:00.0/2nd_group \\
|
||||
.br
|
||||
tx_share 10Mbit tx_max 30Mbit parent 1st_group
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Set parameters to the specified leaf rate object:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate set pci/0000:03:00.0/1 \\
|
||||
.br
|
||||
tx_share 2Mbit tx_max 10Mbit
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Set leaf's parent to "1st_group":
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate set pci/0000:03:00.0/1 parent 1st_group
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Unset leaf's parent:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate set pci/0000:03:00.0/1 noparent
|
||||
.RE
|
||||
|
||||
.PP
|
||||
\fB*\fR Delete node rate object:
|
||||
.RS 4
|
||||
.PP
|
||||
# devlink port function rate del pci/0000:03:00.0/2nd_group
|
||||
.RE
|
||||
|
||||
.SH SEE ALSO
|
||||
.BR devlink (8),
|
||||
.BR devlink-port (8)
|
||||
.br
|
||||
|
||||
.SH AUTHOR
|
||||
Dmytro Linkin <dlinkin@nvidia.com>
|
||||
|
|
@ -59,6 +59,10 @@ Output statistics.
|
|||
.BR "\-N", " \-Netns " <NETNSNAME>
|
||||
Switches to the specified network namespace.
|
||||
|
||||
.TP
|
||||
.BR "\-i", " --iec"
|
||||
Print human readable rates in IEC units (e.g. 1Ki = 1024).
|
||||
|
||||
.SS
|
||||
.I OBJECT
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue