In order to implement devices controller with cgroup v2 we need to add support for BPF programs, cgroup v2 doesn't have devices controller. This introduces required helpers wrapping linux syscalls. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- include/libvirt/virterror.h | 1 + src/libvirt_private.syms | 16 +++ src/util/Makefile.inc.am | 2 + src/util/virbpf.c | 263 ++++++++++++++++++++++++++++++++++++ src/util/virbpf.h | 246 +++++++++++++++++++++++++++++++++ src/util/virerror.c | 1 + 6 files changed, 529 insertions(+) create mode 100644 src/util/virbpf.c create mode 100644 src/util/virbpf.h diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index fbbe2d5624..d47bed4390 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -131,6 +131,7 @@ typedef enum { VIR_FROM_PERF = 65, /* Error from perf */ VIR_FROM_LIBSSH = 66, /* Error from libssh connection transport */ VIR_FROM_RESCTRL = 67, /* Error from resource control */ + VIR_FROM_BPF = 68, /* Error from BPF code */ # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index c3d6306809..0cff580de2 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1477,6 +1477,22 @@ virBitmapToDataBuf; virBitmapToString; +# util/virbpf.h +virBPFAttachProg; +virBPFCreateMap; +virBPFDeleteElem; +virBPFDetachProg; +virBPFGetMap; +virBPFGetMapInfo; +virBPFGetNextElem; +virBPFGetProg; +virBPFGetProgInfo; +virBPFLoadProg; +virBPFLookupElem; +virBPFQueryProg; +virBPFUpdateElem; + + # util/virbuffer.h virBufferAdd; virBufferAddBuffer; diff --git a/src/util/Makefile.inc.am b/src/util/Makefile.inc.am index 4295babac3..1fd7ad2d43 100644 --- a/src/util/Makefile.inc.am +++ b/src/util/Makefile.inc.am @@ -17,6 +17,8 @@ UTIL_SOURCES = \ util/virauthconfig.h \ util/virbitmap.c \ util/virbitmap.h \ + util/virbpf.c \ + util/virbpf.h \ util/virbuffer.c \ util/virbuffer.h \ util/virperf.c \ diff --git a/src/util/virbpf.c b/src/util/virbpf.c new file mode 100644 index 0000000000..be5ebbc033 --- /dev/null +++ b/src/util/virbpf.c @@ -0,0 +1,263 @@ +/* + * virbpf.c: methods for eBPF + * + * Copyright (C) 2018 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ +#include <config.h> + +#include <sys/syscall.h> + +#include "internal.h" + +#include "virbpf.h" +#include "virerror.h" +#include "virfile.h" +#include "virlog.h" +#include "virstring.h" + +VIR_LOG_INIT("util.bpf"); + +#define VIR_FROM_THIS VIR_FROM_BPF + +int +virBPFCreateMap(unsigned int mapType, + unsigned int keySize, + unsigned int valSize, + unsigned int maxEntries) +{ + union bpf_attr attr = { + .map_type = mapType, + .key_size = keySize, + .value_size = valSize, + .max_entries = maxEntries, + }; + + return syscall(SYS_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); +} + +#define LOG_BUF_SIZE (256 * 1024) + +int +virBPFLoadProg(struct bpf_insn *insns, + int progType, + unsigned int insnCnt) +{ + VIR_AUTOFREE(char *) logbuf = NULL; + int progfd = -1; + + if (VIR_ALLOC_N(logbuf, LOG_BUF_SIZE) < 0) + return -1; + + union bpf_attr attr = { + .prog_type = progType, + .insn_cnt = (__u32)insnCnt, + .insns = (__u64)insns, + .license = (__u64)"GPL", + .log_buf = (__u64)logbuf, + .log_size = LOG_BUF_SIZE, + .log_level = 1, + }; + + progfd = syscall(SYS_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); + + if (progfd < 0) + VIR_DEBUG("%s", logbuf); + + return progfd; +} + +int +virBPFAttachProg(int progfd, + int targetfd, + int attachType) +{ + union bpf_attr attr = { + .target_fd = targetfd, + .attach_bpf_fd = progfd, + .attach_type = attachType, + }; + + return syscall(SYS_bpf, BPF_PROG_ATTACH, &attr, sizeof(attr)); +} + +int +virBPFDetachProg(int progfd, + int targetfd, + int attachType) +{ + union bpf_attr attr = { + .target_fd = targetfd, + .attach_bpf_fd = progfd, + .attach_type = attachType, + }; + + return syscall(SYS_bpf, BPF_PROG_DETACH, &attr, sizeof(attr)); +} + +int +virBPFQueryProg(int targetfd, + unsigned int maxprogids, + int attachType, + unsigned int *progcnt, + void *progids) +{ + int rc; + + union bpf_attr attr = { + .query.target_fd = targetfd, + .query.attach_type = attachType, + .query.prog_cnt = maxprogids, + .query.prog_ids = (__u64)progids, + }; + + rc = syscall(SYS_bpf, BPF_PROG_QUERY, &attr, sizeof(attr)); + + if (rc >= 0) + *progcnt = attr.query.prog_cnt; + + return rc; +} + +int +virBPFGetProg(unsigned int id) +{ + union bpf_attr attr = { + .prog_id = id, + }; + + return syscall(SYS_bpf, BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr)); +} + +int +virBPFGetProgInfo(int progfd, + struct bpf_prog_info *info, + unsigned int **mapIDs) +{ + int rc; + + union bpf_attr attr = { + .info.bpf_fd = progfd, + .info.info_len = sizeof(struct bpf_prog_info), + .info.info = (__u64)info, + }; + + rc = syscall(SYS_bpf, BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); + if (rc < 0) + return rc; + + if (mapIDs && info->nr_map_ids > 0) { + unsigned int maplen = info->nr_map_ids; + VIR_AUTOFREE(unsigned int *) retmapIDs = NULL; + + if (VIR_ALLOC_N(retmapIDs, maplen) < 0) + return -1; + + memset(info, 0, sizeof(struct bpf_prog_info)); + info->nr_map_ids = maplen; + info->map_ids = (__u64)retmapIDs; + + memset(&attr, 0, sizeof(attr)); + attr.info.bpf_fd = progfd; + attr.info.info_len = sizeof(struct bpf_prog_info); + attr.info.info = (__u64)info; + + rc = syscall(SYS_bpf, BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); + if (rc < 0) + return rc; + + VIR_STEAL_PTR(*mapIDs, retmapIDs); + } + + return rc; +} + +int +virBPFGetMap(unsigned int id) +{ + union bpf_attr attr = { + .map_id = id, + }; + + return syscall(SYS_bpf, BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr)); +} + +int +virBPFGetMapInfo(int mapfd, + struct bpf_map_info *info) +{ + union bpf_attr attr = { + .info.bpf_fd = mapfd, + .info.info_len = sizeof(struct bpf_map_info), + .info.info = (__u64)info, + }; + + return syscall(SYS_bpf, BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); +} + +int +virBPFLookupElem(int mapfd, + void *key, + void *val) +{ + union bpf_attr attr = { + .map_fd = mapfd, + .key = (__u64)key, + .value = (__u64)val, + }; + + return syscall(SYS_bpf, BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)); +} + +int +virBPFGetNextElem(int mapfd, + void *key, + void *nextKey) +{ + union bpf_attr attr = { + .map_fd = mapfd, + .key = (__u64)key, + .next_key = (__u64)nextKey, + }; + + return syscall(SYS_bpf, BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr)); +} + +int +virBPFUpdateElem(int mapfd, + void *key, + void *val) +{ + union bpf_attr attr = { + .map_fd = mapfd, + .key = (__u64)key, + .value = (__u64)val, + }; + + return syscall(SYS_bpf, BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); +} + +int +virBPFDeleteElem(int mapfd, + void *key) +{ + union bpf_attr attr = { + .map_fd = mapfd, + .key = (__u64)key, + }; + + return syscall(SYS_bpf, BPF_MAP_DELETE_ELEM, &attr, sizeof(attr)); +} diff --git a/src/util/virbpf.h b/src/util/virbpf.h new file mode 100644 index 0000000000..7085edaacc --- /dev/null +++ b/src/util/virbpf.h @@ -0,0 +1,246 @@ +/* + * virbpf.h: methods for eBPF + * + * Copyright (C) 2018 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#ifndef LIBVIRT_VIRBPF_H +# define LIBVIRT_VIRBPF_H + +# include <linux/bpf.h> + +/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */ + +# define VIR_BPF_ALU64_REG(op, dst, src) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_OP(op) | BPF_X, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = 0, \ + .imm = 0, \ + }) + +/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */ + +# define VIR_BPF_ALU64_IMM(op, dst, immval) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_OP(op) | BPF_K, \ + .dst_reg = dst, \ + .src_reg = 0, \ + .off = 0, \ + .imm = immval, \ + }) + +/* Short form of mov, dst_reg = src_reg */ + +# define VIR_BPF_MOV64_REG(dst, src) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_MOV | BPF_X, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = 0, \ + .imm = 0, \ + }) + +/* Short form of mov, dst_reg = imm32 */ + +# define VIR_BPF_MOV64_IMM(dst, immval) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_MOV | BPF_K, \ + .dst_reg = dst, \ + .src_reg = 0, \ + .off = 0, \ + .imm = immval, \ + }) + +# define VIR_BPF_MOV32_IMM(dst, immval) \ + ((struct bpf_insn) { \ + .code = BPF_ALU | BPF_MOV | BPF_K, \ + .dst_reg = dst, \ + .src_reg = 0, \ + .off = 0, \ + .imm = immval, \ + }) + +/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */ +# define VIR_BPF_LD_IMM64(dst, imm) \ + BPF_LD_IMM64_RAW(dst, 0, imm) + +# define VIR_BPF_LD_IMM64_RAW(dst, src, immval) \ + ((struct bpf_insn) { \ + .code = BPF_LD | BPF_DW | BPF_IMM, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = 0, \ + .imm = (__u32)immval, \ + }), \ + ((struct bpf_insn) { \ + .code = 0, \ + .dst_reg = 0, \ + .src_reg = 0, \ + .off = 0, \ + .imm = ((__u64)immval) >> 32, \ + }) + +# ifndef VIR_BPF_PSEUDO_MAP_FD +# define VIR_BPF_PSEUDO_MAP_FD 1 +# endif + +/* pseudo VIR_BPF_LD_IMM64 insn used to refer to process-local map_fd */ +# define VIR_BPF_LD_MAP_FD(dst, mapfd) \ + VIR_BPF_LD_IMM64_RAW(dst, VIR_BPF_PSEUDO_MAP_FD, mapfd) + +/* Memory load, dst_reg = *(uint *) (src_reg + off16) */ + +# define VIR_BPF_LDX_MEM(size, dst, src, offval) \ + ((struct bpf_insn) { \ + .code = BPF_LDX | BPF_SIZE(size) | BPF_MEM, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = offval, \ + .imm = 0, \ + }) + +/* Memory store, *(uint *) (dst_reg + off16) = src_reg */ + +# define VIR_BPF_STX_MEM(size, dst, src, offval) \ + ((struct bpf_insn) { \ + .code = BPF_STX | BPF_SIZE(size) | BPF_MEM, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = offval, \ + .imm = 0, \ + }) + +/* Memory store, *(uint *) (dst_reg + off16) = imm32 */ + +# define VIR_BPF_ST_MEM(size, dst, immval, offval) \ + ((struct bpf_insn) { \ + .code = BPF_ST | BPF_SIZE(size) | BPF_MEM, \ + .dst_reg = dst, \ + .src_reg = 0, \ + .off = offval, \ + .imm = immval, \ + }) + +/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */ + +# define VIR_BPF_JMP_REG(op, dst, src, offval) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_OP(op) | BPF_X, \ + .dst_reg = dst, \ + .src_reg = src, \ + .off = offval, \ + .imm = 0, \ + }) + +/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */ + +# define VIR_BPF_JMP_IMM(op, dst, immval, offval) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_OP(op) | BPF_K, \ + .dst_reg = dst, \ + .src_reg = 0, \ + .off = offval, \ + .imm = immval, \ + }) + +/* Call eBPF function */ + +# define VIR_BPF_CALL_INSN(func) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_CALL, \ + .dst_reg = 0, \ + .src_reg = 0, \ + .off = 0, \ + .imm = func, \ + }) + +/* Program exit */ + +# define VIR_BPF_EXIT_INSN() \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_EXIT, \ + .dst_reg = 0, \ + .src_reg = 0, \ + .off = 0, \ + .imm = 0, \ + }) + +int +virBPFCreateMap(unsigned int mapType, + unsigned int keySize, + unsigned int valSize, + unsigned int maxEntries); + +int +virBPFGetMapInfo(int mapfd, + struct bpf_map_info *info); + +int +virBPFLoadProg(struct bpf_insn *insns, + int progType, + unsigned int insnCnt); + +int +virBPFAttachProg(int progfd, + int targetfd, + int attachType); + +int +virBPFDetachProg(int progfd, + int targetfd, + int attachType); + +int +virBPFQueryProg(int targetfd, + unsigned int maxprogids, + int attachType, + unsigned int *progcnt, + void *progids); + +int +virBPFGetProg(unsigned int id); + +int +virBPFGetProgInfo(int progfd, + struct bpf_prog_info *info, + unsigned int **mapIDs); + +int +virBPFGetMap(unsigned int id); + +int +virBPFLookupElem(int mapfd, + void *key, + void *val); + +int +virBPFGetNextElem(int mapfd, + void *key, + void *nextKey); + +int +virBPFUpdateElem(int mapfd, + void *key, + void *val); + +int +virBPFDeleteElem(int mapfd, + void *key); + +#endif /* LIBVIRT_VIRBPF_H */ diff --git a/src/util/virerror.c b/src/util/virerror.c index 61b47d2be0..a40076f8ec 100644 --- a/src/util/virerror.c +++ b/src/util/virerror.c @@ -138,6 +138,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST, "Perf", /* 65 */ "Libssh transport layer", "Resource control", + "BPF", ) -- 2.20.1

There are two possible ways how to create BPF program: - One way is to write simple C-like code which can by compiled into BPF object file which can be loaded into kernel using elfutils. - The second way is to define macros which looks like assembler instructions and can be used directly to create BPF program that can be directly loaded into kernel. Since the program is not too complex we can use the second option. If there is no program, all devices are allowed, if there is some program it is executed and based on the exit status the access is denied for 0 and allowed for 1. Our program will follow these rules: - first it will try to look for the specific key using major and minor to see if there is any rule for that specific device - if there is no specific rule it will try to look for any rule that matches only major of the device - if there is no match with major it will try the same but with minor of the device - as the last attempt it will try to look for rule for all devices and if there is no match it will return 0 to deny that access Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index 0adc0d4d23..63e3123cd9 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -1595,6 +1595,82 @@ virCgroupV2GetCpuacctStat(virCgroupPtr group, } +static int +virCgroupV2DeviceLoadProg(int mapfd) +{ +# define VIR_CGROUP_BPF_LOOKUP \ + /* prepare key param on stack */ \ + VIR_BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8), \ + VIR_BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), \ + VIR_BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8), \ + /* lookup key (major << 32) | minor in map */ \ + VIR_BPF_LD_MAP_FD(BPF_REG_1, mapfd), \ + VIR_BPF_CALL_INSN(BPF_FUNC_map_lookup_elem) + +# define VIR_CGROUP_BPF_CHECK_PERM \ + /* if no key skip perm check */ \ + VIR_BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 6), \ + /* get perms from map */ \ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0), \ + /* get perms from ctx */ \ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, 0), \ + /* (map perms) & (ctx perms) */ \ + VIR_BPF_ALU64_REG(BPF_AND, BPF_REG_1, BPF_REG_2), \ + /* if (map perms) & (ctx perms) == (ctx perms) exit, otherwise continue */ \ + VIR_BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_2, 2), \ + /* set ret 1 and exit */ \ + VIR_BPF_MOV64_IMM(BPF_REG_0, 1), \ + VIR_BPF_EXIT_INSN() + + struct bpf_insn prog[] = { + /* save ctx, argument passed to BPF program */ + VIR_BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), + + /* get major from ctx and shift << 32 */ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, 4), + VIR_BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 32), + /* get minor from ctx and | to shifted major */ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, 8), + VIR_BPF_ALU64_REG(BPF_OR, BPF_REG_2, BPF_REG_3), + /* lookup ((major << 32) | minor) in map and check perms */ + VIR_CGROUP_BPF_LOOKUP, + VIR_CGROUP_BPF_CHECK_PERM, + + /* get major from ctx and shift << 32 */ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, 4), + VIR_BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 32), + /* use -1 as minor and | to shifted major */ + VIR_BPF_MOV32_IMM(BPF_REG_3, -1), + VIR_BPF_ALU64_REG(BPF_OR, BPF_REG_2, BPF_REG_3), + /* lookup ((major << 32) | -1) in map and check perms */ + VIR_CGROUP_BPF_LOOKUP, + VIR_CGROUP_BPF_CHECK_PERM, + + /* use -1 as major and shift << 32 */ + VIR_BPF_MOV32_IMM(BPF_REG_2, -1), + VIR_BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 32), + /* get minor from ctx and | to shifted major */ + VIR_BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, 8), + VIR_BPF_ALU64_REG(BPF_OR, BPF_REG_2, BPF_REG_3), + /* lookup ((-1 << 32) | minor) in map and check perms */ + VIR_CGROUP_BPF_LOOKUP, + VIR_CGROUP_BPF_CHECK_PERM, + + /* use -1 as key which means major = -1 and minor = -1 */ + VIR_BPF_MOV64_IMM(BPF_REG_2, -1), + /* lookup -1 in map and check perms*/ + VIR_CGROUP_BPF_LOOKUP, + VIR_CGROUP_BPF_CHECK_PERM, + + /* no key was found, exit with 0 */ + VIR_BPF_MOV64_IMM(BPF_REG_0, 0), + VIR_BPF_EXIT_INSN(), + }; + + return virBPFLoadProg(prog, BPF_PROG_TYPE_CGROUP_DEVICE, ARRAY_CARDINALITY(prog)); +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, -- 2.20.1

This function will be called if libvirtd was restarted while some domains were running. It will try to detect existing programs attached to the guest cgroup. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 105 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index 7a8cc040eb..86adbc4ff9 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -1723,6 +1723,111 @@ virCgroupV2DeviceAttachProg(virCgroupPtr group, } +static int +virCgroupV2DeviceCountMapEntries(int mapfd) +{ + int ret = 0; + int rc; + __u64 key = 0; + __u64 prevKey = 0; + + while ((rc = virBPFGetNextElem(mapfd, &prevKey, &key)) == 0) { + ret++; + prevKey = key; + } + + if (rc < 0) + return -1; + + return ret; +} + + +# define MAX_PROG_IDS 10 + +static int +virCgroupV2DeviceDetectProg(virCgroupPtr group) +{ + int ret = -1; + int cgroupfd = -1; + unsigned int progcnt = 0; + unsigned int progids[MAX_PROG_IDS] = { 0 }; + VIR_AUTOFREE(char *) path = NULL; + + if (group->unified.devices.progfd > 0 && group->unified.devices.mapfd > 0) + return 0; + + if (virCgroupV2PathOfController(group, VIR_CGROUP_CONTROLLER_DEVICES, + NULL, &path) < 0) { + return -1; + } + + cgroupfd = open(path, O_RDONLY); + if (cgroupfd < 0) { + virReportSystemError(errno, _("unable to open '%s'"), path); + goto cleanup; + } + + if (virBPFQueryProg(cgroupfd, MAX_PROG_IDS, BPF_CGROUP_DEVICE, + &progcnt, progids) < 0) { + virReportSystemError(errno, "%s", _("unable to query cgroup BPF progs")); + goto cleanup; + } + + if (progcnt > 0) { + int progfd = virBPFGetProg(progids[0]); + int mapfd = -1; + int nitems = -1; + struct bpf_prog_info progInfo = { 0 }; + struct bpf_map_info mapInfo = { 0 }; + VIR_AUTOFREE(unsigned int *) mapIDs = NULL; + + if (progfd < 0) { + virReportSystemError(errno, "%s", _("failed to get cgroup BPF prog FD")); + goto cleanup; + } + + if (virBPFGetProgInfo(progfd, &progInfo, &mapIDs) < 0) { + virReportSystemError(errno, "%s", _("failed to get cgroup BPF prog info")); + goto cleanup; + } + + if (progInfo.nr_map_ids == 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("no map for cgroup BPF prog")); + goto cleanup; + } + + mapfd = virBPFGetMap(mapIDs[0]); + if (mapfd < 0) { + virReportSystemError(errno, "%s", _("failed to get cgroup BPF map FD")); + goto cleanup; + } + + if (virBPFGetMapInfo(mapfd, &mapInfo) < 0) { + virReportSystemError(errno, "%s", _("failed to get cgroup BPF map info")); + goto cleanup; + } + + nitems = virCgroupV2DeviceCountMapEntries(mapfd); + if (nitems < 0) { + virReportSystemError(errno, "%s", _("failed to count cgroup BPF map items")); + goto cleanup; + } + + group->unified.devices.progfd = progfd; + group->unified.devices.mapfd = mapfd; + group->unified.devices.max = mapInfo.max_entries; + group->unified.devices.count = nitems; + } + + ret = 0; + cleanup: + VIR_FORCE_CLOSE(cgroupfd); + return ret; +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, -- 2.20.1

This function will create a new BPF map and copy all existing rules from old BPF map. There is no way how to reallocate existing BPF map so we need to create a new copy if we run out of space in current BPF map. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index ae9aafd9a4..d70eefd92f 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -1846,6 +1846,49 @@ virCgroupV2DeviceCreateMap(size_t size) } +static int +virCgroupV2DeviceReallocMap(int mapfd, + size_t size) +{ + __u64 key = 0; + __u64 prevKey = 0; + int rc; + int newmapfd = virCgroupV2DeviceCreateMap(size); + + VIR_DEBUG("realloc devices map mapfd:%d, size:%lu", mapfd, size); + + if (newmapfd < 0) + return -1; + + while ((rc = virBPFGetNextElem(mapfd, &prevKey, &key)) == 0) { + __u32 val = 0; + + if (virBPFLookupElem(mapfd, &key, &val) < 0) { + virReportSystemError(errno, "%s", + _("failed to lookup device in old map")); + goto error; + } + + if (virBPFUpdateElem(newmapfd, &key, &val) < 0) { + virReportSystemError(errno, "%s", + _("failed to add device into new map")); + goto error; + } + + prevKey = key; + } + + if (rc < 0) + goto error; + + return newmapfd; + + error: + VIR_FORCE_CLOSE(newmapfd); + return -1; +} + + static int virCgroupV2DeviceCreateProg(virCgroupPtr group) { -- 2.20.1

We need to close our FD that we have for BPF program and map in order to let kernel remove all resources once the cgroup is removed as well. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index aea094a328..839f19c1f6 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -454,6 +454,10 @@ virCgroupV2MakeGroup(virCgroupPtr parent ATTRIBUTE_UNUSED, } +static int +virCgroupV2DeviceRemoveProg(virCgroupPtr group); + + static int virCgroupV2Remove(virCgroupPtr group) { @@ -469,6 +473,9 @@ virCgroupV2Remove(virCgroupPtr group) if (virCgroupV2PathOfController(group, controller, "", &grppath) < 0) return 0; + if (virCgroupV2DeviceRemoveProg(group) < 0) + return -1; + return virCgroupRemoveRecursively(grppath); } @@ -1941,6 +1948,25 @@ virCgroupV2DevicePrepareProg(virCgroupPtr group) } +static int +virCgroupV2DeviceRemoveProg(virCgroupPtr group) +{ + if (virCgroupV2DeviceDetectProg(group) < 0) + return -1; + + if (group->unified.devices.progfd <= 0 && group->unified.devices.mapfd <= 0) + return 0; + + if (group->unified.devices.mapfd >= 0) + VIR_FORCE_CLOSE(group->unified.devices.mapfd); + + if (group->unified.devices.progfd >= 0) + VIR_FORCE_CLOSE(group->unified.devices.progfd); + + return 0; +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, -- 2.20.1

Device rules are stored in BPF map that is a hash type, this function will create a key based on major and minor id of device. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index 30f8da22ac..198d2ea03e 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -1993,6 +1993,14 @@ virCgroupV2DeviceGetPerms(int perms, } +static __u64 +virCgroupV2DeviceGetKey(int major, + int minor) +{ + return (__u64)major << 32 | ((__u64)minor & 0x00000000ffffffff); +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, -- 2.20.1

In order to deny device we need to check if there is any entry in BPF map and we need to load the current value from map if there is already entry for that device. If both values are same we can remove that entry but if they are different we need to update the entry because we don't have to deny all access, but for example only write access. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index e579464ff3..aea7ba677f 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -2030,6 +2030,46 @@ virCgroupV2AllowDevice(virCgroupPtr group, } +static int +virCgroupV2DenyDevice(virCgroupPtr group, + char type, + int major, + int minor, + int perms) +{ + __u64 key = virCgroupV2DeviceGetKey(major, minor); + __u32 newval = virCgroupV2DeviceGetPerms(perms, type); + __u32 val = 0; + + if (virCgroupV2DevicePrepareProg(group) < 0) + return -1; + + if (group->unified.devices.count <= 0 || + virBPFLookupElem(group->unified.devices.mapfd, &key, &val) < 0) { + VIR_DEBUG("nothing to do, device is not allowed"); + return 0; + } + + if (newval == val) { + if (virBPFDeleteElem(group->unified.devices.mapfd, &key) < 0) { + virReportSystemError(errno, "%s", + _("failed to remove device from BPF cgroup map")); + return -1; + } + group->unified.devices.count--; + } else { + val ^= val & newval; + if (virBPFUpdateElem(group->unified.devices.mapfd, &key, &val) < 0) { + virReportSystemError(errno, "%s", + _("failed to update device in BPF cgroup map")); + return -1; + } + } + + return 0; +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, @@ -2080,6 +2120,7 @@ virCgroupBackend virCgroupV2Backend = { .getMemSwapUsage = virCgroupV2GetMemSwapUsage, .allowDevice = virCgroupV2AllowDevice, + .denyDevice = virCgroupV2DenyDevice, .setCpuShares = virCgroupV2SetCpuShares, .getCpuShares = virCgroupV2GetCpuShares, -- 2.20.1

If we want to deny all devices we just need to replace any existing program with new program with empty map. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index 6c3f2bf522..4bcdd16814 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -2087,6 +2087,16 @@ virCgroupV2AllowAllDevices(virCgroupPtr group, } +static int +virCgroupV2DenyAllDevices(virCgroupPtr group) +{ + if (virCgroupV2DeviceDetectProg(group) < 0) + return -1; + + return virCgroupV2DeviceCreateProg(group); +} + + virCgroupBackend virCgroupV2Backend = { .type = VIR_CGROUP_BACKEND_TYPE_V2, @@ -2139,6 +2149,7 @@ virCgroupBackend virCgroupV2Backend = { .allowDevice = virCgroupV2AllowDevice, .denyDevice = virCgroupV2DenyDevice, .allowAllDevices = virCgroupV2AllowAllDevices, + .denyAllDevices = virCgroupV2DenyAllDevices, .setCpuShares = virCgroupV2SetCpuShares, .getCpuShares = virCgroupV2GetCpuShares, -- 2.20.1

This function simply removes program from guest cgroup before we remove the cgroup. This is required step because there is a bug [1] in kernel where the program might not be properly freed if you remove cgroup with attached program. [1] <https://bugzilla.redhat.com/show_bug.cgi?id=1656432> Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/vircgroupv2.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/util/vircgroupv2.c b/src/util/vircgroupv2.c index e28703df89..0a4aa15d0b 100644 --- a/src/util/vircgroupv2.c +++ b/src/util/vircgroupv2.c @@ -1954,19 +1954,44 @@ virCgroupV2DevicePrepareProg(virCgroupPtr group) static int virCgroupV2DeviceRemoveProg(virCgroupPtr group) { + int ret = -1; + int cgroupfd = -1; + VIR_AUTOFREE(char *) path = NULL; + if (virCgroupV2DeviceDetectProg(group) < 0) return -1; if (group->unified.devices.progfd <= 0 && group->unified.devices.mapfd <= 0) return 0; + if (virCgroupV2PathOfController(group, VIR_CGROUP_CONTROLLER_DEVICES, + NULL, &path) < 0) { + return -1; + } + + cgroupfd = open(path, O_RDONLY); + if (cgroupfd < 0) { + virReportSystemError(errno, _("unable to open '%s'"), path); + goto cleanup; + } + + if (virBPFDetachProg(group->unified.devices.progfd, + cgroupfd, BPF_CGROUP_DEVICE) < 0) { + virReportSystemError(errno, "%s", _("failed to detach cgroup BPF prog")); + goto cleanup; + } + if (group->unified.devices.mapfd >= 0) VIR_FORCE_CLOSE(group->unified.devices.mapfd); if (group->unified.devices.progfd >= 0) VIR_FORCE_CLOSE(group->unified.devices.progfd); - return 0; + ret = 0; + + cleanup: + VIR_FORCE_CLOSE(cgroupfd); + return ret; } -- 2.20.1

We need to mock virBPFQueryProg() in order to prevent extra syscall which tries to detect whether we can list cgroup BPF programs. Signed-off-by: Pavel Hrdina <phrdina(a)redhat.com&gt; --- src/util/virbpf.h | 5 ++++- tests/vircgroupdata/hybrid.parsed | 2 +- tests/vircgroupmock.c | 11 +++++++++++ tests/vircgrouptest.c | 4 ++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/util/virbpf.h b/src/util/virbpf.h index 7085edaacc..35e89e965c 100644 --- a/src/util/virbpf.h +++ b/src/util/virbpf.h @@ -23,6 +23,8 @@ # include <linux/bpf.h> +# include "internal.h" + /* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */ # define VIR_BPF_ALU64_REG(op, dst, src) \ @@ -211,7 +213,8 @@ virBPFQueryProg(int targetfd, unsigned int maxprogids, int attachType, unsigned int *progcnt, - void *progids); + void *progids) + ATTRIBUTE_NOINLINE; int virBPFGetProg(unsigned int id); diff --git a/tests/vircgroupdata/hybrid.parsed b/tests/vircgroupdata/hybrid.parsed index 7600de5f45..f755eed465 100644 --- a/tests/vircgroupdata/hybrid.parsed +++ b/tests/vircgroupdata/hybrid.parsed @@ -2,7 +2,7 @@ cpu <null> cpuacct <null> cpuset /not/really/sys/fs/cgroup/cpuset memory <null> -devices /not/really/sys/fs/cgroup/devices +devices <null> freezer /not/really/sys/fs/cgroup/freezer blkio <null> net_cls /not/really/sys/fs/cgroup/net_cls diff --git a/tests/vircgroupmock.c b/tests/vircgroupmock.c index 06bd0a5f29..0e5a7b0917 100644 --- a/tests/vircgroupmock.c +++ b/tests/vircgroupmock.c @@ -34,6 +34,7 @@ # include "testutilslxc.h" # include "virstring.h" # include "virfile.h" +# include "virbpf.h" static int (*real_open)(const char *path, int flags, ...); static FILE *(*real_fopen)(const char *path, const char *mode); @@ -701,6 +702,16 @@ int open(const char *path, int flags, ...) free(newpath); return ret; } + +int +virBPFQueryProg(int targetfd ATTRIBUTE_UNUSED, + unsigned int maxprogids ATTRIBUTE_UNUSED, + int attachType ATTRIBUTE_UNUSED, + unsigned int *progcnt ATTRIBUTE_UNUSED, + void *progids ATTRIBUTE_UNUSED) +{ + return 0; +} #else /* Nothing to override on non-__linux__ platforms */ #endif diff --git a/tests/vircgrouptest.c b/tests/vircgrouptest.c index 20f4c57b04..4c1f53d924 100644 --- a/tests/vircgrouptest.c +++ b/tests/vircgrouptest.c @@ -587,6 +587,7 @@ static int testCgroupNewForSelfUnified(const void *args ATTRIBUTE_UNUSED) (1 << VIR_CGROUP_CONTROLLER_CPU) | (1 << VIR_CGROUP_CONTROLLER_CPUACCT) | (1 << VIR_CGROUP_CONTROLLER_MEMORY) | + (1 << VIR_CGROUP_CONTROLLER_DEVICES) | (1 << VIR_CGROUP_CONTROLLER_BLKIO); if (virCgroupNewSelf(&cgroup) < 0) { @@ -609,14 +610,12 @@ static int testCgroupNewForSelfHybrid(const void *args ATTRIBUTE_UNUSED) const char *empty[VIR_CGROUP_CONTROLLER_LAST] = { 0 }; const char *mounts[VIR_CGROUP_CONTROLLER_LAST] = { [VIR_CGROUP_CONTROLLER_CPUSET] = "/not/really/sys/fs/cgroup/cpuset", - [VIR_CGROUP_CONTROLLER_DEVICES] = "/not/really/sys/fs/cgroup/devices", [VIR_CGROUP_CONTROLLER_FREEZER] = "/not/really/sys/fs/cgroup/freezer", [VIR_CGROUP_CONTROLLER_NET_CLS] = "/not/really/sys/fs/cgroup/net_cls", [VIR_CGROUP_CONTROLLER_PERF_EVENT] = "/not/really/sys/fs/cgroup/perf_event", }; const char *placement[VIR_CGROUP_CONTROLLER_LAST] = { [VIR_CGROUP_CONTROLLER_CPUSET] = "/", - [VIR_CGROUP_CONTROLLER_DEVICES] = "/", [VIR_CGROUP_CONTROLLER_FREEZER] = "/", [VIR_CGROUP_CONTROLLER_NET_CLS] = "/", [VIR_CGROUP_CONTROLLER_PERF_EVENT] = "/", @@ -625,6 +624,7 @@ static int testCgroupNewForSelfHybrid(const void *args ATTRIBUTE_UNUSED) (1 << VIR_CGROUP_CONTROLLER_CPU) | (1 << VIR_CGROUP_CONTROLLER_CPUACCT) | (1 << VIR_CGROUP_CONTROLLER_MEMORY) | + (1 << VIR_CGROUP_CONTROLLER_DEVICES) | (1 << VIR_CGROUP_CONTROLLER_BLKIO); if (virCgroupNewSelf(&cgroup) < 0) { -- 2.20.1