Commit 55db92f4 authored by Yonghong Song's avatar Yonghong Song Committed by Alexei Starovoitov
Browse files

bpf: Add BPF_KPTR_PERCPU as a field type 



BPF_KPTR_PERCPU represents a percpu field type like below

  struct val_t {
    ... fields ...
  };
  struct t {
    ...
    struct val_t __percpu_kptr *percpu_data_ptr;
    ...
  };

where
  #define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))

While BPF_KPTR_REF points to a trusted kernel object or a trusted
local object, BPF_KPTR_PERCPU points to a trusted local
percpu object.

This patch added basic support for BPF_KPTR_PERCPU
related to percpu_kptr field parsing, recording and free operations.
BPF_KPTR_PERCPU also supports the same map types
as BPF_KPTR_REF does.

Note that unlike a local kptr, it is possible that
a BPF_KTPR_PERCPU struct may not contain any
special fields like other kptr, bpf_spin_lock, bpf_list_head, etc.

Signed-off-by: default avatarYonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20230827152739.1996391-1-yonghong.song@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 41a5db8d

include/linux/bpf.h

+12 −6
Original line number Diff line number Diff line
@@ -180,14 +180,15 @@ enum btf_field_type { 
	BPF_TIMER      = (1 << 1),

	BPF_KPTR_UNREF = (1 << 2),

	BPF_KPTR_REF   = (1 << 3),

	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF,

	BPF_LIST_HEAD  = (1 << 4),

	BPF_LIST_NODE  = (1 << 5),

	BPF_RB_ROOT    = (1 << 6),

	BPF_RB_NODE    = (1 << 7),

	BPF_KPTR_PERCPU = (1 << 4),

	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,

	BPF_LIST_HEAD  = (1 << 5),

	BPF_LIST_NODE  = (1 << 6),

	BPF_RB_ROOT    = (1 << 7),

	BPF_RB_NODE    = (1 << 8),

	BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD |

				 BPF_RB_NODE | BPF_RB_ROOT,

	BPF_REFCOUNT   = (1 << 8),

	BPF_REFCOUNT   = (1 << 9),

};



typedef void (*btf_dtor_kfunc_t)(void *);
@@ -300,6 +301,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) 
	case BPF_KPTR_UNREF:

	case BPF_KPTR_REF:

		return "kptr";

	case BPF_KPTR_PERCPU:

		return "percpu_kptr";

	case BPF_LIST_HEAD:

		return "bpf_list_head";

	case BPF_LIST_NODE:
@@ -325,6 +328,7 @@ static inline u32 btf_field_type_size(enum btf_field_type type) 
		return sizeof(struct bpf_timer);

	case BPF_KPTR_UNREF:

	case BPF_KPTR_REF:

	case BPF_KPTR_PERCPU:

		return sizeof(u64);

	case BPF_LIST_HEAD:

		return sizeof(struct bpf_list_head);
@@ -351,6 +355,7 @@ static inline u32 btf_field_type_align(enum btf_field_type type) 
		return __alignof__(struct bpf_timer);

	case BPF_KPTR_UNREF:

	case BPF_KPTR_REF:

	case BPF_KPTR_PERCPU:

		return __alignof__(u64);

	case BPF_LIST_HEAD:

		return __alignof__(struct bpf_list_head);
@@ -389,6 +394,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) 
	case BPF_TIMER:

	case BPF_KPTR_UNREF:

	case BPF_KPTR_REF:

	case BPF_KPTR_PERCPU:

		break;

	default:

		WARN_ON_ONCE(1);

kernel/bpf/btf.c

+5 −0
Original line number Diff line number Diff line
@@ -3293,6 +3293,8 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, 
		type = BPF_KPTR_UNREF;

	else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off)))

		type = BPF_KPTR_REF;

	else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))

		type = BPF_KPTR_PERCPU;

	else

		return -EINVAL;
@@ -3457,6 +3459,7 @@ static int btf_find_struct_field(const struct btf *btf, 
			break;

		case BPF_KPTR_UNREF:

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			ret = btf_find_kptr(btf, member_type, off, sz,

					    idx < info_cnt ? &info[idx] : &tmp);

			if (ret < 0)
@@ -3523,6 +3526,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, 
			break;

		case BPF_KPTR_UNREF:

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			ret = btf_find_kptr(btf, var_type, off, sz,

					    idx < info_cnt ? &info[idx] : &tmp);

			if (ret < 0)
@@ -3783,6 +3787,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type 
			break;

		case BPF_KPTR_UNREF:

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);

			if (ret < 0)

				goto end;

kernel/bpf/syscall.c

+4 −0
Original line number Diff line number Diff line
@@ -514,6 +514,7 @@ void btf_record_free(struct btf_record *rec) 
		switch (rec->fields[i].type) {

		case BPF_KPTR_UNREF:

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			if (rec->fields[i].kptr.module)

				module_put(rec->fields[i].kptr.module);

			btf_put(rec->fields[i].kptr.btf);
@@ -560,6 +561,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) 
		switch (fields[i].type) {

		case BPF_KPTR_UNREF:

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			btf_get(fields[i].kptr.btf);

			if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {

				ret = -ENXIO;
@@ -650,6 +652,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) 
			WRITE_ONCE(*(u64 *)field_ptr, 0);

			break;

		case BPF_KPTR_REF:

		case BPF_KPTR_PERCPU:

			xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);

			if (!xchgd_field)

				break;
@@ -1045,6 +1048,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, 
				break;

			case BPF_KPTR_UNREF:

			case BPF_KPTR_REF:

			case BPF_KPTR_PERCPU:

			case BPF_REFCOUNT:

				if (map->map_type != BPF_MAP_TYPE_HASH &&

				    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&