1►
他山之石
在国内学习ebpf技术,就不得不提到《Linux内核观测技术BPF》书籍译者狄卫华老师。狄老师还有一个网站《深入浅出 eBPF》。在网站里,他专门用一篇文章介绍了基于内核源码方式编译ebpf的方式,文章内容叫《【BPF入门系列-3】BPF 环境搭建》,网址:
https://www.ebpf.top/post/ebpf_c_env/
2►
获取内核源码
2.1 获取操作系统环境
cat /etc/centos-release
CentOS Linux release 8.5.2111
uname -r
4.18.0-348.7.1.el8_5.x86_64
2.2 获取开源的内核源码
$ cd /tmp/
$ wget https://mirrors.aliyun.com/linux-kernel/v4.x/linux-4.18.tar.gz
$ tar -zxvf linux-4.18.tar.gz
$ cd linux-4.18
$ cat Makefile | grep -P '^VERSION|^PATCHLEVEL|^SUBLEVEL'
VERSION = 4
PATCHLEVEL = 18
SUBLEVEL = 0
3►
初始化基础环境
$ sudo yum install bison flex openssl-devel
$ sudo yum install clang llvm elfutils-libelf-devel
具体每个实验机器的环境可能略有差别,需要根据自己的情况做细节调整。
4►
编译内核源码中ebpf程序样例
4.1 编译环境初始化
$ cd /tmp/linux-4.18
$ make oldconfig && make init # make oldconfig && make prepare && make scripts
$ make headers_install
4.2 编译内核源码样例
make M=samples/bpf
4.3 执行样例程序
sudo ./samples/bpf/trace_output
recv 1766352 events per sec
5►
内核源码的ebpf编译关键过程提取
5.1 头文件安装 make headers_install
$ cd /tmp/
$ rm -fr /tmp/linux-4.18
$ tar -zxvf linux-4.18.tar.gz
$ cd /tmp/linux-4.18
$ make oldconfig && make init
$ ls usr/include/
ls: cannot access usr/include/: No such file or directory # 此时include目录不存在
$ make headers_install
$ ls usr/include/ -R | grep -v -P ':$' | grep -v -P '^$' | wc -l
931 # 此时include目录下有931个文件
$ diff -rs usr/include/ /usr/include/|grep -P '^Files .+ and .+ are identical$'|wc -l
677
$ rpm -ql kernel-headers | wc -l
964
$ rpm -ql kernel-headers | head
/usr/include/asm
/usr/include/asm-generic
/usr/include/asm-generic/bpf_perf_event.h
5.2 eBPF样例编译 make M=samples/bpf
$ make M=samples/bpf --debug=v,m SHELL="bash -x" > make.log 2>&1
$ gcc -g -fPIC -c -o libbpf.o libbpf.c
$ gcc -g -fPIC -c -o bpf.o bpf.c
$ gcc -g -fPIC -c -o btf.o btf.c
$ gcc -g -fPIC -c -o nlattr.o nlattr.c
$ ld -r -o libbpf-in.o libbpf.o bpf.o nlattr.o btf.o
$ ar rcs libbpf.a libbpf-in.o
$ gcc -O2 -std=gnu89 -c -o bpf_load.o bpf_load.c
$ gcc -O2 -std=gnu89 -c -o trace_output_user.o trace_output_user.c
$ gcc -O2 -std=gnu89 -c -o trace_helpers.o trace_helpers.c
$ gcc -o trace_output bpf_load.o trace_output_user.o trace_helpers.o libbpf.a -lelf -lrt
$ clang -O2 -emit-llvm -c trace_output_kern.c -o -
$ llc -march=bpf -filetype=obj -o trace_output_kern.o
6►
手工编译内核源码中的eBPF样例分析
cd /tmp/
rm -fr /tmp/linux-4.18
tar -zxvf linux-4.18.tar.gz
cd /tmp/linux-4.18
make oldconfig && make init
make headers_install
cd tools/lib/bpf/
6.1 手工步骤A过程解析
$ # gcc -g -fPIC -c -o libbpf.o libbpf.c
$ gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf.o libbpf.c
$ # gcc -g -fPIC -c -o bpf.o bpf.c
$ gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o bpf.o bpf.c
$ # gcc -g -fPIC -c -o btf.o btf.c
$ gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o btf.o btf.c
$ # gcc -g -fPIC -c -o nlattr.o nlattr.c
$ gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o nlattr.o nlattr.c
● -fPIC,告诉编译器输出位置无关目标,为后面生成共享库埋下伏笔。
● -I. 表示需要包含当前目录下的头文件。
6.2 手工步骤B过程解析
$ ld -r -o libbpf-in.o libbpf.o bpf.o nlattr.o btf.o
6.3 手工步骤C过程解析
$ ar rcs libbpf.a libbpf-in.o
6.4 手工步骤D/E/F过程解析
$ # gcc -O2 -std=gnu89 -c -o bpf_load.o bpf_load.c
$ gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./usr/include -Wno-unused-variable -c -o samples/bpf/bpf_load.o samples/bpf/bpf_load.c
$ # gcc -O2 -std=gnu89 -c -o trace_output_user.o trace_output_user.c
$ gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./tools/lib/bpf/ -c -o samples/bpf/trace_output_user.o samples/bpf/trace_output_user.c
$ # gcc -O2 -std=gnu89 -c -o trace_helpers.o trace_helpers.c
$ gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./tools/lib/bpf/ -c -o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.c
● -O2 和 -std=gnu89 是两个核心选项。
6.5 手工步骤G过程解析
$ # gcc -o trace_output bpf_load.o trace_output_user.o trace_helpers.o libbpf.a -lelf -lrt
$ gcc -o samples/bpf/trace_output samples/bpf/bpf_load.o samples/bpf/trace_output_user.o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.o /tmp/linux-4.18/samples/bpf/../../tools/lib/bpf/libbpf.a -lelf -lrt
● -lelf -lrt链接两个类库
6.6 手工步骤H过程解析
$ clang -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/8/include -I./arch/x86/include -I./arch/x86/include/generated -I./include -I./arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -Isamples/bpf -I./tools/testing/selftests/bpf/ -D__KERNEL__ -D__BPF_TRACING__ -D__TARGET_ARCH_x86 -O2 -emit-llvm -c samples/bpf/trace_output_kern.c -o - | llc -march=bpf -filetype=obj -o samples/bpf/trace_output_kern.o
● -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/8/include,这2个选项是一组。nostdinc表示屏蔽掉系统默认的include环境,替换成当前gcc编译器自带的include头文件环境。
sudo ./samples/bpf/trace_output
recv 1760674 events per sec
7►
关于4.9版本内核
$ cd /tmp/linux-4.9/
$ find . -name libbpf.c
./samples/bpf/libbpf.c
./tools/lib/bpf/libbpf.c
8►
进一步探索
附录: eBPF手工纯C编译完整命令清单
cd /tmp/
rm -fr /tmp/linux-4.18
tar -zxvf linux-4.18.tar.gz
cd /tmp/linux-4.18
make oldconfig && make init
make headers_install
cd tools/lib/bpf/
# 步骤A1
# gcc -g -fPIC -c -o libbpf.o libbpf.c
gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf.o libbpf.c
# 步骤A2
# gcc -g -fPIC -c -o bpf.o bpf.c
gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o bpf.o bpf.c
# 步骤A3
# gcc -g -fPIC -c -o btf.o btf.c
gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o btf.o btf.c
# 步骤A4
# gcc -g -fPIC -c -o nlattr.o nlattr.c
gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-4.18/tools/include -I/tmp/linux-4.18/tools/arch/x86/include/uapi -I/tmp/linux-4.18/tools/include/uapi -I/tmp/linux-4.18/tools/perf -D"BUILD_STR(s)=#s" -c -o nlattr.o nlattr.c
# 步骤B
ld -r -o libbpf-in.o libbpf.o bpf.o nlattr.o btf.o
# 步骤C
ar rcs libbpf.a libbpf-in.o
cd /tmp/linux-4.18/
# 步骤D
# gcc -O2 -std=gnu89 -c -o bpf_load.o bpf_load.c
gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./usr/include -Wno-unused-variable -c -o samples/bpf/bpf_load.o samples/bpf/bpf_load.c
# 步骤E
# gcc -O2 -std=gnu89 -c -o trace_output_user.o trace_output_user.c
gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./tools/lib/bpf/ -c -o samples/bpf/trace_output_user.o samples/bpf/trace_output_user.c
# 步骤F
# gcc -O2 -std=gnu89 -c -o trace_helpers.o trace_helpers.c
gcc -O2 -fomit-frame-pointer -std=gnu89 -I./usr/include -I./tools/lib/ -I./tools/testing/selftests/bpf/ -I./tools/lib/ -I./tools/include -I./tools/perf -I./tools/lib/bpf/ -c -o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.c
# 步骤G
# gcc -o trace_output bpf_load.o trace_output_user.o trace_helpers.o libbpf.a -lelf -lrt
gcc -o samples/bpf/trace_output samples/bpf/bpf_load.o samples/bpf/trace_output_user.o samples/bpf/../../tools/testing/selftests/bpf/trace_helpers.o /tmp/linux-4.18/samples/bpf/../../tools/lib/bpf/libbpf.a -lelf -lrt
# 步骤H
clang -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/8/include -I./arch/x86/include -I./arch/x86/include/generated -I./include -I./arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -Isamples/bpf -I./tools/testing/selftests/bpf/ -D__KERNEL__ -D__BPF_TRACING__ -D__TARGET_ARCH_x86 -O2 -emit-llvm -c samples/bpf/trace_output_kern.c -o - | llc -march=bpf -filetype=obj -o samples/bpf/trace_output_kern.o
阅码场服务微信号:小月@阅码场(linuxer2016)
如有疑问和相关需求,点击「阅读原文」,联系我们。