mirror of
git://git.openwrt.org/openwrt/openwrt.git
synced 2025-11-03 22:44:27 -05:00
f2e1e156c0
This adds support for eBPF JIT for 32 bit targets and significantly improves correctness. Signed-off-by: Felix Fietkau <nbd@nbd.name>
388 lines
10 KiB
Diff
388 lines
10 KiB
Diff
From: Johan Almbladh <johan.almbladh@anyfinetworks.com>
|
|
Date: Tue, 5 Oct 2021 18:54:08 +0200
|
|
Subject: [PATCH] mips: bpf: Remove old BPF JIT implementations
|
|
|
|
This patch removes the old 32-bit cBPF and 64-bit eBPF JIT implementations.
|
|
They are replaced by a new eBPF implementation that supports both 32-bit
|
|
and 64-bit MIPS CPUs.
|
|
|
|
Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
|
|
---
|
|
delete mode 100644 arch/mips/net/bpf_jit.c
|
|
delete mode 100644 arch/mips/net/bpf_jit.h
|
|
delete mode 100644 arch/mips/net/bpf_jit_asm.S
|
|
delete mode 100644 arch/mips/net/ebpf_jit.c
|
|
|
|
--- a/arch/mips/net/bpf_jit.h
|
|
+++ /dev/null
|
|
@@ -1,81 +0,0 @@
|
|
-/* SPDX-License-Identifier: GPL-2.0-only */
|
|
-/*
|
|
- * Just-In-Time compiler for BPF filters on MIPS
|
|
- *
|
|
- * Copyright (c) 2014 Imagination Technologies Ltd.
|
|
- * Author: Markos Chandras <markos.chandras@imgtec.com>
|
|
- */
|
|
-
|
|
-#ifndef BPF_JIT_MIPS_OP_H
|
|
-#define BPF_JIT_MIPS_OP_H
|
|
-
|
|
-/* Registers used by JIT */
|
|
-#define MIPS_R_ZERO 0
|
|
-#define MIPS_R_V0 2
|
|
-#define MIPS_R_A0 4
|
|
-#define MIPS_R_A1 5
|
|
-#define MIPS_R_T4 12
|
|
-#define MIPS_R_T5 13
|
|
-#define MIPS_R_T6 14
|
|
-#define MIPS_R_T7 15
|
|
-#define MIPS_R_S0 16
|
|
-#define MIPS_R_S1 17
|
|
-#define MIPS_R_S2 18
|
|
-#define MIPS_R_S3 19
|
|
-#define MIPS_R_S4 20
|
|
-#define MIPS_R_S5 21
|
|
-#define MIPS_R_S6 22
|
|
-#define MIPS_R_S7 23
|
|
-#define MIPS_R_SP 29
|
|
-#define MIPS_R_RA 31
|
|
-
|
|
-/* Conditional codes */
|
|
-#define MIPS_COND_EQ 0x1
|
|
-#define MIPS_COND_GE (0x1 << 1)
|
|
-#define MIPS_COND_GT (0x1 << 2)
|
|
-#define MIPS_COND_NE (0x1 << 3)
|
|
-#define MIPS_COND_ALL (0x1 << 4)
|
|
-/* Conditionals on X register or K immediate */
|
|
-#define MIPS_COND_X (0x1 << 5)
|
|
-#define MIPS_COND_K (0x1 << 6)
|
|
-
|
|
-#define r_ret MIPS_R_V0
|
|
-
|
|
-/*
|
|
- * Use 2 scratch registers to avoid pipeline interlocks.
|
|
- * There is no overhead during epilogue and prologue since
|
|
- * any of the $s0-$s6 registers will only be preserved if
|
|
- * they are going to actually be used.
|
|
- */
|
|
-#define r_skb_hl MIPS_R_S0 /* skb header length */
|
|
-#define r_skb_data MIPS_R_S1 /* skb actual data */
|
|
-#define r_off MIPS_R_S2
|
|
-#define r_A MIPS_R_S3
|
|
-#define r_X MIPS_R_S4
|
|
-#define r_skb MIPS_R_S5
|
|
-#define r_M MIPS_R_S6
|
|
-#define r_skb_len MIPS_R_S7
|
|
-#define r_s0 MIPS_R_T4 /* scratch reg 1 */
|
|
-#define r_s1 MIPS_R_T5 /* scratch reg 2 */
|
|
-#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */
|
|
-#define r_tmp MIPS_R_T7 /* No need to preserve this */
|
|
-#define r_zero MIPS_R_ZERO
|
|
-#define r_sp MIPS_R_SP
|
|
-#define r_ra MIPS_R_RA
|
|
-
|
|
-#ifndef __ASSEMBLY__
|
|
-
|
|
-/* Declare ASM helpers */
|
|
-
|
|
-#define DECLARE_LOAD_FUNC(func) \
|
|
- extern u8 func(unsigned long *skb, int offset); \
|
|
- extern u8 func##_negative(unsigned long *skb, int offset); \
|
|
- extern u8 func##_positive(unsigned long *skb, int offset)
|
|
-
|
|
-DECLARE_LOAD_FUNC(sk_load_word);
|
|
-DECLARE_LOAD_FUNC(sk_load_half);
|
|
-DECLARE_LOAD_FUNC(sk_load_byte);
|
|
-
|
|
-#endif
|
|
-
|
|
-#endif /* BPF_JIT_MIPS_OP_H */
|
|
--- a/arch/mips/net/bpf_jit_asm.S
|
|
+++ /dev/null
|
|
@@ -1,285 +0,0 @@
|
|
-/*
|
|
- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF
|
|
- * compiler.
|
|
- *
|
|
- * Copyright (C) 2015 Imagination Technologies Ltd.
|
|
- * Author: Markos Chandras <markos.chandras@imgtec.com>
|
|
- *
|
|
- * This program is free software; you can redistribute it and/or modify it
|
|
- * under the terms of the GNU General Public License as published by the
|
|
- * Free Software Foundation; version 2 of the License.
|
|
- */
|
|
-
|
|
-#include <asm/asm.h>
|
|
-#include <asm/isa-rev.h>
|
|
-#include <asm/regdef.h>
|
|
-#include "bpf_jit.h"
|
|
-
|
|
-/* ABI
|
|
- *
|
|
- * r_skb_hl skb header length
|
|
- * r_skb_data skb data
|
|
- * r_off(a1) offset register
|
|
- * r_A BPF register A
|
|
- * r_X PF register X
|
|
- * r_skb(a0) *skb
|
|
- * r_M *scratch memory
|
|
- * r_skb_le skb length
|
|
- * r_s0 Scratch register 0
|
|
- * r_s1 Scratch register 1
|
|
- *
|
|
- * On entry:
|
|
- * a0: *skb
|
|
- * a1: offset (imm or imm + X)
|
|
- *
|
|
- * All non-BPF-ABI registers are free for use. On return, we only
|
|
- * care about r_ret. The BPF-ABI registers are assumed to remain
|
|
- * unmodified during the entire filter operation.
|
|
- */
|
|
-
|
|
-#define skb a0
|
|
-#define offset a1
|
|
-#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */
|
|
-
|
|
- /* We know better :) so prevent assembler reordering etc */
|
|
- .set noreorder
|
|
-
|
|
-#define is_offset_negative(TYPE) \
|
|
- /* If offset is negative we have more work to do */ \
|
|
- slti t0, offset, 0; \
|
|
- bgtz t0, bpf_slow_path_##TYPE##_neg; \
|
|
- /* Be careful what follows in DS. */
|
|
-
|
|
-#define is_offset_in_header(SIZE, TYPE) \
|
|
- /* Reading from header? */ \
|
|
- addiu $r_s0, $r_skb_hl, -SIZE; \
|
|
- slt t0, $r_s0, offset; \
|
|
- bgtz t0, bpf_slow_path_##TYPE; \
|
|
-
|
|
-LEAF(sk_load_word)
|
|
- is_offset_negative(word)
|
|
-FEXPORT(sk_load_word_positive)
|
|
- is_offset_in_header(4, word)
|
|
- /* Offset within header boundaries */
|
|
- PTR_ADDU t1, $r_skb_data, offset
|
|
- .set reorder
|
|
- lw $r_A, 0(t1)
|
|
- .set noreorder
|
|
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
|
|
-# if MIPS_ISA_REV >= 2
|
|
- wsbh t0, $r_A
|
|
- rotr $r_A, t0, 16
|
|
-# else
|
|
- sll t0, $r_A, 24
|
|
- srl t1, $r_A, 24
|
|
- srl t2, $r_A, 8
|
|
- or t0, t0, t1
|
|
- andi t2, t2, 0xff00
|
|
- andi t1, $r_A, 0xff00
|
|
- or t0, t0, t2
|
|
- sll t1, t1, 8
|
|
- or $r_A, t0, t1
|
|
-# endif
|
|
-#endif
|
|
- jr $r_ra
|
|
- move $r_ret, zero
|
|
- END(sk_load_word)
|
|
-
|
|
-LEAF(sk_load_half)
|
|
- is_offset_negative(half)
|
|
-FEXPORT(sk_load_half_positive)
|
|
- is_offset_in_header(2, half)
|
|
- /* Offset within header boundaries */
|
|
- PTR_ADDU t1, $r_skb_data, offset
|
|
- lhu $r_A, 0(t1)
|
|
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
|
|
-# if MIPS_ISA_REV >= 2
|
|
- wsbh $r_A, $r_A
|
|
-# else
|
|
- sll t0, $r_A, 8
|
|
- srl t1, $r_A, 8
|
|
- andi t0, t0, 0xff00
|
|
- or $r_A, t0, t1
|
|
-# endif
|
|
-#endif
|
|
- jr $r_ra
|
|
- move $r_ret, zero
|
|
- END(sk_load_half)
|
|
-
|
|
-LEAF(sk_load_byte)
|
|
- is_offset_negative(byte)
|
|
-FEXPORT(sk_load_byte_positive)
|
|
- is_offset_in_header(1, byte)
|
|
- /* Offset within header boundaries */
|
|
- PTR_ADDU t1, $r_skb_data, offset
|
|
- lbu $r_A, 0(t1)
|
|
- jr $r_ra
|
|
- move $r_ret, zero
|
|
- END(sk_load_byte)
|
|
-
|
|
-/*
|
|
- * call skb_copy_bits:
|
|
- * (prototype in linux/skbuff.h)
|
|
- *
|
|
- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len)
|
|
- *
|
|
- * o32 mandates we leave 4 spaces for argument registers in case
|
|
- * the callee needs to use them. Even though we don't care about
|
|
- * the argument registers ourselves, we need to allocate that space
|
|
- * to remain ABI compliant since the callee may want to use that space.
|
|
- * We also allocate 2 more spaces for $r_ra and our return register (*to).
|
|
- *
|
|
- * n64 is a bit different. The *caller* will allocate the space to preserve
|
|
- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no
|
|
- * good reason but it does not matter that much really.
|
|
- *
|
|
- * (void *to) is returned in r_s0
|
|
- *
|
|
- */
|
|
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
|
|
-#define DS_OFFSET(SIZE) (4 * SZREG)
|
|
-#else
|
|
-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE))
|
|
-#endif
|
|
-#define bpf_slow_path_common(SIZE) \
|
|
- /* Quick check. Are we within reasonable boundaries? */ \
|
|
- LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \
|
|
- sltu $r_s0, offset, $r_s1; \
|
|
- beqz $r_s0, fault; \
|
|
- /* Load 4th argument in DS */ \
|
|
- LONG_ADDIU a3, zero, SIZE; \
|
|
- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
|
|
- PTR_LA t0, skb_copy_bits; \
|
|
- PTR_S $r_ra, (5 * SZREG)($r_sp); \
|
|
- /* Assign low slot to a2 */ \
|
|
- PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \
|
|
- jalr t0; \
|
|
- /* Reset our destination slot (DS but it's ok) */ \
|
|
- INT_S zero, (4 * SZREG)($r_sp); \
|
|
- /* \
|
|
- * skb_copy_bits returns 0 on success and -EFAULT \
|
|
- * on error. Our data live in a2. Do not bother with \
|
|
- * our data if an error has been returned. \
|
|
- */ \
|
|
- /* Restore our frame */ \
|
|
- PTR_L $r_ra, (5 * SZREG)($r_sp); \
|
|
- INT_L $r_s0, (4 * SZREG)($r_sp); \
|
|
- bltz v0, fault; \
|
|
- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
|
|
- move $r_ret, zero; \
|
|
-
|
|
-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
|
|
- bpf_slow_path_common(4)
|
|
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
|
|
-# if MIPS_ISA_REV >= 2
|
|
- wsbh t0, $r_s0
|
|
- jr $r_ra
|
|
- rotr $r_A, t0, 16
|
|
-# else
|
|
- sll t0, $r_s0, 24
|
|
- srl t1, $r_s0, 24
|
|
- srl t2, $r_s0, 8
|
|
- or t0, t0, t1
|
|
- andi t2, t2, 0xff00
|
|
- andi t1, $r_s0, 0xff00
|
|
- or t0, t0, t2
|
|
- sll t1, t1, 8
|
|
- jr $r_ra
|
|
- or $r_A, t0, t1
|
|
-# endif
|
|
-#else
|
|
- jr $r_ra
|
|
- move $r_A, $r_s0
|
|
-#endif
|
|
-
|
|
- END(bpf_slow_path_word)
|
|
-
|
|
-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
|
|
- bpf_slow_path_common(2)
|
|
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
|
|
-# if MIPS_ISA_REV >= 2
|
|
- jr $r_ra
|
|
- wsbh $r_A, $r_s0
|
|
-# else
|
|
- sll t0, $r_s0, 8
|
|
- andi t1, $r_s0, 0xff00
|
|
- andi t0, t0, 0xff00
|
|
- srl t1, t1, 8
|
|
- jr $r_ra
|
|
- or $r_A, t0, t1
|
|
-# endif
|
|
-#else
|
|
- jr $r_ra
|
|
- move $r_A, $r_s0
|
|
-#endif
|
|
-
|
|
- END(bpf_slow_path_half)
|
|
-
|
|
-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp)
|
|
- bpf_slow_path_common(1)
|
|
- jr $r_ra
|
|
- move $r_A, $r_s0
|
|
-
|
|
- END(bpf_slow_path_byte)
|
|
-
|
|
-/*
|
|
- * Negative entry points
|
|
- */
|
|
- .macro bpf_is_end_of_data
|
|
- li t0, SKF_LL_OFF
|
|
- /* Reading link layer data? */
|
|
- slt t1, offset, t0
|
|
- bgtz t1, fault
|
|
- /* Be careful what follows in DS. */
|
|
- .endm
|
|
-/*
|
|
- * call skb_copy_bits:
|
|
- * (prototype in linux/filter.h)
|
|
- *
|
|
- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
|
|
- * int k, unsigned int size)
|
|
- *
|
|
- * see above (bpf_slow_path_common) for ABI restrictions
|
|
- */
|
|
-#define bpf_negative_common(SIZE) \
|
|
- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
|
|
- PTR_LA t0, bpf_internal_load_pointer_neg_helper; \
|
|
- PTR_S $r_ra, (5 * SZREG)($r_sp); \
|
|
- jalr t0; \
|
|
- li a2, SIZE; \
|
|
- PTR_L $r_ra, (5 * SZREG)($r_sp); \
|
|
- /* Check return pointer */ \
|
|
- beqz v0, fault; \
|
|
- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
|
|
- /* Preserve our pointer */ \
|
|
- move $r_s0, v0; \
|
|
- /* Set return value */ \
|
|
- move $r_ret, zero; \
|
|
-
|
|
-bpf_slow_path_word_neg:
|
|
- bpf_is_end_of_data
|
|
-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp)
|
|
- bpf_negative_common(4)
|
|
- jr $r_ra
|
|
- lw $r_A, 0($r_s0)
|
|
- END(sk_load_word_negative)
|
|
-
|
|
-bpf_slow_path_half_neg:
|
|
- bpf_is_end_of_data
|
|
-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp)
|
|
- bpf_negative_common(2)
|
|
- jr $r_ra
|
|
- lhu $r_A, 0($r_s0)
|
|
- END(sk_load_half_negative)
|
|
-
|
|
-bpf_slow_path_byte_neg:
|
|
- bpf_is_end_of_data
|
|
-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp)
|
|
- bpf_negative_common(1)
|
|
- jr $r_ra
|
|
- lbu $r_A, 0($r_s0)
|
|
- END(sk_load_byte_negative)
|
|
-
|
|
-fault:
|
|
- jr $r_ra
|
|
- addiu $r_ret, zero, 1
|