[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v1 07/11] target/arm: Implement bfloat16 dot product (indexed)
From: |
Richard Henderson |
Subject: |
[PATCH v1 07/11] target/arm: Implement bfloat16 dot product (indexed) |
Date: |
Fri, 16 Apr 2021 16:59:24 -0700 |
This is BFDOT for both AArch64 AdvSIMD and SVE,
and VDOT.BF16 for AArch32 NEON.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 2 ++
target/arm/neon-shared.decode | 2 ++
target/arm/sve.decode | 3 +++
target/arm/translate-a64.c | 41 +++++++++++++++++++++++++--------
target/arm/translate-sve.c | 12 ++++++++++
target/arm/vec_helper.c | 20 ++++++++++++++++
target/arm/translate-neon.c.inc | 9 ++++++++
7 files changed, 80 insertions(+), 9 deletions(-)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index eb4cb2b65b..af0ee8f693 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -1005,6 +1005,8 @@ DEF_HELPER_FLAGS_5(gvec_usmmla_b, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_bfdot, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfdot_idx, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
#include "helper-a64.h"
diff --git a/target/arm/neon-shared.decode b/target/arm/neon-shared.decode
index 31a0839bbb..fa3cf14e3a 100644
--- a/target/arm/neon-shared.decode
+++ b/target/arm/neon-shared.decode
@@ -81,6 +81,8 @@ VUSDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1
0 vm:4 \
vn=%vn_dp vd=%vd_dp
VSUDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1 1 vm:4 \
vn=%vn_dp vd=%vd_dp
+VDOT_b16_scal 1111 1110 0 . 00 .... .... 1101 . q:1 index:1 0 vm:4 \
+ vn=%vn_dp vd=%vd_dp
%vfml_scalar_q0_rm 0:3 5:1
%vfml_scalar_q1_index 5:1 3:1
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 523140ca56..d5e1e5d400 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1584,3 +1584,6 @@ FMLALB_zzxw 01100100 10 1 ..... 0100.0 ..... .....
@rrxr_3a esz=2
FMLALT_zzxw 01100100 10 1 ..... 0100.1 ..... ..... @rrxr_3a esz=2
FMLSLB_zzxw 01100100 10 1 ..... 0110.0 ..... ..... @rrxr_3a esz=2
FMLSLT_zzxw 01100100 10 1 ..... 0110.1 ..... ..... @rrxr_3a esz=2
+
+### SVE2 floating-point bfloat16 dot-product (indexed)
+BFDOT_zzxz 01100100 01 1 ..... 010000 ..... ..... @rrxr_2 esz=2
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index fc16e0a126..f60afbbd06 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -13457,8 +13457,22 @@ static void disas_simd_indexed(DisasContext *s,
uint32_t insn)
return;
}
break;
- case 0x0f: /* SUDOT, USDOT */
- if (is_scalar || (size & 1) || !dc_isar_feature(aa64_i8mm, s)) {
+ case 0x0f:
+ switch (size) {
+ case 0: /* SUDOT */
+ case 2: /* USDOT */
+ if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) {
+ unallocated_encoding(s);
+ return;
+ }
+ break;
+ case 1: /* BFDOT */
+ if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
+ unallocated_encoding(s);
+ return;
+ }
+ break;
+ default:
unallocated_encoding(s);
return;
}
@@ -13578,13 +13592,22 @@ static void disas_simd_indexed(DisasContext *s,
uint32_t insn)
u ? gen_helper_gvec_udot_idx_b
: gen_helper_gvec_sdot_idx_b);
return;
- case 0x0f: /* SUDOT, USDOT */
- gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
- extract32(insn, 23, 1)
- ? gen_helper_gvec_usdot_idx_b
- : gen_helper_gvec_sudot_idx_b);
- return;
-
+ case 0x0f:
+ switch (extract32(insn, 22, 2)) {
+ case 0: /* SUDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_sudot_idx_b);
+ return;
+ case 1: /* BFDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_bfdot_idx);
+ return;
+ case 2: /* USDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_usdot_idx_b);
+ return;
+ }
+ g_assert_not_reached();
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 3527430c1a..ef6828c632 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8598,3 +8598,15 @@ static bool trans_BFDOT_zzzz(DisasContext *s,
arg_rrrr_esz *a)
}
return true;
}
+
+static bool trans_BFDOT_zzxz(DisasContext *s, arg_rrxr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve_bf16, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, gen_helper_gvec_bfdot_idx,
+ a->rd, a->rn, a->rm, a->ra, a->index);
+ }
+ return true;
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index e227ba6590..3e26fb0e5f 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -2695,3 +2695,23 @@ void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+
+void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ intptr_t index = simd_data(desc);
+ intptr_t elements = opr_sz / 4;
+ intptr_t eltspersegment = MIN(16 / 4, elements);
+ float32 *d = vd, *a = va;
+ uint32_t *n = vn, *m = vm;
+
+ for (i = 0; i < elements; i += eltspersegment) {
+ uint32_t m_idx = m[i + H4(index)];
+
+ for (j = i; j < i + eltspersegment; j++) {
+ d[j] = bfdotadd(a[j], n[j], m_idx);
+ }
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
index aed8a565e0..bb0adf4756 100644
--- a/target/arm/translate-neon.c.inc
+++ b/target/arm/translate-neon.c.inc
@@ -381,6 +381,15 @@ static bool trans_VSUDOT_scalar(DisasContext *s,
arg_VSUDOT_scalar *a)
gen_helper_gvec_sudot_idx_b);
}
+static bool trans_VDOT_b16_scal(DisasContext *s, arg_VDOT_b16_scal *a)
+{
+ if (!dc_isar_feature(aa32_bf16, s)) {
+ return false;
+ }
+ return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
+ gen_helper_gvec_bfdot_idx);
+}
+
static bool trans_VFML_scalar(DisasContext *s, arg_VFML_scalar *a)
{
int opr_sz;
--
2.25.1
- [PATCH v1 for-6.1 00/11] target/arm: Implement BFloat16, Richard Henderson, 2021/04/16
- [PATCH v1 02/11] target/arm: Unify unallocated path in disas_fp_1src, Richard Henderson, 2021/04/16
- [PATCH v1 01/11] target/arm: Add isar_feature_{aa32, aa64, aa64_sve}_bf16, Richard Henderson, 2021/04/16
- [PATCH v1 03/11] target/arm: Implement scalar float32 to bfloat16 conversion, Richard Henderson, 2021/04/16
- [PATCH v1 04/11] target/arm: Implement vector float32 to bfloat16 conversion, Richard Henderson, 2021/04/16
- [PATCH v1 05/11] fpu: Add float_round_to_odd_inf, Richard Henderson, 2021/04/16
- [PATCH v1 06/11] target/arm: Implement bfloat16 dot product (vector), Richard Henderson, 2021/04/16
- [PATCH v1 08/11] target/arm: Implement bfloat16 matrix multiply accumulate, Richard Henderson, 2021/04/16
- [PATCH v1 07/11] target/arm: Implement bfloat16 dot product (indexed),
Richard Henderson <=
- [PATCH v1 09/11] target/arm: Implement bfloat widening fma (vector), Richard Henderson, 2021/04/16
- [PATCH v1 11/11] target/arm: Enable BFloat16 extensions, Richard Henderson, 2021/04/16
- [PATCH v1 10/11] target/arm: Implement bfloat widening fma (indexed), Richard Henderson, 2021/04/16