1 | //===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===// |
---|---|

2 | // |

3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |

4 | // See https://llvm.org/LICENSE.txt for license information. |

5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |

6 | // |

7 | //===----------------------------------------------------------------------===// |

8 | // |

9 | // This file defines the set of ACLE-specified source-level intrinsic |

10 | // functions wrapping the MVE vector instruction set and scalar shift |

11 | // operations. |

12 | // |

13 | // Refer to comments in arm_mve_defs.td for the infrastructure used in |

14 | // here, and to MveEmitter.cpp for how those are used in turn to |

15 | // generate code. |

16 | // |

17 | //===----------------------------------------------------------------------===// |

18 | |

19 | include "arm_mve_defs.td" |

20 | |

21 | let params = T.Usual in |

22 | foreach n = [ 2, 4 ] in { |

23 | def "vst"#n# "q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>), |

24 | (CustomCodegen<"VST24"> n:$NumVectors, |

25 | "Intrinsic::arm_mve_vst"#n# "q":$IRIntr)>; |

26 | def "vld"#n# "q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>), |

27 | (CustomCodegen<"VLD24"> n:$NumVectors, |

28 | "Intrinsic::arm_mve_vld"#n# "q":$IRIntr)>; |

29 | } |

30 | |

31 | multiclass bit_op_fp<IRBuilder bitop> { |

32 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), |

33 | (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>; |

34 | } |

35 | |

36 | multiclass bit_op_fp_with_inv<IRBuilder bitop> { |

37 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), |

38 | (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>; |

39 | } |

40 | |

41 | let params = T.Signed in { |

42 | def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

43 | (IRIntBase<"sadd_sat", [Vector]> $a, $b)>; |

44 | def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

45 | (IRIntBase<"ssub_sat", [Vector]> $a, $b)>; |

46 | let pnt = PNT_NType in { |

47 | def vqaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

48 | (IRIntBase<"sadd_sat", [Vector]> $a, (splat $b))>; |

49 | def vqsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

50 | (IRIntBase<"ssub_sat", [Vector]> $a, (splat $b))>; |

51 | } |

52 | } |

53 | let params = T.Unsigned in { |

54 | def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

55 | (IRIntBase<"uadd_sat", [Vector]> $a, $b)>, |

56 | NameOverride<"vqaddq">; |

57 | def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

58 | (IRIntBase<"usub_sat", [Vector]> $a, $b)>, |

59 | NameOverride<"vqsubq">; |

60 | let pnt = PNT_NType in { |

61 | def vqaddq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

62 | (IRIntBase<"uadd_sat", [Vector]> $a, (splat $b))>, |

63 | NameOverride<"vqaddq_n">; |

64 | def vqsubq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

65 | (IRIntBase<"usub_sat", [Vector]> $a, (splat $b))>, |

66 | NameOverride<"vqsubq_n">; |

67 | } |

68 | } |

69 | |

70 | // Some intrinsics below are implemented not as IR fragments, but as |

71 | // special-purpose IR intrinsics. This is because such a general form |

72 | // (such as NEON uses) required a variable-width vector size, and we are |

73 | // restricted to 128 bit. Although we can possibly get clever with lane |

74 | // operations, the consequent IR representation would be very hard to |

75 | // write sensibly. In particular, doubling a vector's width would be a |

76 | // mess. Other intrinsics just don't translate nicely into IR. |

77 | let params = T.Int in { |

78 | def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>; |

79 | def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

80 | (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; |

81 | def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

82 | (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; |

83 | def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>; |

84 | def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>; |

85 | def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>; |

86 | def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>; |

87 | def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>; |

88 | def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>; |

89 | def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

90 | (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>; |

91 | def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>; |

92 | def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

93 | (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; |

94 | def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

95 | (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; |

96 | def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

97 | (IRInt<"vmull", [DblVector, Vector]> |

98 | $a, $b, (unsignedflag Scalar), 0)>; |

99 | def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

100 | (IRInt<"vmull", [DblVector, Vector]> |

101 | $a, $b, (unsignedflag Scalar), 1)>; |

102 | let pnt = PNT_NType in { |

103 | def vaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

104 | (add $a, (splat $b))>; |

105 | def vsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

106 | (sub $a, (splat $b))>; |

107 | def vmulq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

108 | (mul $a, (splat $b))>; |

109 | def vhaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

110 | (IRInt<"vhadd", [Vector]> $a, (splat $b), |

111 | (unsignedflag Scalar))>; |

112 | def vhsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

113 | (IRInt<"vhsub", [Vector]> $a, (splat $b), |

114 | (unsignedflag Scalar))>; |

115 | } |

116 | } |

117 | let params = T.Signed in { |

118 | def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

119 | (IRInt<"vqdmulh", [Vector]> $a, $b)>; |

120 | def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

121 | (IRInt<"vqrdmulh", [Vector]> $a, $b)>; |

122 | let pnt = PNT_NType in { |

123 | def vqdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

124 | (IRInt<"vqdmulh", [Vector]> $a, (splat $b))>; |

125 | def vqrdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

126 | (IRInt<"vqrdmulh", [Vector]> $a, (splat $b))>; |

127 | } |

128 | } |

129 | |

130 | let params = T.Poly, overrideKindLetter = "p"in { |

131 | def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

132 | (IRInt<"vmull_poly", [DblVector, Vector]> |

133 | $a, $b, 0)>; |

134 | def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

135 | (IRInt<"vmull_poly", [DblVector, Vector]> |

136 | $a, $b, 1)>; |

137 | } |

138 | |

139 | let params = T.Float in { |

140 | def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>, |

141 | NameOverride<"vaddq">; |

142 | defm vandqf: bit_op_fp<and>, NameOverride<"vandq">; |

143 | defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">; |

144 | defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">; |

145 | defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">; |

146 | defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">; |

147 | def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>, |

148 | NameOverride<"vsubq">; |

149 | def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>, |

150 | NameOverride<"vmulq">; |

151 | |

152 | let pnt = PNT_NType in { |

153 | def vaddqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

154 | (fadd $a, (splat $b))>, |

155 | NameOverride<"vaddq_n">; |

156 | def vsubqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

157 | (fsub $a, (splat $b))>, |

158 | NameOverride<"vsubq_n">; |

159 | def vmulqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), |

160 | (fmul $a, (splat $b))>, |

161 | NameOverride<"vmulq_n">; |

162 | } |

163 | } |

164 | |

165 | multiclass FMA<bit add> { |

166 | // FMS instructions are defined in the ArmARM as if they negate the |

167 | // second multiply input. |

168 | defvar m2_cg = !if(add, (id $m2), (fneg $m2)); |

169 | |

170 | defvar unpred_cg = (IRIntBase<"fma", [Vector]> $m1, m2_cg, $addend); |

171 | defvar pred_cg = (IRInt<"fma_predicated", [Vector, Predicate]> |

172 | $m1, m2_cg, $addend, $pred); |

173 | |

174 | def q: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2), |

175 | unpred_cg>; |

176 | |

177 | def q_m: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2, |

178 | Predicate:$pred), pred_cg>; |

179 | |

180 | // Only FMA has the vector/scalar variants, not FMS |

181 | if add then let pnt = PNT_NType in { |

182 | |

183 | def q_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, |

184 | unpromoted<Scalar>:$m2_s), |

185 | (seq (splat $m2_s):$m2, unpred_cg)>; |

186 | def sq_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2, |

187 | unpromoted<Scalar>:$addend_s), |

188 | (seq (splat $addend_s):$addend, unpred_cg)>; |

189 | def q_m_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, |

190 | unpromoted<Scalar>:$m2_s, |

191 | Predicate:$pred), |

192 | (seq (splat $m2_s):$m2, pred_cg)>; |

193 | def sq_m_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2, |

194 | unpromoted<Scalar>:$addend_s, |

195 | Predicate:$pred), |

196 | (seq (splat $addend_s):$addend, pred_cg)>; |

197 | } |

198 | } |

199 | |

200 | let params = T.Float in { |

201 | defm vfma: FMA<1>; |

202 | defm vfms: FMA<0>; |

203 | } |

204 | |

205 | let params = T.Int, pnt = PNT_NType in { |

206 | def vmlaq_n: Intrinsic< |

207 | Vector, (args Vector:$addend, Vector:$m1, unpromoted<Scalar>:$m2_s), |

208 | (add (mul $m1, (splat $m2_s)), $addend)>; |

209 | def vmlasq_n: Intrinsic< |

210 | Vector, (args Vector:$m1, Vector:$m2, unpromoted<Scalar>:$addend_s), |

211 | (add (mul $m1, $m2), (splat $addend_s))>; |

212 | |

213 | def vmlaq_m_n: Intrinsic< |

214 | Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred), |

215 | (IRInt<"vmla_n_predicated", [Vector, Predicate]> |

216 | $addend, $m1, $m2_s, $pred)>; |

217 | def vmlasq_m_n: Intrinsic< |

218 | Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred), |

219 | (IRInt<"vmlas_n_predicated", [Vector, Predicate]> |

220 | $m1, $m2, $addend_s, $pred)>; |

221 | } |

222 | |

223 | multiclass VQDMLA { |

224 | def hq_n: Intrinsic< |

225 | Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s), |

226 | (IRInt<NAME # "h", [Vector]> $addend, $m1, $m2_s)>; |

227 | def shq_n: Intrinsic< |

228 | Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s), |

229 | (IRInt<NAME # "sh", [Vector]> $m1, $m2, $addend_s)>; |

230 | |

231 | def hq_m_n: Intrinsic< |

232 | Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred), |

233 | (IRInt<NAME # "h_predicated", [Vector, Predicate]> |

234 | $addend, $m1, $m2_s, $pred)>; |

235 | def shq_m_n: Intrinsic< |

236 | Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred), |

237 | (IRInt<NAME # "sh_predicated", [Vector, Predicate]> |

238 | $m1, $m2, $addend_s, $pred)>; |

239 | } |

240 | |

241 | let params = T.Signed, pnt = PNT_NType in { |

242 | defm vqdmla: VQDMLA; |

243 | defm vqrdmla: VQDMLA; |

244 | } |

245 | |

246 | multiclass VQDMLAD<int exchange, int round, int subtract> { |

247 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c), |

248 | (IRInt<"vqdmlad", [Vector]> $a, $b, $c, |

249 | (u32 exchange), (u32 round), (u32 subtract))>; |

250 | def _m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c, |

251 | Predicate:$pred), |

252 | (IRInt<"vqdmlad_predicated", [Vector, Predicate]> $a, $b, $c, |

253 | (u32 exchange), (u32 round), (u32 subtract), $pred)>; |

254 | } |

255 | let params = T.Signed in { |

256 | defm vqdmladhq: VQDMLAD<0, 0, 0>; |

257 | defm vqdmladhxq: VQDMLAD<1, 0, 0>; |

258 | defm vqdmlsdhq: VQDMLAD<0, 0, 1>; |

259 | defm vqdmlsdhxq: VQDMLAD<1, 0, 1>; |

260 | defm vqrdmladhq: VQDMLAD<0, 1, 0>; |

261 | defm vqrdmladhxq: VQDMLAD<1, 1, 0>; |

262 | defm vqrdmlsdhq: VQDMLAD<0, 1, 1>; |

263 | defm vqrdmlsdhxq: VQDMLAD<1, 1, 1>; |

264 | } |

265 | |

266 | let params = !listconcat(T.Int16, T.Int32) in { |

267 | let pnt = PNT_None in { |

268 | def vmvnq_n: Intrinsic<Vector, (args imm_simd_vmvn:$imm), |

269 | (not (splat (Scalar $imm)))>; |

270 | } |

271 | defm vmvnq: IntrinsicMX<Vector, (args imm_simd_vmvn:$imm, Predicate:$pred), |

272 | (select $pred, (not (splat (Scalar $imm))), $inactive), |

273 | 1, "_n", PNT_NType, PNT_None>; |

274 | let pnt = PNT_NType in { |

275 | def vbicq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm), |

276 | (and $v, (not (splat (Scalar $imm))))>; |

277 | def vorrq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm), |

278 | (or $v, (splat (Scalar $imm)))>; |

279 | } |

280 | def vbicq_m_n: Intrinsic< |

281 | Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred), |

282 | (select $pred, (and $v, (not (splat (Scalar $imm)))), $v)>; |

283 | def vorrq_m_n: Intrinsic< |

284 | Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred), |

285 | (select $pred, (or $v, (splat (Scalar $imm))), $v)>; |

286 | } |

287 | |

288 | let params = T.Usual in { |

289 | let pnt = PNT_None in |

290 | def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>; |

291 | |

292 | defm vdupq: IntrinsicMX< |

293 | Vector, (args unpromoted<Scalar>:$s, Predicate:$pred), |

294 | (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>; |

295 | } |

296 | |

297 | multiclass vxdup_mc<dag paramsIn, dag paramsOut> { |

298 | defvar UnpredInt = IRInt<NAME, [Vector]>; |

299 | defvar PredInt = IRInt<NAME # "_predicated", [Vector, Predicate]>; |

300 | defvar UnpredIntCall = !con((UnpredInt $base), paramsOut); |

301 | defvar PredIntCall = !con((PredInt $inactive, $base), paramsOut, (? $pred)); |

302 | |

303 | // Straightforward case with neither writeback nor predication |

304 | let pnt = PNT_N in |

305 | def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn), |

306 | (xval UnpredIntCall, 0)>; |

307 | |

308 | // Predicated form without writeback |

309 | defm q: IntrinsicMX< |

310 | Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)), |

311 | (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>; |

312 | |

313 | // Writeback without predication |

314 | let pnt = PNT_WB in |

315 | def q_wb: Intrinsic< |

316 | Vector, !con((args Ptr<u32>:$baseaddr), paramsIn), |

317 | (seq (load $baseaddr):$base, |

318 | UnpredIntCall:$pair, |

319 | (store (xval $pair, 1), $baseaddr), |

320 | (xval $pair, 0))>; |

321 | |

322 | // Both writeback and predicated |

323 | defm q: IntrinsicMX< |

324 | Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)), |

325 | (seq (load $baseaddr):$base, |

326 | PredIntCall:$pair, |

327 | (store (xval $pair, 1), $baseaddr), |

328 | (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>; |

329 | } |

330 | |

331 | let params = T.Unsigned in { |

332 | defm vidup: vxdup_mc<(? imm_1248:$step), (? $step)>; |

333 | defm vddup: vxdup_mc<(? imm_1248:$step), (? $step)>; |

334 | defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>; |

335 | defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>; |

336 | } |

337 | |

338 | let params = T.Int in { |

339 | def vmvnq: Intrinsic<Vector, (args Vector:$a), |

340 | (xor $a, (uint_max Vector))>; |

341 | defm vmvnq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

342 | (IRInt<"mvn_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; |

343 | def vclzq: Intrinsic<Vector, (args Vector:$a), |

344 | (IRIntBase<"ctlz", [Vector]> $a, (i1 0))>; |

345 | defm vclzq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

346 | (IRInt<"clz_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; |

347 | } |

348 | let params = T.Signed in { |

349 | def vclsq: Intrinsic<Vector, (args Vector:$a), (IRInt<"vcls", [Vector]> $a)>; |

350 | defm vclsq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

351 | (IRInt<"cls_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; |

352 | |

353 | def vnegq: Intrinsic<Vector, (args Vector:$a), |

354 | (sub (zeroinit Vector), $a)>; |

355 | def vabsq: Intrinsic<Vector, (args Vector:$a), |

356 | (select (icmp_slt $a, (zeroinit Vector)), |

357 | (sub (zeroinit Vector), $a), $a)>; |

358 | def vqnegq: Intrinsic<Vector, (args Vector:$a), |

359 | (select (icmp_eq $a, (int_min Vector)), |

360 | (int_max Vector), |

361 | (sub (zeroinit Vector), $a))>; |

362 | def vqabsq: Intrinsic<Vector, (args Vector:$a), |

363 | (select (icmp_sgt $a, (zeroinit Vector)), $a, |

364 | (select (icmp_eq $a, (int_min Vector)), |

365 | (int_max Vector), |

366 | (sub (zeroinit Vector), $a)))>; |

367 | |

368 | foreach name = ["qneg", "qabs"] in { |

369 | defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

370 | (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive), |

371 | 0 /* no _x variant for saturating intrinsics */>; |

372 | } |

373 | } |

374 | let params = !listconcat(T.Signed, T.Float) in { |

375 | foreach name = ["neg", "abs"] in { |

376 | defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

377 | (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; |

378 | } |

379 | } |

380 | let params = T.Float in { |

381 | def vnegq_f: Intrinsic<Vector, (args Vector:$a), (fneg $a)>, |

382 | NameOverride<"vnegq">; |

383 | def vabsq_f: Intrinsic<Vector, (args Vector:$a), |

384 | (IRIntBase<"fabs", [Vector]> $a)>, NameOverride< "vabsq">; |

385 | } |

386 | |

387 | // The bitcasting below is not overcomplicating the IR because while |

388 | // Vector and UVector may be different vector types at the C level i.e. |

389 | // vectors of same size signed/unsigned ints. Once they're lowered |

390 | // to IR, they are just bit vectors with no sign at all, so the |

391 | // bitcasts will be automatically elided by IRBuilder. |

392 | multiclass predicated_bit_op_fp<string int_op> { |

393 | def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, |

394 | Predicate:$pred), |

395 | (bitcast (IRInt<int_op, [UVector, Predicate]> |

396 | (bitcast $a, UVector), |

397 | (bitcast $b, UVector), |

398 | $pred, |

399 | (bitcast $inactive, UVector)), Vector)>; |

400 | } |

401 | |

402 | // Plain intrinsics |

403 | let params = T.Usual in { |

404 | def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

405 | (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>; |

406 | } |

407 | |

408 | multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?), |

409 | bit wantXVariant = 1> { |

410 | defm "": IntrinsicMX< |

411 | Vector, (args Vector:$a, Vector:$b, Predicate:$pred), |

412 | !con((IRInt<operation, [Vector, Predicate]> $a, $b), |

413 | extraArgs, (? $pred, $inactive)), wantXVariant>; |

414 | } |

415 | |

416 | multiclass VectorScalarArithmetic<string operation, string basename, |

417 | dag extraArgs = (?), |

418 | bit wantXVariant = 1> { |

419 | defm "": IntrinsicMXNameOverride< |

420 | Vector, (args Vector:$a, unpromoted<Scalar>:$b, Predicate:$pred), |

421 | !con((IRInt<operation, [Vector, Predicate]> $a, (splat $b)), |

422 | extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n", |

423 | PNT_NType, PNT_NType>; |

424 | } |

425 | |

426 | multiclass VectorVectorArithmeticBitcast<string operation> { |

427 | defm "": IntrinsicMX<Vector, (args Vector:$a, Vector:$b, |

428 | Predicate:$pred), |

429 | (bitcast (IRInt<operation, [UVector, Predicate]> |

430 | (bitcast $a, UVector), |

431 | (bitcast $b, UVector), |

432 | $pred, |

433 | (bitcast $inactive, UVector)), Vector)>; |

434 | } |

435 | |

436 | // Predicated intrinsics |

437 | let params = T.Usual in { |

438 | defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>; |

439 | defm vaddq : VectorVectorArithmetic<"add_predicated">; |

440 | defm vsubq : VectorVectorArithmetic<"sub_predicated">; |

441 | defm vmulq : VectorVectorArithmetic<"mul_predicated">; |

442 | defm vandq : VectorVectorArithmeticBitcast<"and_predicated">; |

443 | defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">; |

444 | defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">; |

445 | defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">; |

446 | defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">; |

447 | |

448 | defm : VectorScalarArithmetic<"add_predicated", "vaddq">; |

449 | defm : VectorScalarArithmetic<"sub_predicated", "vsubq">; |

450 | defm : VectorScalarArithmetic<"mul_predicated", "vmulq">; |

451 | } |

452 | |

453 | multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?), |

454 | bit wantXVariant = 1> { |

455 | defm "": IntrinsicMX< |

456 | DblVector, (args Vector:$a, Vector:$b, DblPredicate:$pred), |

457 | !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, $b), |

458 | extraArgs, (? $pred, $inactive)), wantXVariant>; |

459 | } |

460 | |

461 | multiclass DblVectorScalarArithmetic<string operation, string basename, |

462 | dag extraArgs = (?), |

463 | bit wantXVariant = 1> { |

464 | defm "": IntrinsicMXNameOverride< |

465 | DblVector, (args Vector:$a, unpromoted<Scalar>:$b, DblPredicate:$pred), |

466 | !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, (splat $b)), |

467 | extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n", |

468 | PNT_NType, PNT_NType>; |

469 | } |

470 | |

471 | // Predicated intrinsics - Int types only |

472 | let params = T.Int in { |

473 | defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>; |

474 | defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>; |

475 | defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>; |

476 | defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>; |

477 | defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>; |

478 | defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>; |

479 | defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>; |

480 | defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>; |

481 | defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>; |

482 | defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>; |

483 | defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>; |

484 | |

485 | defm : VectorScalarArithmetic<"qadd_predicated", "vqaddq", (? (unsignedflag Scalar)), 0>; |

486 | defm : VectorScalarArithmetic<"hadd_predicated", "vhaddq", (? (unsignedflag Scalar))>; |

487 | defm : VectorScalarArithmetic<"qsub_predicated", "vqsubq", (? (unsignedflag Scalar)), 0>; |

488 | defm : VectorScalarArithmetic<"hsub_predicated", "vhsubq", (? (unsignedflag Scalar))>; |

489 | } |

490 | let params = T.Signed in { |

491 | defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>; |

492 | defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>; |

493 | def vminaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred), |

494 | (IRInt<"vmina_predicated", [UVector,Predicate]> $a, $b, $pred)>; |

495 | def vmaxaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred), |

496 | (IRInt<"vmaxa_predicated", [UVector,Predicate]> $a, $b, $pred)>; |

497 | |

498 | defm : VectorScalarArithmetic<"qdmulh_predicated", "vqdmulhq", (?), 0>; |

499 | defm : VectorScalarArithmetic<"qrdmulh_predicated", "vqrdmulhq", (?), 0>; |

500 | } |

501 | |

502 | let params = T.Poly, overrideKindLetter = "p"in { |

503 | defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>; |

504 | defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>; |

505 | } |

506 | |

507 | let params = [s16, s32] in { |

508 | def vqdmullbq: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

509 | (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 0)>; |

510 | def vqdmulltq: Intrinsic<DblVector, (args Vector:$a, Vector:$b), |

511 | (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 1)>; |

512 | defm vqdmullbq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 0)), 0>; |

513 | defm vqdmulltq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 1)), 0>; |

514 | |

515 | let pnt = PNT_NType in { |

516 | def vqdmullbq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b), |

517 | (IRInt<"vqdmull", [DblVector, Vector]> |

518 | $a, (splat $b), 0)>; |

519 | def vqdmulltq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b), |

520 | (IRInt<"vqdmull", [DblVector, Vector]> |

521 | $a, (splat $b), 1)>; |

522 | } |

523 | defm vqdmullbq_n: DblVectorScalarArithmetic<"vqdmull_predicated", |

524 | "vqdmullbq", (? (u32 0)), 0>; |

525 | defm vqdmulltq_n: DblVectorScalarArithmetic<"vqdmull_predicated", |

526 | "vqdmulltq", (? (u32 1)), 0>; |

527 | } |

528 | |

529 | // Predicated intrinsics - Float types only |

530 | let params = T.Float in { |

531 | defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>; |

532 | defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>; |

533 | def vminnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), |

534 | (IRInt<"vminnma_predicated", [Vector,Predicate]> $a, $b, $pred)>; |

535 | def vmaxnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), |

536 | (IRInt<"vmaxnma_predicated", [Vector,Predicate]> $a, $b, $pred)>; |

537 | } |

538 | |

539 | multiclass Reduction<Type Accumulator, string basename, list<Type> basetypes, |

540 | bit needSign = 0, |

541 | dag postCG = (seq (id $ret)), |

542 | dag accArg = (args Accumulator:$prev), |

543 | dag preCG = (seq)> { |

544 | defvar intArgsBase = (? $prev, $vec); |

545 | defvar intArgsUnpred = !con(intArgsBase, |

546 | !if(needSign, (? (unsignedflag Scalar)), (?))); |

547 | defvar intArgsPred = !con(intArgsUnpred, (? $pred)); |

548 | defvar intUnpred = !setdagop(intArgsUnpred, IRInt<basename, basetypes>); |

549 | defvar intPred = !setdagop(intArgsPred, IRInt< |

550 | basename#"_predicated", !listconcat(basetypes, [Predicate])>); |

551 | |

552 | def "": Intrinsic< |

553 | Accumulator, !con(accArg, (args Vector:$vec)), |

554 | !con(preCG, (seq intUnpred:$ret), postCG)>; |

555 | def _p: Intrinsic< |

556 | Accumulator, !con(accArg, (args Vector:$vec, Predicate:$pred)), |

557 | !con(preCG, (seq intPred:$ret), postCG)>; |

558 | } |

559 | |

560 | let params = T.Int in { |

561 | defm vminvq: Reduction<Scalar, "minv", [Vector], 1, (seq (Scalar $ret))>; |

562 | defm vmaxvq: Reduction<Scalar, "maxv", [Vector], 1, (seq (Scalar $ret))>; |

563 | } |

564 | |

565 | let params = T.Signed in { |

566 | defm vminavq: Reduction<UScalar, "minav", [Vector], 0, (seq (UScalar $ret))>; |

567 | defm vmaxavq: Reduction<UScalar, "maxav", [Vector], 0, (seq (UScalar $ret))>; |

568 | } |

569 | |

570 | let params = T.Float in { |

571 | defm vminnmvq: Reduction<Scalar, "minnmv", [Scalar, Vector]>; |

572 | defm vmaxnmvq: Reduction<Scalar, "maxnmv", [Scalar, Vector]>; |

573 | defm vminnmavq: Reduction<Scalar, "minnmav", [Scalar, Vector]>; |

574 | defm vmaxnmavq: Reduction<Scalar, "maxnmav", [Scalar, Vector]>; |

575 | } |

576 | |

577 | foreach half = [ "b", "t"] in { |

578 | defvar halfconst = !if(!eq(half, "b"), 0, 1); |

579 | |

580 | let params = [f32], pnt = PNT_None in { |

581 | def vcvt#half#q_f16: Intrinsic< |

582 | VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a), |

583 | (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>; |

584 | def vcvt#half#q_m_f16: Intrinsic< |

585 | VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred), |

586 | (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>; |

587 | } // params = [f32], pnt = PNT_None |

588 | |

589 | let params = [f16], pnt = PNT_None in { |

590 | def vcvt#half#q_f32: Intrinsic<VecOf<f32>, (args Vector:$a), |

591 | (IRInt<"vcvt_widen"> $a, halfconst)>; |

592 | defm vcvt#half#q: IntrinsicMX< |

593 | VecOf<f32>, (args Vector:$a, PredOf<f32>:$pred), |

594 | (IRInt<"vcvt_widen_predicated"> $inactive, $a, halfconst, $pred), |

595 | 1, "_f32">; |

596 | } // params = [f16], pnt = PNT_None |

597 | } // loop over half = "b", "t" |

598 | |

599 | multiclass float_int_conversions<Type FScalar, Type IScalar, IRBuilderBase ftoi, IRBuilderBase itof> { |

600 | defvar FVector = VecOf<FScalar>; |

601 | defvar IVector = VecOf<IScalar>; |

602 | |

603 | let params = [IScalar] in { |

604 | let pnt = PNT_2Type in { |

605 | def : Intrinsic<FVector, (args IVector:$a), (itof $a, FVector)>, |

606 | NameOverride<"vcvtq_"# FScalar>; |

607 | } |

608 | defm vcvtq: IntrinsicMX<FVector, (args IVector:$a, Predicate:$pred), |

609 | (IRInt<"vcvt_fp_int_predicated", [FVector, IVector, Predicate]> |

610 | $a, (unsignedflag IScalar), $pred, $inactive), |

611 | 1, "_"# FScalar, PNT_2Type, PNT_2Type>; |

612 | } |

613 | let params = [FScalar] in { |

614 | let pnt = PNT_None in { |

615 | def : Intrinsic<IVector, (args FVector:$a), (ftoi $a, IVector)>, |

616 | NameOverride<"vcvtq_"# IScalar>; |

617 | |

618 | foreach suffix = ["a", "n", "p", "m"] in |

619 | def : Intrinsic<IVector, (args FVector:$a), |

620 | (IRInt<"vcvt"#suffix, [IVector, FVector]> |

621 | (unsignedflag IScalar), $a)>, |

622 | NameOverride<"vcvt"#suffix# "q_"# IScalar>; |

623 | } |

624 | defm vcvtq: IntrinsicMX<IVector, (args FVector:$a, Predicate:$pred), |

625 | (IRInt<"vcvt_fp_int_predicated", [IVector, FVector, Predicate]> |

626 | $a, (unsignedflag IScalar), $pred, $inactive), |

627 | 1, "_"# IScalar, PNT_2Type, PNT_None>; |

628 | |

629 | foreach suffix = ["a", "n", "p", "m"] in { |

630 | defm "vcvt"#suffix# "q": IntrinsicMX< |

631 | IVector, (args FVector:$a, Predicate:$pred), |

632 | (IRInt<"vcvt"#suffix# "_predicated", [IVector, FVector, Predicate]> |

633 | (unsignedflag IScalar), $inactive, $a, $pred), |

634 | 1, "_"# IScalar, PNT_2Type, PNT_None>; |

635 | } |

636 | } |

637 | } |

638 | |

639 | defm "": float_int_conversions<f32, u32, fptoui, uitofp>; |

640 | defm "": float_int_conversions<f16, u16, fptoui, uitofp>; |

641 | defm "": float_int_conversions<f32, s32, fptosi, sitofp>; |

642 | defm "": float_int_conversions<f16, s16, fptosi, sitofp>; |

643 | |

644 | multiclass vmovl<bit top> { |

645 | let params = [s8, u8, s16, u16] in { |

646 | def "": Intrinsic<DblVector, (args Vector:$a), |

647 | (extend (unzip $a, top), DblVector, (unsignedflag Scalar))>; |

648 | defm "": IntrinsicMX<DblVector, (args Vector:$a, DblPredicate:$pred), |

649 | (IRInt<"vmovl_predicated", [DblVector, Vector, DblPredicate]> |

650 | $a, (unsignedflag Scalar), top, $pred, $inactive)>; |

651 | } |

652 | } |

653 | |

654 | defm vmovlbq: vmovl<0>; |

655 | defm vmovltq: vmovl<1>; |

656 | |

657 | multiclass vmovn<bit top, dag wide_result> { |

658 | let params = [s16, u16, s32, u32] in { |

659 | def "": Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a), |

660 | (trunc wide_result, HalfVector)>; |

661 | def _m: Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a, |

662 | Predicate:$pred), |

663 | (IRInt<"vmovn_predicated", [HalfVector, Vector, Predicate]> |

664 | $inactive, $a, top, $pred)>; |

665 | } |

666 | } |

667 | |

668 | defm vmovntq: vmovn<1, (zip (vreinterpret $inactive, Vector), $a)>; |

669 | defm vmovnbq: vmovn<0, |

670 | (zip $a, (vreinterpret (vrev $inactive, (bitsize Scalar)), Vector))>; |

671 | |

672 | multiclass vqmovn<bit top, Type RetScalar> { |

673 | defvar RetVector = VecOf<RetScalar>; |

674 | |

675 | let params = [s16, u16, s32, u32] in { |

676 | def : Intrinsic< |

677 | RetVector, (args RetVector:$inactive, Vector:$a), |

678 | (IRInt<"vqmovn", [RetVector, Vector]> |

679 | $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), top)>, |

680 | NameOverride<NAME>; |

681 | def: Intrinsic< |

682 | RetVector, (args RetVector:$inactive, Vector:$a, Predicate:$pred), |

683 | (IRInt<"vqmovn_predicated", [RetVector, Vector, Predicate]> |

684 | $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), |

685 | top, $pred)>, |

686 | NameOverride<NAME # "_m">; |

687 | } |

688 | } |

689 | |

690 | let params = [s16, s32, u16, u32] in { |

691 | defm vqmovntq: vqmovn<1, HalfScalar>; |

692 | defm vqmovnbq: vqmovn<0, HalfScalar>; |

693 | } |

694 | let params = [s16, s32] in { |

695 | defm vqmovuntq: vqmovn<1, UHalfScalar>; |

696 | defm vqmovunbq: vqmovn<0, UHalfScalar>; |

697 | } |

698 | |

699 | multiclass vrnd<IRIntBase ir_int, string suffix> { |

700 | let params = T.Float in { |

701 | def "": Intrinsic<Vector, (args Vector:$a), (ir_int $a)>; |

702 | defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

703 | (IRInt<"vrint"#suffix# "_predicated", [Vector, Predicate]> |

704 | $a, $pred, $inactive)>; |

705 | } |

706 | } |

707 | |

708 | defm vrndq: vrnd<IRIntBase<"trunc", [Vector]>, "z">; |

709 | defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">; |

710 | defm vrndpq: vrnd<IRIntBase<"ceil", [Vector]>, "p">; |

711 | defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">; |

712 | defm vrndxq: vrnd<IRIntBase<"rint", [Vector]>, "x">; |

713 | defm vrndnq: vrnd<IRInt<"vrintn", [Vector]>, "n">; |

714 | |

715 | multiclass compare_with_pred<string condname, dag arguments, |

716 | dag cmp, string suffix> { |

717 | // Make the predicated and unpredicated versions of a single comparison. |

718 | def: Intrinsic<Predicate, arguments, |

719 | (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>, |

720 | NameOverride<"vcmp"# condname # "q"# suffix>; |

721 | def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)), |

722 | (u16 (IRInt<"pred_v2i", [Predicate]> ( and $inpred, cmp)))>, |

723 | NameOverride<"vcmp"# condname # "q_m"# suffix>; |

724 | } |

725 | |

726 | multiclass compare<string condname, IRBuilder cmpop> { |

727 | // Make all four variants of a comparison: the vector/vector and |

728 | // vector/scalar forms, each using compare_with_pred to make a |

729 | // predicated and unpredicated version. |

730 | defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb), |

731 | (cmpop $va, $vb), "">; |

732 | let pnt = PNT_NType in { |

733 | defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb), |

734 | (cmpop $va, (splat $sb)), "_n">; |

735 | } |

736 | } |

737 | let params = T.Int in { |

738 | defm: compare<"eq", icmp_eq>; |

739 | defm: compare<"ne", icmp_ne>; |

740 | } |

741 | let params = T.Signed in { |

742 | defm: compare<"gt", icmp_sgt>; |

743 | defm: compare<"ge", icmp_sge>; |

744 | defm: compare<"lt", icmp_slt>; |

745 | defm: compare<"le", icmp_sle>; |

746 | } |

747 | let params = T.Unsigned in { |

748 | defm: compare<"hi", icmp_ugt>; |

749 | defm: compare<"cs", icmp_uge>; |

750 | } |

751 | let params = T.Float in { |

752 | defm: compare<"eq", fcmp_eq>; |

753 | defm: compare<"ne", fcmp_ne>; |

754 | defm: compare<"gt", fcmp_gt>; |

755 | defm: compare<"ge", fcmp_ge>; |

756 | defm: compare<"lt", fcmp_lt>; |

757 | defm: compare<"le", fcmp_le>; |

758 | } |

759 | |

760 | let params = T.Signed in { |

761 | def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

762 | (select (icmp_sle $a, $b), $a, $b)>; |

763 | def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

764 | (select (icmp_sge $a, $b), $a, $b)>; |

765 | def vminaq: Intrinsic<UVector, (args UVector:$a, Vector:$b), |

766 | (seq (select (icmp_slt $b, (zeroinit Vector)), |

767 | (sub (zeroinit Vector), $b), $b):$absb, |

768 | (select (icmp_ule $a, $absb), $a, $absb))>; |

769 | def vmaxaq: Intrinsic<UVector, (args UVector:$a, Vector:$b), |

770 | (seq (select (icmp_slt $b, (zeroinit Vector)), |

771 | (sub (zeroinit Vector), $b), $b):$absb, |

772 | (select (icmp_uge $a, $absb), $a, $absb))>; |

773 | } |

774 | let params = T.Unsigned in { |

775 | def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

776 | (select (icmp_ule $a, $b), $a, $b)>, |

777 | NameOverride<"vminq">; |

778 | def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

779 | (select (icmp_uge $a, $b), $a, $b)>, |

780 | NameOverride<"vmaxq">; |

781 | } |

782 | let params = T.Float in { |

783 | def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

784 | (IRIntBase<"minnum", [Vector]> $a, $b)>; |

785 | def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

786 | (IRIntBase<"maxnum", [Vector]> $a, $b)>; |

787 | def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

788 | (IRIntBase<"minnum", [Vector]> |

789 | (IRIntBase<"fabs", [Vector]> $a), |

790 | (IRIntBase<"fabs", [Vector]> $b))>; |

791 | def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b), |

792 | (IRIntBase<"maxnum", [Vector]> |

793 | (IRIntBase<"fabs", [Vector]> $a), |

794 | (IRIntBase<"fabs", [Vector]> $b))>; |

795 | } |

796 | |

797 | def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred), |

798 | (select $pred, $t, $f)> { let params = T.Usual; } |

799 | def vpselq_64: Intrinsic< |

800 | Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred), |

801 | (bitcast (select $pred, (bitcast $t, VecOf<u32>), |

802 | (bitcast $f, VecOf<u32>)), Vector)>, |

803 | NameOverride<"vpselq"> { let params = T.All64; } |

804 | |

805 | let params = [Void], pnt = PNT_None in { |

806 | |

807 | multiclass vctp<Type pred, string intname> { |

808 | def "": Intrinsic<pred, (args u32:$val), |

809 | (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>; |

810 | def _m: Intrinsic<pred, (args u32:$val, pred:$inpred), |

811 | (u16 (IRInt<"pred_v2i", [pred]> ( and $inpred, |

812 | (IRIntBase<intname> $val))))>; |

813 | } |

814 | defm vctp8q: vctp<PredOf<u8>, "arm_mve_vctp8">; |

815 | defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">; |

816 | defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">; |

817 | defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">; |

818 | |

819 | def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred), |

820 | (xor $pred, (u16 65535))>; |

821 | |

822 | } |

823 | |

824 | multiclass contiguous_load<string mnemonic, PrimitiveType memtype, |

825 | list<Type> same_size, list<Type> wider> { |

826 | // Intrinsics named with explicit memory and element sizes that match: |

827 | // vldrbq_?8, vldrhq_?16, vldrwq_?32. |

828 | let params = same_size, pnt = PNT_None in { |

829 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), |

830 | (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, |

831 | NameOverride<mnemonic>; |

832 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, |

833 | Predicate:$pred), |

834 | (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> |

835 | (CPtr<Vector> $addr), !srl(memtype.size,3), |

836 | $pred, (zeroinit Vector))>, |

837 | NameOverride<mnemonic # "_z">; |

838 | } |

839 | |

840 | // Synonyms for the above, with the generic name vld1q that just means |

841 | // 'memory and element sizes match', and allows convenient polymorphism with |

842 | // the memory and element types covariant. |

843 | let params = same_size in { |

844 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), |

845 | (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, |

846 | NameOverride<"vld1q">; |

847 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, |

848 | Predicate:$pred), |

849 | (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> |

850 | (CPtr<Vector> $addr), !srl(memtype.size,3), |

851 | $pred, (zeroinit Vector))>, |

852 | NameOverride<"vld1q_z">; |

853 | } |

854 | |

855 | // Intrinsics with the memory size narrower than the vector element, so that |

856 | // they load less than 128 bits of memory and sign/zero extend each loaded |

857 | // value into a wider vector lane. |

858 | let params = wider, pnt = PNT_None in { |

859 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), |

860 | (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>> |

861 | $addr), !srl(memtype.size,3))), |

862 | Vector, (unsignedflag Scalar))>, |

863 | NameOverride<mnemonic>; |

864 | def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, |

865 | Predicate:$pred), |

866 | (extend (IRIntBase<"masked_load", |

867 | [NarrowedVecOf<memtype,Vector>, |

868 | CPtr<NarrowedVecOf<memtype,Vector>>]> |

869 | (CPtr<NarrowedVecOf<memtype,Vector>> $addr), |

870 | !srl(memtype.size,3), $pred, |

871 | (zeroinit NarrowedVecOf<memtype,Vector>)), |

872 | Vector, (unsignedflag Scalar))>, |

873 | NameOverride<mnemonic # "_z">; |

874 | } |

875 | } |

876 | |

877 | defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; |

878 | defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>; |

879 | defm: contiguous_load<"vldrwq", u32, T.All32, []>; |

880 | |

881 | multiclass contiguous_store<string mnemonic, PrimitiveType memtype, |

882 | list<Type> same_size, list<Type> wider> { |

883 | // Intrinsics named with explicit memory and element sizes that match: |

884 | // vstrbq_?8, vstrhq_?16, vstrwq_?32. |

885 | let params = same_size in { |

886 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

887 | Vector:$value), |

888 | (store $value, |

889 | (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, |

890 | NameOverride<mnemonic>; |

891 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

892 | Vector:$value, Predicate:$pred), |

893 | (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> |

894 | $value, (Ptr<Vector> $addr), |

895 | !srl(memtype.size,3), $pred)>, |

896 | NameOverride<mnemonic # "_p">; |

897 | } |

898 | |

899 | // Synonyms for the above, with the generic name vst1q that just means |

900 | // 'memory and element sizes match', and allows convenient polymorphism with |

901 | // the memory and element types covariant. |

902 | let params = same_size in { |

903 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

904 | Vector:$value), |

905 | (store $value, |

906 | (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, |

907 | NameOverride<"vst1q">; |

908 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

909 | Vector:$value, Predicate:$pred), |

910 | (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> |

911 | $value, (Ptr<Vector> $addr), |

912 | !srl(memtype.size,3), $pred)>, |

913 | NameOverride<"vst1q_p">; |

914 | } |

915 | |

916 | // Intrinsics with the memory size narrower than the vector element, so that |

917 | // they store less than 128 bits of memory, truncating each vector lane into |

918 | // a narrower value to store. |

919 | let params = wider in { |

920 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

921 | Vector:$value), |

922 | (store (trunc $value, NarrowedVecOf<memtype,Vector>), |

923 | (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr), |

924 | !srl(memtype.size,3)))>, |

925 | NameOverride<mnemonic>; |

926 | def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, |

927 | Vector:$value, Predicate:$pred), |

928 | (IRIntBase<"masked_store", |

929 | [NarrowedVecOf<memtype,Vector>, |

930 | Ptr<NarrowedVecOf<memtype,Vector>>]> |

931 | (trunc $value, NarrowedVecOf<memtype,Vector>), |

932 | (Ptr<NarrowedVecOf<memtype,Vector>> $addr), |

933 | !srl(memtype.size,3), $pred)>, |

934 | NameOverride<mnemonic # "_p">; |

935 | } |

936 | } |

937 | |

938 | defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; |

939 | defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>; |

940 | defm: contiguous_store<"vstrwq", u32, T.All32, []>; |

941 | |

942 | multiclass gather_base<list<Type> types, int size> { |

943 | let params = types, pnt = PNT_None in { |

944 | def _gather_base: Intrinsic< |

945 | Vector, (args UVector:$addr, imm_mem7bit<size>:$offset), |

946 | (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>; |

947 | |

948 | def _gather_base_z: Intrinsic< |

949 | Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred), |

950 | (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]> |

951 | $addr, $offset, $pred)>; |

952 | |

953 | def _gather_base_wb: Intrinsic< |

954 | Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset), |

955 | (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]> |

956 | (load $addr), $offset):$pair, |

957 | (store (xval $pair, 1), $addr), |

958 | (xval $pair, 0))>; |

959 | |

960 | def _gather_base_wb_z: Intrinsic< |

961 | Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, |

962 | Predicate:$pred), |

963 | (seq (IRInt<"vldr_gather_base_wb_predicated", |

964 | [Vector, UVector, Predicate]> |

965 | (load $addr), $offset, $pred):$pair, |

966 | (store (xval $pair, 1), $addr), |

967 | (xval $pair, 0))>; |

968 | } |

969 | } |

970 | |

971 | defm vldrwq: gather_base<T.All32, 4>; |

972 | defm vldrdq: gather_base<T.All64, 8>; |

973 | |

974 | multiclass scatter_base<list<Type> types, int size> { |

975 | let params = types in { |

976 | def _scatter_base: Intrinsic< |

977 | Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data), |

978 | (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>; |

979 | |

980 | def _scatter_base_p: Intrinsic< |

981 | Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data, |

982 | Predicate:$pred), |

983 | (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]> |

984 | $addr, $offset, $data, $pred)>; |

985 | |

986 | def _scatter_base_wb: Intrinsic< |

987 | Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data), |

988 | (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]> |

989 | (load $addr), $offset, $data):$wbaddr, |

990 | (store $wbaddr, $addr))>; |

991 | |

992 | def _scatter_base_wb_p: Intrinsic< |

993 | Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, |

994 | Vector:$data, Predicate:$pred), |

995 | (seq (IRInt<"vstr_scatter_base_wb_predicated", |

996 | [UVector, Vector, Predicate]> |

997 | (load $addr), $offset, $data, $pred):$wbaddr, |

998 | (store $wbaddr, $addr))>; |

999 | } |

1000 | } |

1001 | |

1002 | defm vstrwq: scatter_base<T.All32, 4>; |

1003 | defm vstrdq: scatter_base<T.All64, 8>; |

1004 | |

1005 | multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> { |

1006 | let params = types in { |

1007 | def _gather_offset: Intrinsic< |

1008 | Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), |

1009 | (IRInt<"vldr_gather_offset", |

1010 | [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> |

1011 | $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>; |

1012 | def _gather_offset_z: Intrinsic< |

1013 | Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1014 | Predicate:$pred), |

1015 | (IRInt<"vldr_gather_offset_predicated", |

1016 | [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> |

1017 | $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>; |

1018 | } |

1019 | } |

1020 | |

1021 | multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype, |

1022 | int shift> { |

1023 | let params = types in { |

1024 | def _gather_shifted_offset: Intrinsic< |

1025 | Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), |

1026 | (IRInt<"vldr_gather_offset", |

1027 | [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> |

1028 | $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>; |

1029 | def _gather_shifted_offset_z: Intrinsic< |

1030 | Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1031 | Predicate:$pred), |

1032 | (IRInt<"vldr_gather_offset_predicated", |

1033 | [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> |

1034 | $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>; |

1035 | } |

1036 | } |

1037 | |

1038 | multiclass gather_offset_both<list<Type> types, PrimitiveType memtype, |

1039 | int shift> { |

1040 | defm "": gather_offset_unshifted<types, memtype>; |

1041 | defm "": gather_offset_shifted<types, memtype, shift>; |

1042 | } |

1043 | |

1044 | defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>; |

1045 | defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; |

1046 | defm vldrwq: gather_offset_both<T.All32, u32, 2>; |

1047 | defm vldrdq: gather_offset_both<T.Int64, u64, 3>; |

1048 | |

1049 | multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> { |

1050 | let params = types in { |

1051 | def _scatter_offset: Intrinsic< |

1052 | Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1053 | Vector:$data), |

1054 | (IRInt<"vstr_scatter_offset", |

1055 | [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> |

1056 | $base, $offsets, $data, memtype.size, 0)>; |

1057 | def _scatter_offset_p: Intrinsic< |

1058 | Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1059 | Vector:$data, Predicate:$pred), |

1060 | (IRInt<"vstr_scatter_offset_predicated", |

1061 | [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> |

1062 | $base, $offsets, $data, memtype.size, 0, $pred)>; |

1063 | } |

1064 | } |

1065 | |

1066 | multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype, |

1067 | int shift> { |

1068 | let params = types in { |

1069 | def _scatter_shifted_offset: Intrinsic< |

1070 | Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1071 | Vector:$data), |

1072 | (IRInt<"vstr_scatter_offset", |

1073 | [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> |

1074 | $base, $offsets, $data, memtype.size, shift)>; |

1075 | def _scatter_shifted_offset_p: Intrinsic< |

1076 | Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, |

1077 | Vector:$data, Predicate:$pred), |

1078 | (IRInt<"vstr_scatter_offset_predicated", |

1079 | [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> |

1080 | $base, $offsets, $data, memtype.size, shift, $pred)>; |

1081 | } |

1082 | } |

1083 | |

1084 | multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype, |

1085 | int shift> { |

1086 | defm "": scatter_offset_unshifted<types, memtype>; |

1087 | defm "": scatter_offset_shifted<types, memtype, shift>; |

1088 | } |

1089 | |

1090 | defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>; |

1091 | defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; |

1092 | defm vstrwq: scatter_offset_both<T.All32, u32, 2>; |

1093 | defm vstrdq: scatter_offset_both<T.Int64, u64, 3>; |

1094 | |

1095 | let params = T.Int in { |

1096 | def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), |

1097 | (shl $v, (splat (Scalar $sh)))>; |

1098 | defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh, |

1099 | Predicate:$pred), |

1100 | (IRInt<"shl_imm_predicated", [Vector, Predicate]> |

1101 | $v, $sh, $pred, $inactive), 1, "_n">; |

1102 | |

1103 | let pnt = PNT_NType in { |

1104 | def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), |

1105 | (immshr $v, $sh, (unsignedflag Scalar))>; |

1106 | defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, |

1107 | Predicate:$pred), |

1108 | (IRInt<"shr_imm_predicated", [Vector, Predicate]> |

1109 | $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; |

1110 | } |

1111 | } |

1112 | |

1113 | let params = T.Int in { |

1114 | def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), |

1115 | (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; |

1116 | def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v, |

1117 | imm_0toNm1:$sh, Predicate:$pred), |

1118 | (IRInt<"vqshl_imm_predicated", [Vector, Predicate]> |

1119 | $v, $sh, (unsignedflag Scalar), $pred, $inactive)>; |

1120 | |

1121 | let pnt = PNT_NType in { |

1122 | def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), |

1123 | (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; |

1124 | defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, |

1125 | Predicate:$pred), |

1126 | (IRInt<"vrshr_imm_predicated", [Vector, Predicate]> |

1127 | $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; |

1128 | } |

1129 | } |

1130 | |

1131 | let params = T.Signed, pnt = PNT_NType in { |

1132 | def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh), |

1133 | (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>; |

1134 | def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v, |

1135 | imm_0toNm1:$sh, Predicate:$pred), |

1136 | (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]> |

1137 | $v, $sh, $pred, $inactive)>; |

1138 | } |

1139 | |

1140 | multiclass vshll_imm<int top> { |

1141 | let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in { |

1142 | def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh), |

1143 | (IRInt<"vshll_imm", [DblVector, Vector]> |

1144 | $v, $sh, (unsignedflag Scalar), top)>; |

1145 | defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh, |

1146 | DblPredicate:$pred), |

1147 | (IRInt<"vshll_imm_predicated", [DblVector, Vector, DblPredicate]> |

1148 | $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">; |

1149 | } |

1150 | } |

1151 | defm vshllbq : vshll_imm<0>; |

1152 | defm vshlltq : vshll_imm<1>; |

1153 | |

1154 | multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME, |

1155 | dag extraargs = (?)> { |

1156 | defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)), |

1157 | [Vector], [outtype, Vector]); |

1158 | |

1159 | def q_n: Intrinsic< |

1160 | outtype, (args outtype:$a, Vector:$b, imm:$sh), |

1161 | !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>; |

1162 | |

1163 | def q_m_n: Intrinsic< |

1164 | outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred), |

1165 | !con((IRInt<intname # "_predicated", intparams # [Predicate]> |

1166 | $a, $b, $sh), extraargs, (? $pred))>; |

1167 | } |

1168 | |

1169 | multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> { |

1170 | defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>; |

1171 | defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>; |

1172 | } |

1173 | |

1174 | let params = [s16, s32, u16, u32], pnt = PNT_NType in { |

1175 | defvar U = (unsignedflag Scalar); |

1176 | defm vshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>; |

1177 | defm vqshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>; |

1178 | defm vrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>; |

1179 | defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>; |

1180 | } |

1181 | let params = [s16, s32], pnt = PNT_NType in { |

1182 | defm vqshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>; |

1183 | defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>; |

1184 | } |

1185 | let params = T.Int, pnt = PNT_NType in { |

1186 | defm vsli : DyadicImmShift<Vector, imm_0toNm1>; |

1187 | defm vsri : DyadicImmShift<Vector, imm_1toN>; |

1188 | } |

1189 | |

1190 | multiclass VSHL_non_imm<string scalarSuffix, int q, int r, |

1191 | PolymorphicNameType pnt_scalar_unpred = PNT_Type> { |

1192 | let pnt = pnt_scalar_unpred in { |

1193 | def scalarSuffix: Intrinsic< |

1194 | Vector, (args Vector:$in, s32:$sh), |

1195 | (IRInt<"vshl_scalar", [Vector]> $in, $sh, |

1196 | q, r, (unsignedflag Scalar))>; |

1197 | } |

1198 | def "_m"# scalarSuffix: Intrinsic< |

1199 | Vector, (args Vector:$in, s32:$sh, Predicate:$pred), |

1200 | (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh, |

1201 | q, r, (unsignedflag Scalar), $pred)>; |

1202 | |

1203 | def "": Intrinsic< |

1204 | Vector, (args Vector:$in, SVector:$sh), |

1205 | (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh, |

1206 | q, r, (unsignedflag Scalar))>; |

1207 | defm "": IntrinsicMX< |

1208 | Vector, (args Vector:$in, SVector:$sh, Predicate:$pred), |

1209 | (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh, |

1210 | q, r, (unsignedflag Scalar), $pred, $inactive), |

1211 | // The saturating shift intrinsics don't have an x variant, so we |

1212 | // set wantXVariant to 1 iff q == 0 |

1213 | !eq(q, 0)>; |

1214 | } |

1215 | |

1216 | let params = T.Int in { |

1217 | defm vshlq : VSHL_non_imm<"_r", 0, 0>; |

1218 | defm vqshlq : VSHL_non_imm<"_r", 1, 0>; |

1219 | defm vrshlq : VSHL_non_imm<"_n", 0, 1, PNT_NType>; |

1220 | defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>; |

1221 | } |

1222 | |

1223 | // Base class for the scalar shift intrinsics. |

1224 | class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: |

1225 | Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> { |

1226 | let params = [Void]; |

1227 | let pnt = PNT_None; |

1228 | } |

1229 | |

1230 | // Subclass that includes the machinery to take a 64-bit input apart |

1231 | // into halves, retrieve the two halves of a shifted output as a pair, |

1232 | // and glue the pieces of the pair back into an i64 for output. |

1233 | class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: |

1234 | ScalarShift<argtype, shiftCountArg, |

1235 | (seq (u32 (lshr $value, (argtype 32))):$hi, |

1236 | (u32 $value):$lo, |

1237 | shiftCodeGen:$pair, |

1238 | (or (shl (u64 (xval $pair, 1)), (u64 32)), |

1239 | (u64 (xval $pair, 0))))>; |

1240 | |

1241 | // The family of saturating/rounding scalar shifts that take an |

1242 | // immediate shift count. They come in matched 32- and 64-bit pairs. |

1243 | multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> { |

1244 | def "": ScalarShift<arg32, (args imm_1to32:$sh), |

1245 | (IRInt<NAME> $value, $sh)>; |

1246 | def l: LongScalarShift<arg64, (args imm_1to32:$sh), |

1247 | (IRInt<NAME # "l"> $lo, $hi, $sh)>; |

1248 | } |

1249 | defm uqshl: ScalarSaturatingShiftImm<u32, u64>; |

1250 | defm urshr: ScalarSaturatingShiftImm<u32, u64>; |

1251 | defm sqshl: ScalarSaturatingShiftImm<s32, s64>; |

1252 | defm srshr: ScalarSaturatingShiftImm<s32, s64>; |

1253 | |

1254 | // The family of saturating/rounding scalar shifts that take a |

1255 | // register shift count. They also have 32- and 64-bit forms, but the |

1256 | // 64-bit form also has a version that saturates to 48 bits, so the IR |

1257 | // intrinsic takes an extra saturation-type operand. |

1258 | multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> { |

1259 | def "": ScalarShift<arg32, (args s32:$sh), |

1260 | (IRInt<NAME> $value, $sh)>; |

1261 | def l: LongScalarShift<arg64, (args s32:$sh), |

1262 | (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>; |

1263 | def l_sat48: LongScalarShift<arg64, (args s32:$sh), |

1264 | (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>; |

1265 | } |

1266 | defm uqrshl: ScalarSaturatingShiftReg<u32, u64>; |

1267 | defm sqrshr: ScalarSaturatingShiftReg<s32, s64>; |

1268 | |

1269 | // The intrinsics for LSLL and ASRL come in 64-bit versions only, with |

1270 | // no saturation count. |

1271 | def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>; |

1272 | def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>; |

1273 | |

1274 | multiclass vadcsbc { |

1275 | def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), |

1276 | (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair, |

1277 | (store (and 1, (lshr (xval $pair, 1), 29)), $carry), |

1278 | (xval $pair, 0))>; |

1279 | def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), |

1280 | (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair, |

1281 | (store (and 1, (lshr (xval $pair, 1), 29)), $carry), |

1282 | (xval $pair, 0))>; |

1283 | def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, |

1284 | Ptr<uint>:$carry, Predicate:$pred), |

1285 | (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b, |

1286 | (shl (load $carry), 29), $pred):$pair, |

1287 | (store (and 1, (lshr (xval $pair, 1), 29)), $carry), |

1288 | (xval $pair, 0))>; |

1289 | def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, |

1290 | Ptr<uint>:$carry, Predicate:$pred), |

1291 | (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b, |

1292 | 0, $pred):$pair, |

1293 | (store (and 1, (lshr (xval $pair, 1), 29)), $carry), |

1294 | (xval $pair, 0))>; |

1295 | } |

1296 | let params = T.Int32 in { |

1297 | defm vadc: vadcsbc; |

1298 | defm vsbc: vadcsbc; |

1299 | } |

1300 | |

1301 | let params = T.Int in { |

1302 | def vshlcq: Intrinsic< |

1303 | Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm), |

1304 | (seq (load $ps):$s, |

1305 | (IRInt<"vshlc", [Vector]> $v, $s, $imm):$pair, |

1306 | (store (xval $pair, 0), $ps), |

1307 | (xval $pair, 1))>; |

1308 | def vshlcq_m: Intrinsic< |

1309 | Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm, Predicate:$pred), |

1310 | (seq (load $ps):$s, |

1311 | (IRInt<"vshlc_predicated", [Vector, Predicate]> |

1312 | $v, $s, $imm, $pred):$pair, |

1313 | (store (xval $pair, 0), $ps), |

1314 | (xval $pair, 1))>; |

1315 | } |

1316 | |

1317 | multiclass VectorComplexAddPred<dag not_halving, dag angle> { |

1318 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), |

1319 | (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>; |

1320 | defm "": IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), |

1321 | (IRInt<"vcaddq_predicated", [Vector, Predicate]> |

1322 | not_halving, angle, $inactive, $a, $b, $pred)>; |

1323 | } |

1324 | |

1325 | multiclass VectorComplexMulPred<dag angle> { |

1326 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), |

1327 | (IRInt<"vcmulq", [Vector]> angle, $a, $b)>; |

1328 | defm "": IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), |

1329 | (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b, |

1330 | $pred)>; |

1331 | } |

1332 | |

1333 | multiclass VectorComplexMLAPred<dag angle> { |

1334 | def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c), |

1335 | (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>; |

1336 | def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c, |

1337 | Predicate:$pred), |

1338 | (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>; |

1339 | } |

1340 | |

1341 | multiclass VectorComplexAddAngle<dag not_halving> { |

1342 | defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>; |

1343 | defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>; |

1344 | } |

1345 | |

1346 | multiclass VectorComplexMulAngle { |

1347 | defm "": VectorComplexMulPred<(u32 0)>; |

1348 | defm _rot90 : VectorComplexMulPred<(u32 1)>; |

1349 | defm _rot180 : VectorComplexMulPred<(u32 2)>; |

1350 | defm _rot270 : VectorComplexMulPred<(u32 3)>; |

1351 | } |

1352 | |

1353 | multiclass VectorComplexMLAAngle { |

1354 | defm "": VectorComplexMLAPred<(u32 0)>; |

1355 | defm _rot90 : VectorComplexMLAPred<(u32 1)>; |

1356 | defm _rot180 : VectorComplexMLAPred<(u32 2)>; |

1357 | defm _rot270 : VectorComplexMLAPred<(u32 3)>; |

1358 | } |

1359 | |

1360 | let params = T.Usual in |

1361 | defm vcaddq : VectorComplexAddAngle<(u32 1)>; |

1362 | |

1363 | let params = T.Signed in |

1364 | defm vhcaddq : VectorComplexAddAngle<(u32 0)>; |

1365 | |

1366 | let params = T.Float in { |

1367 | defm vcmulq : VectorComplexMulAngle; |

1368 | defm vcmlaq : VectorComplexMLAAngle; |

1369 | } |

1370 | |

1371 | multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> { |

1372 | def xsuffix#"q" |

1373 | : Intrinsic<Scalar32, (args Vector:$a, Vector:$b), |

1374 | (IRInt<"vmldava", [Vector]> |

1375 | (unsignedflag Scalar), subtract, exchange, |

1376 | (zeroinit Scalar32), $a, $b)>; |

1377 | def xsuffix#"q_p" |

1378 | : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred), |

1379 | (IRInt<"vmldava_predicated", [Vector, Predicate]> |

1380 | (unsignedflag Scalar), subtract, exchange, |

1381 | (zeroinit Scalar32), $a, $b, $pred)>; |

1382 | |

1383 | def "a"#xsuffix# "q" |

1384 | : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c), |

1385 | (IRInt<"vmldava", [Vector]> |

1386 | (unsignedflag Scalar), subtract, exchange, |

1387 | $a, $b, $c)>; |

1388 | def "a"#xsuffix# "q_p" |

1389 | : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c, |

1390 | Predicate:$pred), |

1391 | (IRInt<"vmldava_predicated", [Vector, Predicate]> |

1392 | (unsignedflag Scalar), subtract, exchange, |

1393 | $a, $b, $c, $pred)>; |

1394 | } |

1395 | |

1396 | class IntrSplit64<Type resty, dag args, dag codegen> |

1397 | : Intrinsic<resty, args, |

1398 | (seq (u32 (lshr $a, (u64 32))):$hi, |

1399 | (u32 $a):$lo, |

1400 | codegen:$pair, |

1401 | (or (shl (u64 (xval $pair, 1)), (u64 32)), |

1402 | (u64 (xval $pair, 0))))>; |

1403 | |

1404 | class IntrSplit64ZeroInit<Type resty, dag args, dag codegen> |

1405 | : Intrinsic<resty, args, |

1406 | (seq (zeroinit u32):$hi, |

1407 | (zeroinit u32):$lo, |

1408 | codegen:$pair, |

1409 | (or (shl (u64 (xval $pair, 1)), (u64 32)), |

1410 | (u64 (xval $pair, 0))))>; |

1411 | |

1412 | multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange, |

1413 | string xsuffix, string irname> { |

1414 | def xsuffix#"q" |

1415 | : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b), |

1416 | (IRInt<irname, [Vector]> |

1417 | (unsignedflag Scalar), subtract, exchange, |

1418 | $lo, $hi, $a, $b)>; |

1419 | def xsuffix#"q_p" |

1420 | : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b, |

1421 | Predicate:$pred), |

1422 | (IRInt<irname#"_predicated", [Vector, Predicate]> |

1423 | (unsignedflag Scalar), subtract, exchange, |

1424 | $lo, $hi, $a, $b, $pred)>; |

1425 | |

1426 | def "a"#xsuffix# "q" |

1427 | : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c), |

1428 | (IRInt<irname, [Vector]> |

1429 | (unsignedflag Scalar), subtract, exchange, |

1430 | $lo, $hi, $b, $c)>; |

1431 | def "a"#xsuffix# "q_p" |

1432 | : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c, |

1433 | Predicate:$pred), |

1434 | (IRInt<irname#"_predicated", [Vector, Predicate]> |

1435 | (unsignedflag Scalar), subtract, exchange, |

1436 | $lo, $hi, $b, $c, $pred)>; |

1437 | } |

1438 | |

1439 | multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> { |

1440 | defm "": MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">; |

1441 | } |

1442 | |

1443 | multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> { |

1444 | defm "": MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, |

1445 | "vrmlldavha">; |

1446 | } |

1447 | |

1448 | multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> { |

1449 | defvar accArg = !if(acc, (args Scalar:$acc), (args)); |

1450 | defvar predArg = !if(pred, (args Predicate:$pred), (args)); |

1451 | defvar intrinsic = !if(pred, |

1452 | IRInt<intbase # "_predicated", [Vector, Predicate]>, |

1453 | IRInt<intbase, [Vector]>); |

1454 | defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)), |

1455 | !if(pred, (? $pred), (?))); |

1456 | defvar accCG = !if(acc, (add intCG, $acc), intCG); |

1457 | |

1458 | def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>; |

1459 | } |

1460 | |

1461 | let params = T.Int in { |

1462 | defm vaddvq : VADDV<0, 0, "addv", Scalar32>; |

1463 | defm vaddvaq : VADDV<1, 0, "addv", Scalar32>; |

1464 | defm vaddvq_p : VADDV<0, 1, "addv", Scalar32>; |

1465 | defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>; |

1466 | } |

1467 | |

1468 | let params = [s32, u32] in { |

1469 | defm vaddlvq : VADDV<0, 0, "addlv", Scalar64>; |

1470 | defm vaddlvaq : VADDV<1, 0, "addlv", Scalar64>; |

1471 | defm vaddlvq_p : VADDV<0, 1, "addlv", Scalar64>; |

1472 | defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>; |

1473 | } |

1474 | |

1475 | let params = T.Int in { |

1476 | def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c), |

1477 | (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>; |

1478 | def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c, |

1479 | Predicate:$pred), |

1480 | (IRInt<"vabav_predicated", [Vector, Predicate]> |

1481 | (unsignedflag Scalar), $a, $b, $c, $pred)>; |

1482 | |

1483 | defm vmladav : MVEBinaryVectorHoriz32<V.False, V.False, "">; |

1484 | } |

1485 | |

1486 | let params = T.Signed in { |

1487 | defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">; |

1488 | defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">; |

1489 | defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">; |

1490 | } |

1491 | |

1492 | let params = [u16, s16, u32, s32] in |

1493 | defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">; |

1494 | |

1495 | let params = [s16, s32] in { |

1496 | defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">; |

1497 | defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">; |

1498 | defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">; |

1499 | } |

1500 | |

1501 | let params = T.Int32 in |

1502 | defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">; |

1503 | |

1504 | let params = [s32] in { |

1505 | defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">; |

1506 | defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">; |

1507 | defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">; |

1508 | } |

1509 | |

1510 | multiclass vrev_predicated<int revsize> { |

1511 | defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), |

1512 | (IRInt<"vrev_predicated", [Vector, Predicate]> |

1513 | $a, revsize, $pred, $inactive)>; |

1514 | } |

1515 | |

1516 | let params = T.All8 in { |

1517 | def vrev16q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 16)>; |

1518 | defm vrev16q: vrev_predicated<16>; |

1519 | } |

1520 | let params = !listconcat(T.All8, T.All16) in { |

1521 | def vrev32q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 32)>; |

1522 | defm vrev32q: vrev_predicated<32>; |

1523 | } |

1524 | let params = T.Usual in { |

1525 | def vrev64q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 64)>; |

1526 | defm vrev64q: vrev_predicated<64>; |

1527 | } |

1528 | |

1529 | foreach desttype = T.All in { |

1530 | // We want a vreinterpretq between every pair of supported vector types |

1531 | // _except_ that there shouldn't be one from a type to itself. |

1532 | // |

1533 | // So this foldl expression implements what you'd write in Python as |

1534 | // [srctype for srctype in T.All if srctype != desttype] |

1535 | let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist, |

1536 | !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype]))) |

1537 | in { |

1538 | def "vreinterpretq_"# desttype: Intrinsic< |

1539 | VecOf<desttype>, (args Vector:$x), (vreinterpret $x, VecOf<desttype>)>; |

1540 | } |

1541 | } |

1542 | |

1543 | let params = T.All in { |

1544 | let pnt = PNT_None in { |

1545 | def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b), |

1546 | (vreinterpret (ielt_const (ielt_const (undef VecOf<u64>), $a, 0), |

1547 | $b, 1), Vector)>; |

1548 | def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>; |

1549 | } |

1550 | |

1551 | // This is the polymorphic form of vuninitializedq, which takes no type |

1552 | // suffix, but takes an _unevaluated_ vector parameter and returns an |

1553 | // uninitialized vector of the same vector type. |

1554 | // |

1555 | // This intrinsic has no _non_-polymorphic form exposed to the user. But each |

1556 | // separately typed version of it still has to have its own clang builtin id, |

1557 | // which can't be called vuninitializedq_u32 or similar because that would |

1558 | // collide with the explicit nullary versions above. So I'm calling them |

1559 | // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that |

1560 | // full name never appears in the header file due to the polymorphicOnly |

1561 | // flag, and the _polymorphic suffix is omitted from the shortened name by |

1562 | // the custom PolymorphicNameType here. |

1563 | let polymorphicOnly = 1, nonEvaluating = 1, |

1564 | pnt = PolymorphicNameType<1, "polymorphic"> in { |

1565 | def vuninitializedq_polymorphic: Intrinsic< |

1566 | Vector, (args Vector), (undef Vector)>; |

1567 | } |

1568 | |

1569 | def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane), |

1570 | (xelt_var $v, $lane)>; |

1571 | def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane), |

1572 | (ielt_var $v, $e, $lane)>; |

1573 | } |

1574 | |

1575 | foreach desttype = !listconcat(T.Int16, T.Int32, T.Float) in { |

1576 | defvar is_dest_float = !eq(desttype.kind, "f"); |

1577 | defvar is_dest_unsigned = !eq(desttype.kind, "u"); |

1578 | // First immediate operand of the LLVM intrinsic |

1579 | defvar unsigned_flag = !if(is_dest_float, (unsignedflag Scalar), |

1580 | !if(is_dest_unsigned, V.True, V.False)); |

1581 | // For float->int conversions _n and _x_n intrinsics are not polymorphic |

1582 | // because the signedness of the destination type cannot be inferred. |

1583 | defvar pnt_nx = !if(is_dest_float, PNT_2Type, PNT_None); |

1584 | |

1585 | let params = !if(is_dest_float, |

1586 | !if(!eq(desttype.size, 16), T.Int16, T.Int32), |

1587 | !if(!eq(desttype.size, 16), [f16], [f32])) in { |

1588 | let pnt = pnt_nx in |

1589 | def "vcvtq_n_"#desttype : Intrinsic<VecOf<desttype>, |

1590 | (args Vector:$a, imm_1toN:$b), |

1591 | (IRInt<"vcvt_fix", [VecOf<desttype>, Vector]> unsigned_flag, $a, $b)>; |

1592 | |

1593 | defm "vcvtq": IntrinsicMX<VecOf<desttype>, |

1594 | (args Vector:$a, imm_1toN:$b, Predicate:$p), |

1595 | (IRInt<"vcvt_fix_predicated", [VecOf<desttype>, Vector, Predicate]> |

1596 | unsigned_flag, $inactive, $a, $b, $p), |

1597 | 1, "_n_"#desttype, PNT_2Type, pnt_nx>; |

1598 | } |

1599 | } |

1600 | |

1601 | let params = T.Usual in { |

1602 | let pnt = PNT_NType in |

1603 | def vbrsrq_n: Intrinsic<Vector, (args Vector:$a, s32:$b), |

1604 | (IRInt<"vbrsr", [Vector]> $a, $b)>; |

1605 | defm vbrsrq : IntrinsicMX<Vector, (args Vector:$a, s32:$b, Predicate:$pred), |

1606 | (IRInt<"vbrsr_predicated", [Vector, Predicate]> |

1607 | $inactive, $a, $b, $pred), 1, "_n", |

1608 | PNT_NType, PNT_NType>; |

1609 | } |

1610 |