firefox-main/js/src/jit/riscv64/MacroAssembler-riscv64.cpp

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Copyright 2021 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "jit/riscv64/MacroAssembler-riscv64.h"
#include <bit>
#include "jit/Bailouts.h"
#include "jit/BaselineFrame.h"
#include "jit/JitFrames.h"
#include "jit/JitRuntime.h"
#include "jit/MacroAssembler.h"
#include "jit/MoveEmitter.h"
#include "util/Memory.h"
#include "util/PortableMath.h"
#include "vm/JitActivation.h" // jit::JitActivation
#include "vm/JSContext.h"
#include "wasm/WasmStubs.h"
#include "jit/MacroAssembler-inl.h"
namespace js {
namespace jit {
MacroAssembler& MacroAssemblerRiscv64::asMasm() {
return *static_cast<MacroAssembler*>(this);
}
const MacroAssembler& MacroAssemblerRiscv64::asMasm() const {
return *static_cast<const MacroAssembler*>(this);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Register lhs, ImmWord imm,
Condition c) {
if (imm.value <= INT32_MAX) {
ma_cmp_set(dst, lhs, Imm32(uint32_t(imm.value)), c);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
ma_cmp_set(dst, lhs, scratch, c);
}
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Register lhs, ImmPtr imm,
Condition c) {
ma_cmp_set(dst, lhs, ImmWord(uintptr_t(imm.value)), c);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Register lhs, ImmGCPtr imm,
Condition c) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
ma_cmp_set(dst, lhs, scratch, c);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Address address,
Register rhs, Condition c) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_load(scratch2, address, SizeDouble);
ma_cmp_set(dst, Register(scratch2), rhs, c);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Address address, Imm32 imm,
Condition c) {
// TODO(riscv): 32-bit ma_cmp_set?
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_load(scratch2, address, SizeWord);
ma_cmp_set(dst, Register(scratch2), imm, c);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Address address,
ImmWord imm, Condition c) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_load(scratch2, address, SizeDouble);
ma_cmp_set(dst, Register(scratch2), imm, c);
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Register lhs, Imm32 imm,
Condition c) {
if (imm.value == 0) {
switch (c) {
case Equal:
case BelowOrEqual:
ma_sltu(dst, lhs, Operand(1));
break;
case NotEqual:
case Above:
sltu(dst, zero, lhs);
break;
case AboveOrEqual:
case Below:
ori(dst, zero, c == AboveOrEqual ? 1 : 0);
break;
case GreaterThan:
case LessThanOrEqual:
slt(dst, zero, lhs);
if (c == LessThanOrEqual) {
xori(dst, dst, 1);
}
break;
case LessThan:
case GreaterThanOrEqual:
slt(dst, lhs, zero);
if (c == GreaterThanOrEqual) {
xori(dst, dst, 1);
}
break;
case Zero:
ma_sltu(dst, lhs, Operand(1));
break;
case NonZero:
sltu(dst, zero, lhs);
break;
case Signed:
slt(dst, lhs, zero);
break;
case NotSigned:
slt(dst, lhs, zero);
xori(dst, dst, 1);
break;
default:
MOZ_CRASH("Invalid condition.");
}
return;
}
switch (c) {
case Equal:
case NotEqual:
ma_xor(dst, lhs, imm);
if (c == Equal) {
ma_sltu(dst, dst, Operand(1));
} else {
sltu(dst, zero, dst);
}
break;
case Zero:
case NonZero:
case Signed:
case NotSigned:
MOZ_CRASH("Invalid condition.");
default:
Condition cond = ma_cmp(dst, lhs, imm, c);
MOZ_ASSERT(cond == Equal || cond == NotEqual);
if (cond == Equal) xori(dst, dst, 1);
}
}
Assembler::Condition MacroAssemblerRiscv64::ma_cmp(Register rd, Register lhs,
Register rhs, Condition c) {
switch (c) {
case Above:
// bgtu s,t,label =>
// sltu at,t,s
// bne at,$zero,offs
sltu(rd, rhs, lhs);
return NotEqual;
case AboveOrEqual:
// bgeu s,t,label =>
// sltu at,s,t
// beq at,$zero,offs
sltu(rd, lhs, rhs);
return Equal;
case Below:
// bltu s,t,label =>
// sltu at,s,t
// bne at,$zero,offs
sltu(rd, lhs, rhs);
return NotEqual;
case BelowOrEqual:
// bleu s,t,label =>
// sltu at,t,s
// beq at,$zero,offs
sltu(rd, rhs, lhs);
return Equal;
case GreaterThan:
// bgt s,t,label =>
// slt at,t,s
// bne at,$zero,offs
slt(rd, rhs, lhs);
return NotEqual;
case GreaterThanOrEqual:
// bge s,t,label =>
// slt at,s,t
// beq at,$zero,offs
slt(rd, lhs, rhs);
return Equal;
case LessThan:
// blt s,t,label =>
// slt at,s,t
// bne at,$zero,offs
slt(rd, lhs, rhs);
return NotEqual;
case LessThanOrEqual:
// ble s,t,label =>
// slt at,t,s
// beq at,$zero,offs
slt(rd, rhs, lhs);
return Equal;
default:
MOZ_CRASH("Invalid condition.");
}
return Always;
}
Assembler::Condition MacroAssemblerRiscv64::ma_cmp(Register rd, Register lhs,
Imm32 imm, Condition c) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_RELEASE_ASSERT(lhs != scratch);
switch (c) {
case Above:
case BelowOrEqual:
if (imm.value != 0x7fffffff && is_intn(imm.value + 1, 12) &&
imm.value != -1) {
// lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow
ma_sltu(rd, lhs, Operand(imm.value + 1));
return (c == BelowOrEqual ? NotEqual : Equal);
} else {
ma_li(scratch, imm);
sltu(rd, scratch, lhs);
return (c == BelowOrEqual ? Equal : NotEqual);
}
case AboveOrEqual:
case Below:
if (is_intn(imm.value, 12)) {
ma_sltu(rd, lhs, Operand(imm.value));
} else {
ma_li(scratch, imm);
sltu(rd, lhs, scratch);
}
return (c == AboveOrEqual ? Equal : NotEqual);
case GreaterThan:
case LessThanOrEqual:
if (imm.value != 0x7fffffff && is_intn(imm.value + 1, 12)) {
// lhs <= rhs via lhs < rhs + 1.
ma_slt(rd, lhs, Operand(imm.value + 1));
return (c == LessThanOrEqual ? NotEqual : Equal);
} else {
ma_li(scratch, imm);
slt(rd, scratch, lhs);
return (c == LessThanOrEqual ? Equal : NotEqual);
}
case GreaterThanOrEqual:
case LessThan:
if (is_intn(imm.value, 12)) {
ma_slt(rd, lhs, imm);
} else {
ma_li(scratch, imm);
slt(rd, lhs, scratch);
}
return (c == GreaterThanOrEqual ? Equal : NotEqual);
default:
MOZ_CRASH("Invalid condition.");
}
return Always;
}
void MacroAssemblerRiscv64::ma_cmp_set(Register dst, Register lhs, Register rhs,
Condition c) {
switch (c) {
case Equal:
// seq d,s,t =>
// xor d,s,t
// sltiu d,d,1
xor_(dst, lhs, rhs);
ma_sltu(dst, dst, Operand(1));
break;
case NotEqual:
// sne d,s,t =>
// xor d,s,t
// sltu d,$zero,d
xor_(dst, lhs, rhs);
sltu(dst, zero, dst);
break;
case Above:
// sgtu d,s,t =>
// sltu d,t,s
sltu(dst, rhs, lhs);
break;
case AboveOrEqual:
// sgeu d,s,t =>
// sltu d,s,t
// xori d,d,1
sltu(dst, lhs, rhs);
xori(dst, dst, 1);
break;
case Below:
// sltu d,s,t
sltu(dst, lhs, rhs);
break;
case BelowOrEqual:
// sleu d,s,t =>
// sltu d,t,s
// xori d,d,1
sltu(dst, rhs, lhs);
xori(dst, dst, 1);
break;
case GreaterThan:
// sgt d,s,t =>
// slt d,t,s
slt(dst, rhs, lhs);
break;
case GreaterThanOrEqual:
// sge d,s,t =>
// slt d,s,t
// xori d,d,1
slt(dst, lhs, rhs);
xori(dst, dst, 1);
break;
case LessThan:
// slt d,s,t
slt(dst, lhs, rhs);
break;
case LessThanOrEqual:
// sle d,s,t =>
// slt d,t,s
// xori d,d,1
slt(dst, rhs, lhs);
xori(dst, dst, 1);
break;
case Zero:
MOZ_ASSERT(lhs == rhs);
// seq d,s,$zero =>
// sltiu d,s,1
ma_sltu(dst, lhs, Operand(1));
break;
case NonZero:
MOZ_ASSERT(lhs == rhs);
// sne d,s,$zero =>
// sltu d,$zero,s
sltu(dst, zero, lhs);
break;
case Signed:
MOZ_ASSERT(lhs == rhs);
slt(dst, lhs, zero);
break;
case NotSigned:
MOZ_ASSERT(lhs == rhs);
// sge d,s,$zero =>
// slt d,s,$zero
// xori d,d,1
slt(dst, lhs, zero);
xori(dst, dst, 1);
break;
default:
MOZ_CRASH("Invalid condition.");
}
}
void MacroAssemblerRiscv64::ma_compareF32(Register rd, DoubleCondition cc,
FloatRegister cmp1,
FloatRegister cmp2) {
switch (cc) {
case DoubleEqual:
feq_s(rd, cmp1, cmp2);
return;
case DoubleEqualOrUnordered: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
flt_s(rd, cmp1, cmp2);
flt_s(scratch, cmp2, cmp1);
or_(rd, rd, scratch);
NegateBool(rd, rd);
return;
}
case DoubleNotEqual: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
flt_s(rd, cmp1, cmp2);
flt_s(scratch, cmp2, cmp1);
or_(rd, rd, scratch);
return;
}
case DoubleNotEqualOrUnordered:
feq_s(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleLessThan:
flt_s(rd, cmp1, cmp2);
return;
case DoubleLessThanOrUnordered:
fle_s(rd, cmp2, cmp1);
NegateBool(rd, rd);
return;
case DoubleGreaterThanOrEqual:
fle_s(rd, cmp2, cmp1);
return;
case DoubleGreaterThanOrEqualOrUnordered:
flt_s(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleLessThanOrEqual:
fle_s(rd, cmp1, cmp2);
return;
case DoubleLessThanOrEqualOrUnordered:
flt_s(rd, cmp2, cmp1);
NegateBool(rd, rd);
return;
case DoubleGreaterThan:
flt_s(rd, cmp2, cmp1);
return;
case DoubleGreaterThanOrUnordered:
fle_s(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleOrdered:
CompareIsNotNanF32(rd, cmp1, cmp2);
return;
case DoubleUnordered:
CompareIsNanF32(rd, cmp1, cmp2);
return;
}
}
void MacroAssemblerRiscv64::ma_compareF64(Register rd, DoubleCondition cc,
FloatRegister cmp1,
FloatRegister cmp2) {
switch (cc) {
case DoubleEqual:
feq_d(rd, cmp1, cmp2);
return;
case DoubleEqualOrUnordered: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
flt_d(rd, cmp1, cmp2);
flt_d(scratch, cmp2, cmp1);
or_(rd, rd, scratch);
NegateBool(rd, rd);
return;
}
case DoubleNotEqual: {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
flt_d(rd, cmp1, cmp2);
flt_d(scratch, cmp2, cmp1);
or_(rd, rd, scratch);
return;
}
case DoubleNotEqualOrUnordered:
feq_d(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleLessThan:
flt_d(rd, cmp1, cmp2);
return;
case DoubleLessThanOrUnordered:
fle_d(rd, cmp2, cmp1);
NegateBool(rd, rd);
return;
case DoubleGreaterThanOrEqual:
fle_d(rd, cmp2, cmp1);
return;
case DoubleGreaterThanOrEqualOrUnordered:
flt_d(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleLessThanOrEqual:
fle_d(rd, cmp1, cmp2);
return;
case DoubleLessThanOrEqualOrUnordered:
flt_d(rd, cmp2, cmp1);
NegateBool(rd, rd);
return;
case DoubleGreaterThan:
flt_d(rd, cmp2, cmp1);
return;
case DoubleGreaterThanOrUnordered:
fle_d(rd, cmp1, cmp2);
NegateBool(rd, rd);
return;
case DoubleOrdered:
CompareIsNotNanF64(rd, cmp1, cmp2);
return;
case DoubleUnordered:
CompareIsNanF64(rd, cmp1, cmp2);
return;
}
}
void MacroAssemblerRiscv64Compat::movePtr(Register src, Register dest) {
mv(dest, src);
}
void MacroAssemblerRiscv64Compat::movePtr(ImmWord imm, Register dest) {
ma_li(dest, imm);
}
void MacroAssemblerRiscv64Compat::movePtr(ImmGCPtr imm, Register dest) {
ma_li(dest, imm);
}
void MacroAssemblerRiscv64Compat::movePtr(ImmPtr imm, Register dest) {
movePtr(ImmWord(uintptr_t(imm.value)), dest);
}
void MacroAssemblerRiscv64Compat::movePtr(wasm::SymbolicAddress imm,
Register dest) {
DEBUG_PRINTF("[ %s\n", __FUNCTION__);
BlockTrampolinePoolScope block_trampoline_pool(this, 8);
append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
ma_liPatchable(dest, ImmWord(-1), Li64);
DEBUG_PRINTF("]\n");
}
bool MacroAssemblerRiscv64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
asMasm().Push(FrameDescriptor(FrameType::IonJS)); // descriptor_
asMasm().Push(ImmPtr(fakeReturnAddr));
asMasm().Push(FramePointer);
return true;
}
void MacroAssemblerRiscv64Compat::convertUInt32ToDouble(Register src,
FloatRegister dest) {
fcvt_d_wu(dest, src);
}
void MacroAssemblerRiscv64Compat::convertUInt64ToDouble(Register src,
FloatRegister dest) {
fcvt_d_lu(dest, src);
}
void MacroAssemblerRiscv64Compat::convertUInt32ToFloat32(Register src,
FloatRegister dest) {
fcvt_s_wu(dest, src);
}
void MacroAssemblerRiscv64Compat::convertDoubleToFloat32(FloatRegister src,
FloatRegister dest) {
fcvt_s_d(dest, src);
}
void MacroAssemblerRiscv64Compat::minMax32(Register lhs, Register rhs,
Register dest, bool isMax) {
if (rhs == dest) {
std::swap(lhs, rhs);
}
if (HasZbbExtension()) {
UseScratchRegisterScope temps(this);
const Register rhsSExt = temps.Acquire();
move32(rhs, rhsSExt);
move32(lhs, dest);
// Using signed max/min to match the (signed)
// Assembler::GreaterThan/Assembler::LessThan below
if (isMax) {
max(dest, dest, rhsSExt);
} else {
min(dest, dest, rhsSExt);
}
return;
}
auto cond = isMax ? Assembler::GreaterThan : Assembler::LessThan;
if (lhs != dest) {
move32(lhs, dest);
}
asMasm().cmp32Move32(cond, rhs, lhs, rhs, dest);
}
void MacroAssemblerRiscv64Compat::minMax32(Register lhs, Imm32 rhs,
Register dest, bool isMax) {
if (HasZbbExtension()) {
UseScratchRegisterScope temps(this);
Register realRhs;
if (rhs.value == 0) {
realRhs = zero;
} else {
realRhs = temps.Acquire();
ma_li(realRhs, rhs);
}
// Using signed max/min to match the (signed)
// Assembler::GreaterThan/Assembler::LessThan below
move32(lhs, dest);
if (isMax) {
max(dest, dest, realRhs);
} else {
min(dest, dest, realRhs);
}
return;
}
if (rhs.value == 0) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (isMax) {
// dest = -(lhs > 0 ? 1 : 0) & lhs
sgtz(scratch, lhs);
neg(scratch, scratch);
and_(dest, lhs, scratch);
} else {
// dest = (lhs >> 31) & lhs
sraiw(scratch, lhs, 31);
and_(dest, lhs, scratch);
}
return;
}
// Uses branches because "Zicond" extension isn't yet supported.
auto cond =
isMax ? Assembler::GreaterThanOrEqual : Assembler::LessThanOrEqual;
if (lhs != dest) {
move32(lhs, dest);
}
Label done;
asMasm().branch32(cond, lhs, rhs, &done);
move32(rhs, dest);
bind(&done);
}
void MacroAssemblerRiscv64Compat::minMaxPtr(Register lhs, Register rhs,
Register dest, bool isMax) {
if (HasZbbExtension()) {
// Using signed max/min to match the (signed)
// Assembler::GreaterThan/Assembler::LessThan below
if (isMax) {
max(dest, lhs, rhs);
} else {
min(dest, lhs, rhs);
}
return;
}
if (rhs == dest) {
std::swap(lhs, rhs);
}
auto cond = isMax ? Assembler::GreaterThan : Assembler::LessThan;
if (lhs != dest) {
movePtr(lhs, dest);
}
asMasm().cmpPtrMovePtr(cond, rhs, lhs, rhs, dest);
}
void MacroAssemblerRiscv64Compat::minMaxPtr(Register lhs, ImmWord rhs,
Register dest, bool isMax) {
if (HasZbbExtension()) {
UseScratchRegisterScope temps(this);
Register realRhs;
if (rhs.value == 0) {
realRhs = zero;
} else {
realRhs = temps.Acquire();
ma_li(realRhs, rhs);
}
// Using signed max/min to match the (signed)
// Assembler::GreaterThan/Assembler::LessThan below
if (isMax) {
max(dest, lhs, realRhs);
} else {
min(dest, lhs, realRhs);
}
return;
}
if (rhs.value == 0) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (isMax) {
// dest = -(lhs > 0 ? 1 : 0) & lhs
sgtz(scratch, lhs);
neg(scratch, scratch);
and_(dest, lhs, scratch);
} else {
// dest = (lhs >> 63) & lhs
srai(scratch, lhs, 63);
and_(dest, lhs, scratch);
}
return;
}
// Uses branches because "Zicond" extension isn't yet supported.
auto cond =
isMax ? Assembler::GreaterThanOrEqual : Assembler::LessThanOrEqual;
if (lhs != dest) {
movePtr(lhs, dest);
}
Label done;
asMasm().branchPtr(cond, lhs, rhs, &done);
movePtr(rhs, dest);
bind(&done);
}
template <typename F>
void MacroAssemblerRiscv64::RoundHelper(FPURegister dst, FPURegister src,
FPURegister fpu_scratch,
FPURoundingMode mode) {
BlockTrampolinePoolScope block_trampoline_pool(this, 20, 2);
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
MOZ_ASSERT((std::is_same<float, F>::value) ||
(std::is_same<double, F>::value));
// Need at least two FPRs, so check against dst == src == fpu_scratch
MOZ_ASSERT(!(dst == src && dst == fpu_scratch));
const int kFloatMantissaBits =
sizeof(F) == 4 ? kFloat32MantissaBits : kFloat64MantissaBits;
const int kFloatExponentBits =
sizeof(F) == 4 ? kFloat32ExponentBits : kFloat64ExponentBits;
const int kFloatExponentBias =
sizeof(F) == 4 ? kFloat32ExponentBias : kFloat64ExponentBias;
Label done;
{
UseScratchRegisterScope temps2(this);
Register scratch = temps2.Acquire();
// extract exponent value of the source floating-point to scratch
if (std::is_same<F, double>::value) {
fmv_x_d(scratch, src);
} else {
fmv_x_w(scratch, src);
}
ExtractBits(scratch2, scratch, kFloatMantissaBits, kFloatExponentBits);
}
// if src is NaN/+-Infinity/+-Zero or if the exponent is larger than # of bits
// in mantissa, the result is the same as src, so move src to dest (to avoid
// generating another branch)
if (dst != src) {
if (std::is_same<F, double>::value) {
fmv_d(dst, src);
} else {
fmv_s(dst, src);
}
}
{
Label not_NaN;
UseScratchRegisterScope temps2(this);
Register scratch = temps2.Acquire();
// According to the wasm spec
// if input is canonical NaN, then output is canonical NaN, and if input is
// any other NaN, then output is any NaN with most significant bit of
// payload is 1. In RISC-V, feq_d will set scratch to 0 if src is a NaN. If
// src is not a NaN, branch to the label and do nothing, but if it is,
// fmin_d will set dst to the canonical NaN.
if (std::is_same<F, double>::value) {
feq_d(scratch, src, src);
bnez(scratch, &not_NaN);
fmin_d(dst, src, src);
} else {
feq_s(scratch, src, src);
bnez(scratch, &not_NaN);
fmin_s(dst, src, src);
}
bind(&not_NaN);
}
// If real exponent (i.e., scratch2 - kFloatExponentBias) is greater than
// kFloat32MantissaBits, it means the floating-point value has no fractional
// part, thus the input is already rounded, jump to done. Note that, NaN and
// Infinity in floating-point representation sets maximal exponent value, so
// they also satisfy (scratch2 - kFloatExponentBias >= kFloatMantissaBits),
// and JS round semantics specify that rounding of NaN (Infinity) returns NaN
// (Infinity), so NaN and Infinity are considered rounded value too.
ma_branch(&done, GreaterThanOrEqual, scratch2,
Operand(kFloatExponentBias + kFloatMantissaBits));
// Actual rounding is needed along this path
// old_src holds the original input, needed for the case of src == dst
FPURegister old_src = src;
if (src == dst) {
MOZ_ASSERT(fpu_scratch != dst);
fmv_d(fpu_scratch, src);
old_src = fpu_scratch;
}
// Since only input whose real exponent value is less than kMantissaBits
// (i.e., 23 or 52-bits) falls into this path, the value range of the input
// falls into that of 23- or 53-bit integers. So we round the input to integer
// values, then convert them back to floating-point.
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (std::is_same<F, double>::value) {
fcvt_l_d(scratch, src, mode);
fcvt_d_l(dst, scratch, mode);
} else {
fcvt_w_s(scratch, src, mode);
fcvt_s_w(dst, scratch, mode);
}
}
// A special handling is needed if the input is a very small positive/negative
// number that rounds to zero. JS semantics requires that the rounded result
// retains the sign of the input, so a very small positive (negative)
// floating-point number should be rounded to positive (negative) 0.
// Therefore, we use sign-bit injection to produce +/-0 correctly. Instead of
// testing for zero w/ a branch, we just insert sign-bit for everyone on this
// path (this is where old_src is needed)
if (std::is_same<F, double>::value) {
fsgnj_d(dst, dst, old_src);
} else {
fsgnj_s(dst, dst, old_src);
}
bind(&done);
}
template <typename CvtFunc>
void MacroAssemblerRiscv64::RoundFloatingPointToInteger(Register rd,
FPURegister fs,
Register result,
CvtFunc fcvt_generator,
bool Inexact) {
// Save csr_fflags to scratch & clear exception flags
if (result != Register::Invalid()) {
BlockTrampolinePoolScope block_trampoline_pool(this, 6);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
int exception_flags = kInvalidOperation;
if (Inexact) exception_flags |= kInexact;
csrrci(scratch, csr_fflags, exception_flags);
// actual conversion instruction
fcvt_generator(this, rd, fs);
// check kInvalidOperation flag (out-of-range, NaN)
// set result to 1 if normal, otherwise set result to 0 for abnormal
frflags(result);
andi(result, result, exception_flags);
seqz(result, result); // result <-- 1 (normal), result <-- 0 (abnormal)
// restore csr_fflags
csrw(csr_fflags, scratch);
} else {
// actual conversion instruction
fcvt_generator(this, rd, fs);
}
}
void MacroAssemblerRiscv64::Trunc_uw_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_wu_d(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_w_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_d(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_uw_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_wu_s(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_w_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_s(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_ul_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_lu_d(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_l_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_d(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_ul_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_lu_s(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Trunc_l_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_s(dst, src, RTZ);
},
Inexact);
}
void MacroAssemblerRiscv64::Floor_d_d(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<double>(fd, fs, fpu_scratch, RDN);
}
void MacroAssemblerRiscv64::Ceil_d_d(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<double>(fd, fs, fpu_scratch, RUP);
}
void MacroAssemblerRiscv64::Trunc_d_d(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<double>(fd, fs, fpu_scratch, RTZ);
}
void MacroAssemblerRiscv64::Round_d_d(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<double>(fd, fs, fpu_scratch, RNE);
}
void MacroAssemblerRiscv64::Floor_s_s(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<float>(fd, fs, fpu_scratch, RDN);
}
void MacroAssemblerRiscv64::Ceil_s_s(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<float>(fd, fs, fpu_scratch, RUP);
}
void MacroAssemblerRiscv64::Trunc_s_s(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<float>(fd, fs, fpu_scratch, RTZ);
}
void MacroAssemblerRiscv64::Round_s_s(FPURegister fd, FPURegister fs,
FPURegister fpu_scratch) {
RoundHelper<float>(fd, fs, fpu_scratch, RNE);
}
void MacroAssemblerRiscv64::Round_w_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_s(dst, src, RNE);
},
Inexact);
}
void MacroAssemblerRiscv64::Round_w_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_d(dst, src, RNE);
},
Inexact);
}
void MacroAssemblerRiscv64::Ceil_w_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_s(dst, src, RUP);
},
Inexact);
}
void MacroAssemblerRiscv64::Ceil_l_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_d(dst, src, RUP);
},
Inexact);
}
void MacroAssemblerRiscv64::Ceil_l_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_s(dst, src, RUP);
},
Inexact);
}
void MacroAssemblerRiscv64::Ceil_w_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_d(dst, src, RUP);
},
Inexact);
}
void MacroAssemblerRiscv64::Floor_w_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_s(dst, src, RDN);
},
Inexact);
}
void MacroAssemblerRiscv64::Floor_w_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_w_d(dst, src, RDN);
},
Inexact);
}
void MacroAssemblerRiscv64::Floor_l_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_s(dst, src, RDN);
},
Inexact);
}
void MacroAssemblerRiscv64::Floor_l_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_d(dst, src, RDN);
},
Inexact);
}
void MacroAssemblerRiscv64::RoundMaxMag_l_s(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_s(dst, src, RMM);
},
Inexact);
}
void MacroAssemblerRiscv64::RoundMaxMag_l_d(Register rd, FPURegister fs,
Register result, bool Inexact) {
RoundFloatingPointToInteger(
rd, fs, result,
[](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
masm->fcvt_l_d(dst, src, RMM);
},
Inexact);
}
// Checks whether a double is representable as a 32-bit integer. If so, the
// integer is written to the output register. Otherwise, a bailout is taken to
// the given snapshot. This function overwrites the scratch float register.
void MacroAssemblerRiscv64Compat::convertDoubleToInt32(FloatRegister src,
Register dest,
Label* fail,
bool negativeZeroCheck) {
if (negativeZeroCheck) {
fclass_d(dest, src);
ma_b(dest, Imm32(kNegativeZero), fail, Equal);
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_w_d(dest, src, scratch, true);
ma_b(scratch, Imm32(0), fail, Equal);
}
void MacroAssemblerRiscv64Compat::convertDoubleToPtr(FloatRegister src,
Register dest, Label* fail,
bool negativeZeroCheck) {
if (negativeZeroCheck) {
fclass_d(dest, src);
ma_b(dest, Imm32(kNegativeZero), fail, Equal);
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_l_d(dest, src, scratch, true);
ma_b(scratch, Imm32(0), fail, Equal);
}
// Checks whether a float32 is representable as a 32-bit integer. If so, the
// integer is written to the output register. Otherwise, a bailout is taken to
// the given snapshot. This function overwrites the scratch float register.
void MacroAssemblerRiscv64Compat::convertFloat32ToInt32(
FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) {
if (negativeZeroCheck) {
fclass_d(dest, src);
ma_b(dest, Imm32(kNegativeZero), fail, Equal);
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_w_s(dest, src, scratch, true);
ma_b(scratch, Imm32(0), fail, Equal);
}
void MacroAssemblerRiscv64Compat::convertFloat32ToDouble(FloatRegister src,
FloatRegister dest) {
fcvt_d_s(dest, src);
}
void MacroAssemblerRiscv64Compat::convertInt32ToFloat32(Register src,
FloatRegister dest) {
fcvt_s_w(dest, src);
}
void MacroAssemblerRiscv64Compat::convertInt32ToFloat32(const Address& src,
FloatRegister dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
load32(src, scratch);
fcvt_s_w(dest, scratch);
}
void MacroAssemblerRiscv64Compat::truncateFloat32ModUint32(FloatRegister src,
Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Convert scalar to signed 64-bit fixed-point, rounding toward zero.
// In the case of overflow or NaN, the output is saturated.
// In the case of -0, the output is zero.
Trunc_l_s(dest, src);
// Unsigned subtraction of INT64_MAX returns 1 resp. 0 for INT64_{MIN,MAX}.
ma_li(scratch, Imm64(0x7fff'ffff'ffff'ffff));
ma_sub64(scratch, dest, scratch);
// If scratch u< 2, then scratch = 0; else scratch = -1.
ma_sltu(scratch, scratch, Imm32(2));
ma_add32(scratch, scratch, Imm32(-1));
// Clear |dest| if the truncation result was saturated.
ma_and(dest, dest, scratch);
// Clear upper 32 bits.
SignExtendWord(dest, dest);
}
// Memory.
FaultingCodeOffset MacroAssemblerRiscv64::ma_loadDouble(FloatRegister dest,
Address address) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
fld(dest, base, encodedOffset);
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_loadFloat(FloatRegister dest,
Address address) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
flw(dest, base, encodedOffset);
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_load(
Register dest, Address address, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
switch (size) {
case SizeByte:
if (ZeroExtend == extension) {
lbu(dest, base, encodedOffset);
} else {
lb(dest, base, encodedOffset);
}
break;
case SizeHalfWord:
if (ZeroExtend == extension) {
lhu(dest, base, encodedOffset);
} else {
lh(dest, base, encodedOffset);
}
break;
case SizeWord:
if (ZeroExtend == extension) {
lwu(dest, base, encodedOffset);
} else {
lw(dest, base, encodedOffset);
}
break;
case SizeDouble:
ld(dest, base, encodedOffset);
break;
default:
MOZ_CRASH("Invalid argument for ma_load");
}
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
Register data, const BaseIndex& dest, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
computeScaledAddress(dest, scratch2);
return ma_store(data, Address(scratch2, dest.offset), size, extension);
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
Imm32 imm, const BaseIndex& dest, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
Register address = temps.Acquire();
computeScaledAddress(dest, address);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
return ma_store(scratch, Address(address, dest.offset), size, extension);
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
Imm32 imm, Address address, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
return ma_store(scratch, address, size, extension);
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
Register data, Address address, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
switch (size) {
case SizeByte:
sb(data, base, encodedOffset);
break;
case SizeHalfWord:
sh(data, base, encodedOffset);
break;
case SizeWord:
sw(data, base, encodedOffset);
break;
case SizeDouble:
sd(data, base, encodedOffset);
break;
default:
MOZ_CRASH("Invalid argument for ma_store");
}
return fco;
}
// Memory.
void MacroAssemblerRiscv64::ma_storeDouble(FloatRegister dest,
Address address) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
fsd(dest, base, encodedOffset);
}
void MacroAssemblerRiscv64::ma_storeFloat(FloatRegister dest, Address address) {
UseScratchRegisterScope temps(this);
int16_t encodedOffset;
Register base;
if (!is_int12(address.offset)) {
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(address.offset));
add(scratch, address.base, scratch);
base = scratch;
encodedOffset = 0;
} else {
encodedOffset = address.offset;
base = address.base;
}
fsw(dest, base, encodedOffset);
}
void MacroAssemblerRiscv64::computeScaledAddress(const BaseIndex& address,
Register dest) {
Register base = address.base;
Register index = address.index;
int32_t shift = Imm32::ShiftOf(address.scale).value;
UseScratchRegisterScope temps(this);
if (shift && base == zero) {
MOZ_ASSERT(shift <= 4);
slli(dest, index, shift);
} else if (shift) {
MOZ_ASSERT(shift <= 4);
if (HasZbaExtension()) {
switch (shift) {
case 1:
sh1add(dest, index, base);
return;
case 2:
sh2add(dest, index, base);
return;
case 3:
sh3add(dest, index, base);
return;
default:
break;
}
}
Register tmp = dest == base ? temps.Acquire() : dest;
slli(tmp, index, shift);
add(dest, base, tmp);
} else {
add(dest, base, index);
}
}
void MacroAssemblerRiscv64::computeScaledAddress32(const BaseIndex& address,
Register dest) {
Register base = address.base;
Register index = address.index;
int32_t shift = Imm32::ShiftOf(address.scale).value;
UseScratchRegisterScope temps(this);
if (shift && base == zero) {
MOZ_ASSERT(shift <= 4);
slliw(dest, index, shift);
} else if (shift) {
MOZ_ASSERT(shift <= 4);
if (HasZbaExtension()) {
switch (shift) {
case 1:
sh1add_uw(dest, index, base);
return;
case 2:
sh2add_uw(dest, index, base);
return;
case 3:
sh3add_uw(dest, index, base);
return;
default:
break;
}
}
Register tmp = dest == base ? temps.Acquire() : dest;
slliw(tmp, index, shift);
addw(dest, base, tmp);
} else {
addw(dest, base, index);
}
}
void MacroAssemblerRiscv64Compat::wasmLoadI64Impl(
const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
Register ptrScratch, Register64 output, Register tmp) {
access.assertOffsetInGuardPages();
uint32_t offset = access.offset32();
MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
// Maybe add the offset.
if (offset) {
asMasm().addPtr(ImmWord(offset), ptrScratch);
ptr = ptrScratch;
}
asMasm().memoryBarrierBefore(access.sync());
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
FaultingCodeOffset fco;
switch (access.type()) {
case Scalar::Int8:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lb(output.reg, scratch, 0);
break;
case Scalar::Uint8:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lbu(output.reg, scratch, 0);
break;
case Scalar::Int16:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lh(output.reg, scratch, 0);
break;
case Scalar::Uint16:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lhu(output.reg, scratch, 0);
break;
case Scalar::Int32:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lw(output.reg, scratch, 0);
break;
case Scalar::Uint32:
// TODO(riscv): Why need zero-extension here?
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lwu(output.reg, scratch, 0);
break;
case Scalar::Int64:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
ld(output.reg, scratch, 0);
break;
default:
MOZ_CRASH("unexpected array type");
}
append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
asMasm().memoryBarrierAfter(access.sync());
}
void MacroAssemblerRiscv64Compat::wasmStoreI64Impl(
const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
Register ptr, Register ptrScratch, Register tmp) {
access.assertOffsetInGuardPages();
uint32_t offset = access.offset32();
MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
// Maybe add the offset.
if (offset) {
asMasm().addPtr(ImmWord(offset), ptrScratch);
ptr = ptrScratch;
}
asMasm().memoryBarrierBefore(access.sync());
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
FaultingCodeOffset fco;
switch (access.type()) {
case Scalar::Int8:
case Scalar::Uint8:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
sb(value.reg, scratch, 0);
break;
case Scalar::Int16:
case Scalar::Uint16:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
sh(value.reg, scratch, 0);
break;
case Scalar::Int32:
case Scalar::Uint32:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
sw(value.reg, scratch, 0);
break;
case Scalar::Int64:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
sd(value.reg, scratch, 0);
break;
default:
MOZ_CRASH("unexpected array type");
}
append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
asMasm().memoryBarrierAfter(access.sync());
}
void MacroAssemblerRiscv64Compat::profilerEnterFrame(Register framePtr,
Register scratch) {
asMasm().loadJSContext(scratch);
loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
storePtr(framePtr,
Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
storePtr(ImmPtr(nullptr),
Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
}
void MacroAssemblerRiscv64Compat::profilerExitFrame() {
jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
}
void MacroAssemblerRiscv64Compat::move32(Imm32 imm, Register dest) {
ma_li(dest, imm);
}
void MacroAssemblerRiscv64Compat::move32(Register src, Register dest) {
SignExtendWord(dest, src);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load8ZeroExtend(
const Address& address, Register dest) {
return ma_load(dest, address, SizeByte, ZeroExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load8ZeroExtend(
const BaseIndex& src, Register dest) {
return ma_load(dest, src, SizeByte, ZeroExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load8SignExtend(
const Address& address, Register dest) {
return ma_load(dest, address, SizeByte, SignExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load8SignExtend(
const BaseIndex& src, Register dest) {
return ma_load(dest, src, SizeByte, SignExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load16ZeroExtend(
const Address& address, Register dest) {
return ma_load(dest, address, SizeHalfWord, ZeroExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load16ZeroExtend(
const BaseIndex& src, Register dest) {
return ma_load(dest, src, SizeHalfWord, ZeroExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load16SignExtend(
const Address& address, Register dest) {
return ma_load(dest, address, SizeHalfWord, SignExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load16SignExtend(
const BaseIndex& src, Register dest) {
return ma_load(dest, src, SizeHalfWord, SignExtend);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(const Address& address,
Register dest) {
return ma_load(dest, address, SizeWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(const BaseIndex& address,
Register dest) {
return ma_load(dest, address, SizeWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(AbsoluteAddress address,
Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(ImmPtr(address.addr), scratch);
return load32(Address(scratch, 0), dest);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(
wasm::SymbolicAddress address, Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(address, scratch);
return load32(Address(scratch, 0), dest);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(const Address& address,
Register dest) {
return ma_load(dest, address, SizeDouble);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(const BaseIndex& src,
Register dest) {
return ma_load(dest, src, SizeDouble);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(AbsoluteAddress address,
Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(ImmPtr(address.addr), scratch);
return loadPtr(Address(scratch, 0), dest);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(
wasm::SymbolicAddress address, Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(address, scratch);
return loadPtr(Address(scratch, 0), dest);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPrivate(
const Address& address, Register dest) {
return loadPtr(address, dest);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Imm32 imm,
const Address& address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
return ma_store(scratch, address, SizeByte);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Register src,
const Address& address) {
return ma_store(src, address, SizeByte);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(
Imm32 imm, const BaseIndex& address) {
return ma_store(imm, address, SizeByte);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(
Register src, const BaseIndex& address) {
return ma_store(src, address, SizeByte);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
Imm32 imm, const Address& address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
return ma_store(scratch, address, SizeHalfWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
Register src, const Address& address) {
return ma_store(src, address, SizeHalfWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
Imm32 imm, const BaseIndex& address) {
return ma_store(imm, address, SizeHalfWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
Register src, const BaseIndex& address) {
return ma_store(src, address, SizeHalfWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
Register src, AbsoluteAddress address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(ImmPtr(address.addr), scratch);
return store32(src, Address(scratch, 0));
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
Register src, const Address& address) {
return ma_store(src, address, SizeWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
Imm32 src, const Address& address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
move32(src, scratch);
return ma_store(scratch, address, SizeWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
Imm32 src, const BaseIndex& address) {
return ma_store(src, address, SizeWord);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
Register src, const BaseIndex& address) {
return ma_store(src, address, SizeWord);
}
template <typename T>
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmWord imm,
T address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
return ma_store(scratch, address, SizeDouble);
}
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
ImmWord imm, Address address);
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
ImmWord imm, BaseIndex address);
template <typename T>
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmPtr imm,
T address) {
return storePtr(ImmWord(uintptr_t(imm.value)), address);
}
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
ImmPtr imm, Address address);
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
ImmPtr imm, BaseIndex address);
template <typename T>
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmGCPtr imm,
T address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(imm, scratch);
return storePtr(scratch, address);
}
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
ImmGCPtr imm, Address address);
template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
ImmGCPtr imm, BaseIndex address);
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(
Register src, const Address& address) {
return ma_store(src, address, SizeDouble);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(
Register src, const BaseIndex& address) {
return ma_store(src, address, SizeDouble);
}
FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(Register src,
AbsoluteAddress dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
movePtr(ImmPtr(dest.addr), scratch);
return storePtr(src, Address(scratch, 0));
}
void MacroAssemblerRiscv64Compat::testNullSet(Condition cond,
const ValueOperand& value,
Register dest) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
splitTag(value, scratch);
ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_NULL), cond);
}
void MacroAssemblerRiscv64Compat::testObjectSet(Condition cond,
const ValueOperand& value,
Register dest) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
splitTag(value, scratch);
ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_OBJECT), cond);
}
void MacroAssemblerRiscv64Compat::testUndefinedSet(Condition cond,
const ValueOperand& value,
Register dest) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
splitTag(value, scratch);
ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_UNDEFINED), cond);
}
void MacroAssemblerRiscv64Compat::unboxInt32(const ValueOperand& operand,
Register dest) {
SignExtendWord(dest, operand.valueReg());
}
void MacroAssemblerRiscv64Compat::unboxInt32(Register src, Register dest) {
SignExtendWord(dest, src);
}
void MacroAssemblerRiscv64Compat::unboxInt32(const Address& src,
Register dest) {
load32(Address(src.base, src.offset), dest);
}
void MacroAssemblerRiscv64Compat::unboxInt32(const BaseIndex& src,
Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(src, scratch);
load32(Address(scratch, src.offset), dest);
}
void MacroAssemblerRiscv64Compat::unboxBoolean(const ValueOperand& operand,
Register dest) {
ExtractBits(dest, operand.valueReg(), 0, 32);
}
void MacroAssemblerRiscv64Compat::unboxBoolean(Register src, Register dest) {
ExtractBits(dest, src, 0, 32);
}
void MacroAssemblerRiscv64Compat::unboxBoolean(const Address& src,
Register dest) {
ma_load(dest, Address(src.base, src.offset), SizeWord, ZeroExtend);
}
void MacroAssemblerRiscv64Compat::unboxBoolean(const BaseIndex& src,
Register dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(src, scratch);
ma_load(dest, Address(scratch, src.offset), SizeWord, ZeroExtend);
}
void MacroAssemblerRiscv64Compat::unboxDouble(const ValueOperand& operand,
FloatRegister dest) {
fmv_d_x(dest, operand.valueReg());
}
void MacroAssemblerRiscv64Compat::unboxDouble(const Address& src,
FloatRegister dest) {
ma_loadDouble(dest, Address(src.base, src.offset));
}
void MacroAssemblerRiscv64Compat::unboxDouble(const BaseIndex& src,
FloatRegister dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
loadPtr(src, scratch);
unboxDouble(ValueOperand(scratch), dest);
}
void MacroAssemblerRiscv64Compat::unboxString(const ValueOperand& operand,
Register dest) {
unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
}
void MacroAssemblerRiscv64Compat::unboxString(Register src, Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
}
void MacroAssemblerRiscv64Compat::unboxString(const Address& src,
Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
}
void MacroAssemblerRiscv64Compat::unboxSymbol(const ValueOperand& operand,
Register dest) {
unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
}
void MacroAssemblerRiscv64Compat::unboxSymbol(Register src, Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
}
void MacroAssemblerRiscv64Compat::unboxSymbol(const Address& src,
Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
}
void MacroAssemblerRiscv64Compat::unboxBigInt(const ValueOperand& operand,
Register dest) {
unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
}
void MacroAssemblerRiscv64Compat::unboxBigInt(Register src, Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
}
void MacroAssemblerRiscv64Compat::unboxBigInt(const Address& src,
Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
}
void MacroAssemblerRiscv64Compat::unboxObject(const ValueOperand& operand,
Register dest) {
unboxNonDouble(operand, dest, JSVAL_TYPE_OBJECT);
}
void MacroAssemblerRiscv64Compat::unboxObject(Register src, Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
}
void MacroAssemblerRiscv64Compat::unboxObject(const Address& src,
Register dest) {
unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
}
void MacroAssemblerRiscv64Compat::unboxValue(const ValueOperand& operand,
AnyRegister dest,
JSValueType type) {
if (dest.isFloat()) {
Label notInt32, end;
asMasm().branchTestInt32(Assembler::NotEqual, operand, &notInt32);
convertInt32ToDouble(operand.valueReg(), dest.fpu());
ma_branch(&end);
bind(&notInt32);
unboxDouble(operand, dest.fpu());
bind(&end);
} else {
unboxNonDouble(operand, dest.gpr(), type);
}
}
void MacroAssemblerRiscv64Compat::boxDouble(FloatRegister src,
const ValueOperand& dest,
FloatRegister) {
fmv_x_d(dest.valueReg(), src);
}
#ifdef DEBUG
static constexpr int32_t PayloadSize(JSValueType type) {
switch (type) {
case JSVAL_TYPE_UNDEFINED:
case JSVAL_TYPE_NULL:
return 0;
case JSVAL_TYPE_BOOLEAN:
return 1;
case JSVAL_TYPE_INT32:
case JSVAL_TYPE_MAGIC:
return 32;
case JSVAL_TYPE_STRING:
case JSVAL_TYPE_SYMBOL:
case JSVAL_TYPE_PRIVATE_GCTHING:
case JSVAL_TYPE_BIGINT:
case JSVAL_TYPE_OBJECT:
return JSVAL_TAG_SHIFT;
case JSVAL_TYPE_DOUBLE:
case JSVAL_TYPE_UNKNOWN:
break;
}
MOZ_CRASH("bad value type");
}
#endif
static void AssertValidPayload(MacroAssemblerRiscv64Compat& masm,
JSValueType type, Register payload,
Register scratch) {
#ifdef DEBUG
if (type == JSVAL_TYPE_INT32) {
// Ensure the payload is a properly sign-extended int32.
Label signExtended;
masm.SignExtendWord(scratch, payload);
masm.ma_b(payload, scratch, &signExtended, Assembler::Equal, ShortJump);
masm.breakpoint();
masm.bind(&signExtended);
} else {
// All bits above the payload must be zeroed.
Label zeroed;
masm.srli(scratch, payload, PayloadSize(type));
masm.ma_b(scratch, Imm32(0), &zeroed, Assembler::Equal, ShortJump);
masm.breakpoint();
masm.bind(&zeroed);
}
#endif
}
void MacroAssemblerRiscv64Compat::boxValue(JSValueType type, Register src,
Register dest) {
MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL);
MOZ_ASSERT(src != dest);
AssertValidPayload(*this, type, src, dest);
switch (type) {
case JSVAL_TYPE_INT32: {
// Loading the shifted tag requires only two instructions.
ma_li(dest, ImmShiftedTag(type));
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Insert low 32 bits as payload, removing all high bits from |src|.
ZeroExtendWord(scratch, src);
or_(dest, dest, scratch);
return;
}
case JSVAL_TYPE_BOOLEAN:
case JSVAL_TYPE_MAGIC:
case JSVAL_TYPE_STRING:
case JSVAL_TYPE_SYMBOL:
case JSVAL_TYPE_PRIVATE_GCTHING:
case JSVAL_TYPE_BIGINT:
case JSVAL_TYPE_OBJECT: {
// Loading the shifted tag requires only two instructions.
ma_li(dest, ImmShiftedTag(type));
// Insert payload.
or_(dest, dest, src);
return;
}
case JSVAL_TYPE_DOUBLE:
case JSVAL_TYPE_UNDEFINED:
case JSVAL_TYPE_NULL:
case JSVAL_TYPE_UNKNOWN:
break;
}
MOZ_CRASH("bad value type");
}
void MacroAssemblerRiscv64Compat::boxValue(Register type, Register src,
Register dest) {
MOZ_ASSERT(src != dest);
#ifdef DEBUG
Label done, isNullOrUndefined, isBoolean, isInt32OrMagic, isPointerSized;
asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_NULL),
&isNullOrUndefined);
asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_UNDEFINED),
&isNullOrUndefined);
asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_BOOLEAN),
&isBoolean);
asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_INT32),
&isInt32OrMagic);
asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_MAGIC),
&isInt32OrMagic);
// GCThing types aren't currently supported, because SignExtendWord truncates
// payloads above UINT32_MAX.
breakpoint();
{
bind(&isNullOrUndefined);
// Ensure no payload for null and undefined.
ma_b(src, src, &done, Assembler::Zero, ShortJump);
breakpoint();
}
{
bind(&isBoolean);
// Ensure boolean values are either 0 or 1.
ma_b(src, Imm32(1), &done, Assembler::BelowOrEqual, ShortJump);
breakpoint();
}
{
bind(&isInt32OrMagic);
// Ensure |src| is sign-extended.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
SignExtendWord(scratch, src);
ma_b(src, scratch, &done, Assembler::Equal, ShortJump);
breakpoint();
}
bind(&done);
#endif
// JSVAL_TAG_MAX_DOUBLE can't be directly encoded in a single `ori`
// instruction. Sign-extend the tag, taking the bits into account which will
// later be shifted out, into a shorter immediate which fits into `ori`.
constexpr int64_t tag =
int64_t(uint64_t(JSVAL_TAG_MAX_DOUBLE) << JSVAL_TAG_SHIFT) >>
JSVAL_TAG_SHIFT;
static_assert(is_int12(tag), "ori requires int12 immediate");
ori(dest, type, tag);
slli(dest, dest, JSVAL_TAG_SHIFT);
// Insert low 32 bits as payload, removing all high bits from |src|.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ZeroExtendWord(scratch, src);
ma_or(dest, dest, scratch);
}
void MacroAssemblerRiscv64Compat::loadConstantFloat32(float f,
FloatRegister dest) {
ma_lis(dest, f);
}
void MacroAssemblerRiscv64Compat::loadInt32OrDouble(const Address& src,
FloatRegister dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Label notInt32, end;
{
// Inlined |branchTestInt32| to use a short-jump.
Register tag = extractTag(src, scratch);
ma_b(tag, ImmTag(JSVAL_TAG_INT32), &notInt32, Assembler::NotEqual,
ShortJump);
}
{
// If it's an int, convert it to double.
unboxInt32(src, scratch);
convertInt32ToDouble(scratch, dest);
ma_branch(&end);
}
bind(&notInt32);
{
// Not an int, just load as double.
unboxDouble(src, dest);
}
bind(&end);
}
void MacroAssemblerRiscv64Compat::loadInt32OrDouble(const BaseIndex& addr,
FloatRegister dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(addr, scratch);
loadInt32OrDouble(Address(scratch, addr.offset), dest);
}
void MacroAssemblerRiscv64Compat::loadConstantDouble(double dp,
FloatRegister dest) {
ma_lid(dest, dp);
}
Register MacroAssemblerRiscv64Compat::extractObject(const Address& address,
Register scratch) {
loadPtr(address, scratch);
ExtractBits(scratch, scratch, 0, JSVAL_TAG_SHIFT);
return scratch;
}
Register MacroAssemblerRiscv64Compat::extractTag(const Address& address,
Register scratch) {
loadPtr(address, scratch);
ExtractBits(scratch, scratch, JSVAL_TAG_SHIFT, 64 - JSVAL_TAG_SHIFT);
return scratch;
}
Register MacroAssemblerRiscv64Compat::extractTag(const BaseIndex& address,
Register scratch) {
computeScaledAddress(address, scratch);
return extractTag(Address(scratch, address.offset), scratch);
}
/////////////////////////////////////////////////////////////////
// X86/X64-common/ARM/LoongArch interface.
/////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////
// X86/X64-common/ARM/MIPS interface.
/////////////////////////////////////////////////////////////////
void MacroAssemblerRiscv64Compat::storeValue(ValueOperand val,
const BaseIndex& dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(dest, scratch);
storeValue(val, Address(scratch, dest.offset));
}
void MacroAssemblerRiscv64Compat::storeValue(JSValueType type, Register reg,
BaseIndex dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(dest, scratch);
int32_t offset = dest.offset;
if (!is_int12(offset)) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_li(scratch2, Imm32(offset));
add(scratch, scratch, scratch2);
offset = 0;
}
storeValue(type, reg, Address(scratch, offset));
}
void MacroAssemblerRiscv64Compat::storeValue(ValueOperand val,
const Address& dest) {
storePtr(val.valueReg(), Address(dest.base, dest.offset));
}
void MacroAssemblerRiscv64Compat::storeValue(JSValueType type, Register reg,
Address dest) {
if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
#ifdef DEBUG
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
AssertValidPayload(*this, type, reg, scratch);
}
#endif
store32(reg, dest);
JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
store32(Imm64(tag).hi(), Address(dest.base, dest.offset + 4));
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(dest.base != scratch);
boxValue(type, reg, scratch);
storePtr(scratch, Address(dest.base, dest.offset));
}
}
void MacroAssemblerRiscv64Compat::storeValue(const Value& val, Address dest) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
if (val.isGCThing()) {
writeDataRelocation(val);
movWithPatch(ImmWord(val.asRawBits()), scratch2);
} else {
ma_li(scratch2, ImmWord(val.asRawBits()));
}
storePtr(scratch2, Address(dest.base, dest.offset));
}
void MacroAssemblerRiscv64Compat::storeValue(const Value& val, BaseIndex dest) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(dest, scratch);
int32_t offset = dest.offset;
if (!is_int12(offset)) {
Register scratch2 = temps.Acquire();
ma_li(scratch2, Imm32(offset));
add(scratch, scratch, scratch2);
offset = 0;
}
storeValue(val, Address(scratch, offset));
}
void MacroAssemblerRiscv64Compat::loadValue(const BaseIndex& src,
ValueOperand val) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(src, scratch);
loadValue(Address(scratch, src.offset), val);
}
void MacroAssemblerRiscv64Compat::loadValue(Address src, ValueOperand val) {
loadPtr(Address(src.base, src.offset), val.valueReg());
}
void MacroAssemblerRiscv64Compat::tagValue(JSValueType type, Register payload,
ValueOperand dest) {
MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL);
JitSpew(JitSpew_Codegen, "[ tagValue");
if (payload == dest.valueReg()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(dest.valueReg() != scratch);
AssertValidPayload(*this, type, payload, scratch);
switch (type) {
case JSVAL_TYPE_INT32: {
// Loading the shifted tag requires only two instructions.
ma_li(scratch, ImmShiftedTag(type));
// Insert low 32 bits as payload, removing all high bits from |payload|.
ZeroExtendWord(payload, payload);
or_(dest.valueReg(), payload, scratch);
break;
}
case JSVAL_TYPE_BOOLEAN:
case JSVAL_TYPE_MAGIC:
case JSVAL_TYPE_STRING:
case JSVAL_TYPE_SYMBOL:
case JSVAL_TYPE_PRIVATE_GCTHING:
case JSVAL_TYPE_BIGINT:
case JSVAL_TYPE_OBJECT: {
// Loading the shifted tag requires only two instructions.
ma_li(scratch, ImmShiftedTag(type));
// Insert payload.
or_(dest.valueReg(), payload, scratch);
break;
}
case JSVAL_TYPE_DOUBLE:
case JSVAL_TYPE_UNDEFINED:
case JSVAL_TYPE_NULL:
case JSVAL_TYPE_UNKNOWN:
MOZ_CRASH("bad value type");
}
} else {
boxNonDouble(type, payload, dest);
}
JitSpew(JitSpew_Codegen, "]");
}
void MacroAssemblerRiscv64Compat::pushValue(ValueOperand val) {
// Allocate stack slots for Value. One for each.
asMasm().subPtr(Imm32(sizeof(Value)), StackPointer);
// Store Value
storeValue(val, Address(StackPointer, 0));
}
void MacroAssemblerRiscv64Compat::pushValue(const Address& addr) {
// Load value before allocate stack, addr.base may be is sp.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
loadPtr(Address(addr.base, addr.offset), scratch);
ma_sub64(StackPointer, StackPointer, Imm32(sizeof(Value)));
storePtr(scratch, Address(StackPointer, 0));
}
void MacroAssemblerRiscv64Compat::popValue(ValueOperand val) {
ld(val.valueReg(), StackPointer, 0);
ma_add64(StackPointer, StackPointer, Imm32(sizeof(Value)));
}
void MacroAssemblerRiscv64Compat::breakpoint(uint32_t value) { break_(value); }
void MacroAssemblerRiscv64Compat::handleFailureWithHandlerTail(
Label* profilerExitTail, Label* bailoutTail,
uint32_t* returnValueCheckOffset) {
// Reserve space for exception information.
int size = (sizeof(ResumeFromException) + ABIStackAlignment) &
~(ABIStackAlignment - 1);
asMasm().subPtr(Imm32(size), StackPointer);
mv(a0, StackPointer); // Use a0 since it is a first function argument
// Call the handler.
using Fn = void (*)(ResumeFromException* rfe);
asMasm().setupUnalignedABICall(a1);
asMasm().passABIArg(a0);
asMasm().callWithABI<Fn, HandleException>(
ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
*returnValueCheckOffset = currentOffset();
Label entryFrame;
Label catch_;
Label finally;
Label returnBaseline;
Label returnIon;
Label bailout;
Label wasmInterpEntry;
Label wasmCatch;
// Already clobbered a0, so use it...
load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0);
asMasm().branch32(Assembler::Equal, a0,
Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch),
&catch_);
asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally),
&finally);
asMasm().branch32(Assembler::Equal, a0,
Imm32(ExceptionResumeKind::ForcedReturnBaseline),
&returnBaseline);
asMasm().branch32(Assembler::Equal, a0,
Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout),
&bailout);
asMasm().branch32(Assembler::Equal, a0,
Imm32(ExceptionResumeKind::WasmInterpEntry),
&wasmInterpEntry);
asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch),
&wasmCatch);
breakpoint(); // Invalid kind.
// No exception handler. Load the error value, restore state and return from
// the entry frame.
bind(&entryFrame);
asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
// We're going to be returning by the ion calling convention
ma_pop(ra);
jump(ra);
nop();
// If we found a catch handler, this must be a baseline frame. Restore
// state and jump to the catch block.
bind(&catch_);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
jump(a0);
// If we found a finally block, this must be a baseline frame. Push three
// values expected by the finally block: the exception, the exception stack,
// and BooleanValue(true).
bind(&finally);
ValueOperand exception = ValueOperand(a1);
loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
ValueOperand exceptionStack = ValueOperand(a2);
loadValue(Address(sp, ResumeFromException::offsetOfExceptionStack()),
exceptionStack);
loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0);
loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
pushValue(exception);
pushValue(exceptionStack);
pushValue(BooleanValue(true));
jump(a0);
// Return BaselineFrame->returnValue() to the caller.
// Used in debug mode and for GeneratorReturn.
Label profilingInstrumentation;
bind(&returnBaseline);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
JSReturnOperand);
jump(&profilingInstrumentation);
// Return the given value to the caller.
bind(&returnIon);
loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
JSReturnOperand);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
// If profiling is enabled, then update the lastProfilingFrame to refer to
// caller frame before returning. This code is shared by ForcedReturnIon
// and ForcedReturnBaseline.
bind(&profilingInstrumentation);
{
Label skipProfilingInstrumentation;
// Test if profiler enabled.
AbsoluteAddress addressOfEnabled(
asMasm().runtime()->geckoProfiler().addressOfEnabled());
asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
&skipProfilingInstrumentation);
jump(profilerExitTail);
bind(&skipProfilingInstrumentation);
}
mv(StackPointer, FramePointer);
pop(FramePointer);
ret();
// If we are bailing out to baseline to handle an exception, jump to
// the bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
bind(&bailout);
loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
ma_li(ReturnReg, Imm32(1));
jump(bailoutTail);
// Reset SP and FP; SP is pointing to the unwound return address to the wasm
// interpreter entry, so we can just ret().
bind(&wasmInterpEntry);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
FramePointer);
loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
StackPointer);
ma_li(InstanceReg, ImmWord(wasm::InterpFailInstanceReg));
ret();
// Found a wasm catch handler, restore state and jump to it.
bind(&wasmCatch);
wasm::GenerateJumpToCatchHandler(asMasm(), sp, a1, a2);
}
CodeOffset MacroAssemblerRiscv64Compat::toggledJump(Label* label) {
CodeOffset ret(nextOffset().getOffset());
BranchShort(label);
return ret;
}
CodeOffset MacroAssemblerRiscv64Compat::toggledCall(JitCode* target,
bool enabled) {
DEBUG_PRINTF("\ttoggledCall\n");
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 8);
BufferOffset bo = nextOffset();
CodeOffset offset(bo.getOffset());
addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
ma_liPatchable(scratch, ImmPtr(target->raw()));
if (enabled) {
jalr(scratch);
} else {
nop();
}
MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() ==
ToggledCallSize(nullptr));
return offset;
}
void MacroAssembler::subFromStackPtr(Imm32 imm32) {
if (imm32.value) {
asMasm().subPtr(imm32, StackPointer);
}
}
void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
Round_w_d(output, input);
Clear_if_nan_d(output, input);
clampIntToUint8(output);
}
//{{{ check_macroassembler_style
// ===============================================================
// MacroAssembler high-level usage.
bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
CodeOffset MacroAssembler::call(Label* label) { return BranchAndLink(label); }
CodeOffset MacroAssembler::call(Register reg) {
jalr(reg, 0);
return CodeOffset(currentOffset());
}
CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
movePtr(imm, CallReg);
return call(CallReg);
}
CodeOffset MacroAssembler::farJumpWithPatch() {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Register scratch2 = temps.Acquire();
// Allocate space which will be patched by patchFarJump().
CodeOffset farJump(nextInstrOffset(5).getOffset());
auipc(scratch, 0);
lw(scratch2, scratch, 4 * sizeof(Instr));
add(scratch, scratch, scratch2);
jr(scratch, 0);
spew(".space 32bit initValue 0xffff ffff");
emit(UINT32_MAX);
return farJump;
}
CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
return movWithPatch(ImmPtr(nullptr), dest);
}
CodeOffset MacroAssembler::nopPatchableToCall() {
BlockTrampolinePoolScope block_trampoline_pool(this, 7);
// riscv64
nop(); // lui(rd, (int32_t)high_20);
nop(); // addi(rd, rd, low_12); // 31 bits in rd.
nop(); // slli(rd, rd, 11); // Space for next 11 bis
nop(); // ori(rd, rd, b11); // 11 bits are put in. 42 bit in rd
nop(); // slli(rd, rd, 6); // Space for next 6 bits
nop(); // ori(rd, rd, a6); // 6 bits are put in. 48 bis in rd
nop(); // jirl
return CodeOffset(currentOffset());
}
FaultingCodeOffset MacroAssembler::wasmTrapInstruction() {
BlockTrampolinePoolScope block_trampoline_pool(this, 2);
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
illegal_trap(kWasmTrapCode);
ebreak();
return fco;
}
size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
}
template <typename T>
void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
const T& value, Register temp,
Label* label) {
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done;
branchTestGCThing(Assembler::NotEqual, value,
cond == Assembler::Equal ? &done : label);
// temp may be InvalidReg, use scratch2 instead.
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
getGCThingValueChunk(value, scratch2);
loadPtr(Address(scratch2, gc::ChunkStoreBufferOffset), scratch2);
branchPtr(InvertCondition(cond), scratch2, ImmWord(0), label);
bind(&done);
}
template <typename T>
void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
MIRType valueType, const T& dest) {
MOZ_ASSERT(valueType < MIRType::Value);
if (valueType == MIRType::Double) {
boxDouble(value.reg().typedReg().fpu(), dest);
return;
}
if (value.constant()) {
storeValue(value.value(), dest);
} else {
storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
dest);
}
}
template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
MIRType valueType,
const Address& dest);
template void MacroAssembler::storeUnboxedValue(
const ConstantOrRegister& value, MIRType valueType,
const BaseObjectElementIndex& dest);
// ===============================================================
// Jit Frames.
uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
CodeLabel cl;
ma_li(scratch, &cl);
Push(scratch);
bind(&cl);
uint32_t retAddr = currentOffset();
addCodeLabel(cl);
return retAddr;
}
//===============================
// AtomicOp
template <typename T>
static void AtomicExchange(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Scalar::Type type, Synchronization sync,
const T& mem, Register value, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
bool signExtend = Scalar::isSignedIntType(type);
unsigned nbytes = Scalar::byteSize(type);
UseScratchRegisterScope temps(&masm);
switch (nbytes) {
case 1:
case 2:
break;
case 4:
MOZ_ASSERT(valueTemp == InvalidReg);
MOZ_ASSERT(offsetTemp == InvalidReg);
MOZ_ASSERT(maskTemp == InvalidReg);
break;
default:
MOZ_CRASH();
}
Label again;
Register scratch = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch);
Register scratch2 = temps.Acquire();
if (nbytes == 4) {
masm.memoryBarrierBefore(sync);
masm.bind(&again);
BlockTrampolinePoolScope block_trampoline_pool(&masm,
/* 1 + 1 + 1 + 4 + 1 = */ 8,
1);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, output, scratch);
masm.or_(scratch2, value, zero);
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
masm.memoryBarrierAfter(sync);
return;
}
masm.andi(offsetTemp, scratch, 3);
masm.subPtr(offsetTemp, scratch);
masm.slliw(offsetTemp, offsetTemp, 3);
masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
masm.sllw(maskTemp, maskTemp, offsetTemp);
masm.not_(maskTemp, maskTemp);
switch (nbytes) {
case 1:
masm.andi(valueTemp, value, 0xff);
break;
case 2:
masm.ma_and(valueTemp, value, Imm32(0xffff));
break;
}
masm.sllw(valueTemp, valueTemp, offsetTemp);
masm.memoryBarrierBefore(sync);
masm.bind(&again);
BlockTrampolinePoolScope block_trampoline_pool(
&masm, /* 1 + 1 + 1 + 1 + 4 + 1 + 2 + 1 = */ 12, 1);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, output, scratch);
masm.and_(scratch2, output, maskTemp);
masm.or_(scratch2, scratch2, valueTemp);
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
masm.srlw(output, output, offsetTemp);
switch (nbytes) {
case 1:
if (signExtend) {
masm.SignExtendByte(output, output);
} else {
masm.andi(output, output, 0xff);
}
break;
case 2:
if (signExtend) {
masm.SignExtendShort(output, output);
} else {
masm.ma_and(output, Imm32(0xffff));
}
break;
}
masm.memoryBarrierAfter(sync);
}
template <typename T>
static void AtomicExchange64(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Synchronization sync, const T& mem,
Register64 value, Register64 output) {
MOZ_ASSERT(value != output);
UseScratchRegisterScope temps(&masm);
Register scratch = temps.Acquire();
Register scratch2 = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch2);
Label tryAgain;
masm.memoryBarrierBefore(sync);
masm.bind(&tryAgain);
BlockTrampolinePoolScope block_trampoline_pool(&masm,
/* 1 + 1 + 1 + 4 + 1 = */ 8,
1);
if (access) {
masm.append(*access, js::wasm::TrapMachineInsn::Load64,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_d(true, true, output.reg, scratch2);
masm.movePtr(value.reg, scratch);
masm.sc_d(true, true, scratch, scratch2, scratch);
masm.ma_b(scratch, scratch, &tryAgain, Assembler::NonZero, ShortJump);
masm.memoryBarrierAfter(sync);
}
template <typename T>
static void AtomicFetchOp64(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Synchronization sync, AtomicOp op, Register64 value,
const T& mem, Register64 temp, Register64 output) {
MOZ_ASSERT(value != output);
MOZ_ASSERT(value != temp);
UseScratchRegisterScope temps(&masm);
Register scratch2 = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch2);
Label tryAgain;
masm.memoryBarrierBefore(sync);
masm.bind(&tryAgain);
BlockTrampolinePoolScope block_trampoline_pool(&masm,
/* 1 + 1 + 1 + 4 + 1 = */ 8,
1);
if (access) {
masm.append(*access, js::wasm::TrapMachineInsn::Load64,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_d(true, true, output.reg, scratch2);
switch (op) {
case AtomicOp::Add:
masm.add(temp.reg, output.reg, value.reg);
break;
case AtomicOp::Sub:
masm.sub(temp.reg, output.reg, value.reg);
break;
case AtomicOp::And:
masm.and_(temp.reg, output.reg, value.reg);
break;
case AtomicOp::Or:
masm.or_(temp.reg, output.reg, value.reg);
break;
case AtomicOp::Xor:
masm.xor_(temp.reg, output.reg, value.reg);
break;
default:
MOZ_CRASH();
}
masm.sc_d(true, true, temp.reg, scratch2, temp.reg);
masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::NonZero, ShortJump);
masm.memoryBarrierAfter(sync);
}
template <typename T>
static void AtomicEffectOp(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Scalar::Type type, Synchronization sync, AtomicOp op,
const T& mem, Register value, Register valueTemp,
Register offsetTemp, Register maskTemp) {
UseScratchRegisterScope temps(&masm);
unsigned nbytes = Scalar::byteSize(type);
switch (nbytes) {
case 1:
case 2:
break;
case 4:
MOZ_ASSERT(valueTemp == InvalidReg);
MOZ_ASSERT(offsetTemp == InvalidReg);
MOZ_ASSERT(maskTemp == InvalidReg);
break;
default:
MOZ_CRASH();
}
Label again;
Register scratch = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch);
Register scratch2 = temps.Acquire();
if (nbytes == 4) {
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, scratch2, scratch);
switch (op) {
case AtomicOp::Add:
masm.addw(scratch2, scratch2, value);
break;
case AtomicOp::Sub:
masm.subw(scratch2, scratch2, value);
break;
case AtomicOp::And:
masm.and_(scratch2, scratch2, value);
break;
case AtomicOp::Or:
masm.or_(scratch2, scratch2, value);
break;
case AtomicOp::Xor:
masm.xor_(scratch2, scratch2, value);
break;
default:
MOZ_CRASH();
}
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
masm.memoryBarrierAfter(sync);
return;
}
masm.andi(offsetTemp, scratch, 3);
masm.subPtr(offsetTemp, scratch);
masm.slliw(offsetTemp, offsetTemp, 3);
masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
masm.sllw(maskTemp, maskTemp, offsetTemp);
masm.not_(maskTemp, maskTemp);
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, scratch2, scratch);
masm.srlw(valueTemp, scratch2, offsetTemp);
switch (op) {
case AtomicOp::Add:
masm.addw(valueTemp, valueTemp, value);
break;
case AtomicOp::Sub:
masm.subw(valueTemp, valueTemp, value);
break;
case AtomicOp::And:
masm.and_(valueTemp, valueTemp, value);
break;
case AtomicOp::Or:
masm.or_(valueTemp, valueTemp, value);
break;
case AtomicOp::Xor:
masm.xor_(valueTemp, valueTemp, value);
break;
default:
MOZ_CRASH();
}
switch (nbytes) {
case 1:
masm.andi(valueTemp, valueTemp, 0xff);
break;
case 2:
masm.ma_and(valueTemp, valueTemp, Imm32(0xffff));
break;
}
masm.sllw(valueTemp, valueTemp, offsetTemp);
masm.and_(scratch2, scratch2, maskTemp);
masm.or_(scratch2, scratch2, valueTemp);
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
masm.memoryBarrierAfter(sync);
}
template <typename T>
static void AtomicFetchOp(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Scalar::Type type, Synchronization sync, AtomicOp op,
const T& mem, Register value, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
UseScratchRegisterScope temps(&masm);
bool signExtend = Scalar::isSignedIntType(type);
unsigned nbytes = Scalar::byteSize(type);
switch (nbytes) {
case 1:
case 2:
break;
case 4:
MOZ_ASSERT(valueTemp == InvalidReg);
MOZ_ASSERT(offsetTemp == InvalidReg);
MOZ_ASSERT(maskTemp == InvalidReg);
break;
default:
MOZ_CRASH();
}
Label again;
Register scratch = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch);
Register scratch2 = temps.Acquire();
if (nbytes == 4) {
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, output, scratch);
switch (op) {
case AtomicOp::Add:
masm.addw(scratch2, output, value);
break;
case AtomicOp::Sub:
masm.subw(scratch2, output, value);
break;
case AtomicOp::And:
masm.and_(scratch2, output, value);
break;
case AtomicOp::Or:
masm.or_(scratch2, output, value);
break;
case AtomicOp::Xor:
masm.xor_(scratch2, output, value);
break;
default:
MOZ_CRASH();
}
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
masm.memoryBarrierAfter(sync);
return;
}
masm.andi(offsetTemp, scratch, 3);
masm.subPtr(offsetTemp, scratch);
masm.slliw(offsetTemp, offsetTemp, 3);
masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
masm.sllw(maskTemp, maskTemp, offsetTemp);
masm.not_(maskTemp, maskTemp);
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, scratch2, scratch);
masm.srlw(output, scratch2, offsetTemp);
switch (op) {
case AtomicOp::Add:
masm.addw(valueTemp, output, value);
break;
case AtomicOp::Sub:
masm.subw(valueTemp, output, value);
break;
case AtomicOp::And:
masm.and_(valueTemp, output, value);
break;
case AtomicOp::Or:
masm.or_(valueTemp, output, value);
break;
case AtomicOp::Xor:
masm.xor_(valueTemp, output, value);
break;
default:
MOZ_CRASH();
}
switch (nbytes) {
case 1:
masm.andi(valueTemp, valueTemp, 0xff);
break;
case 2:
masm.ma_and(valueTemp, Imm32(0xffff));
break;
}
masm.sllw(valueTemp, valueTemp, offsetTemp);
masm.and_(scratch2, scratch2, maskTemp);
masm.or_(scratch2, scratch2, valueTemp);
masm.sc_w(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
ShortJump);
switch (nbytes) {
case 1:
if (signExtend) {
masm.SignExtendByte(output, output);
} else {
masm.andi(output, output, 0xff);
}
break;
case 2:
if (signExtend) {
masm.SignExtendShort(output, output);
} else {
masm.ma_and(output, Imm32(0xffff));
}
break;
}
masm.memoryBarrierAfter(sync);
}
// ========================================================================
// JS atomic operations.
template <typename T>
static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
Synchronization sync, const T& mem,
Register oldval, Register newval,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
if (arrayType == Scalar::Uint32) {
masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
offsetTemp, maskTemp, temp);
masm.convertUInt32ToDouble(temp, output.fpu());
} else {
masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
offsetTemp, maskTemp, output.gpr());
}
}
template <typename T>
static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
Synchronization sync, const T& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
if (arrayType == Scalar::Uint32) {
masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
maskTemp, temp);
masm.convertUInt32ToDouble(temp, output.fpu());
} else {
masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
maskTemp, output.gpr());
}
}
template <typename T>
static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
Synchronization sync, AtomicOp op, Register value,
const T& mem, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register temp, AnyRegister output) {
if (arrayType == Scalar::Uint32) {
masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
maskTemp, temp);
masm.convertUInt32ToDouble(temp, output.fpu());
} else {
masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
maskTemp, output.gpr());
}
}
void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
Synchronization sync, AtomicOp op,
Register value, const BaseIndex& mem,
Register valueTemp, Register offsetTemp,
Register maskTemp) {
AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
offsetTemp, maskTemp);
}
void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
Synchronization sync, AtomicOp op,
Register value, const Address& mem,
Register valueTemp, Register offsetTemp,
Register maskTemp) {
AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
offsetTemp, maskTemp);
}
void MacroAssembler::atomicExchange64(Synchronization sync, const Address& mem,
Register64 value, Register64 output) {
AtomicExchange64(*this, nullptr, sync, mem, value, output);
}
void MacroAssembler::atomicExchange64(Synchronization sync,
const BaseIndex& mem, Register64 value,
Register64 output) {
AtomicExchange64(*this, nullptr, sync, mem, value, output);
}
void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
Synchronization sync, const Address& mem,
Register value, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register temp, AnyRegister output) {
AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
maskTemp, temp, output);
}
void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
Synchronization sync,
const BaseIndex& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
maskTemp, temp, output);
}
void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync,
const Address& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register output) {
AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
maskTemp, output);
}
void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync,
const BaseIndex& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register output) {
AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
maskTemp, output);
}
void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
Synchronization sync, AtomicOp op,
Register value, const Address& mem,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
maskTemp, temp, output);
}
void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
Synchronization sync, AtomicOp op,
Register value, const BaseIndex& mem,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
maskTemp, temp, output);
}
void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync,
AtomicOp op, Register value,
const Address& mem, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
offsetTemp, maskTemp, output);
}
void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync,
AtomicOp op, Register value,
const BaseIndex& mem, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
offsetTemp, maskTemp, output);
}
void MacroAssembler::atomicPause() {
// `pause` hint defined in Zihintpause extension.
// It is encoded as `fence w, 0`.
fence(0b0001, 0b0000);
}
void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
Register temp, Label* label) {
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(ptr != temp);
MOZ_ASSERT(temp != InvalidReg);
ma_and(temp, ptr, Imm32(int32_t(~gc::ChunkMask)));
branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
zero, label);
}
void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
const Value& rhs, Label* label) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
MOZ_ASSERT(!rhs.isNaN());
if (!rhs.isGCThing()) {
ma_b(lhs.valueReg(), ImmWord(rhs.asRawBits()), label, cond);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(lhs.valueReg() != scratch);
moveValue(rhs, ValueOperand(scratch));
ma_b(lhs.valueReg(), scratch, label, cond);
}
}
void MacroAssembler::branchTestNaNValue(Condition cond, const ValueOperand& val,
Register temp, Label* label) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(val.valueReg() != scratch);
// When testing for NaN, we want to ignore the sign bit.
// Clear the top bit by shifting left and then right.
slli(temp, val.valueReg(), 1);
srli(temp, temp, 1);
// Compare against a NaN with sign bit 0.
static_assert(JS::detail::CanonicalizedNaNSignBit == 0);
moveValue(DoubleValue(JS::GenericNaN()), ValueOperand(scratch));
ma_b(temp, scratch, label, cond);
}
void MacroAssembler::branchValueIsNurseryCell(Condition cond,
const Address& address,
Register temp, Label* label) {
branchValueIsNurseryCellImpl(cond, address, temp, label);
}
void MacroAssembler::branchValueIsNurseryCell(Condition cond,
ValueOperand value, Register temp,
Label* label) {
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
CodeOffset MacroAssembler::call(const Address& addr) {
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
loadPtr(addr, CallReg);
return call(CallReg);
}
void MacroAssembler::call(ImmPtr imm) {
BufferOffset bo = m_buffer.nextOffset();
addPendingJump(bo, imm, RelocationKind::HARDCODED);
ma_call(imm);
}
void MacroAssembler::call(ImmWord imm) { call(ImmPtr((void*)imm.value)); }
void MacroAssembler::call(JitCode* c) {
DEBUG_PRINTF("[ %s\n", __FUNCTION__);
BlockTrampolinePoolScope block_trampoline_pool(this, 8);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BufferOffset bo = m_buffer.nextOffset();
addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
ma_liPatchable(scratch, ImmPtr(c->raw()));
callJitNoProfiler(scratch);
DEBUG_PRINTF("]\n");
}
void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
MOZ_ASSERT(inCall_);
uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
// Reserve place for $ra.
stackForCall += sizeof(intptr_t);
if (dynamicAlignment_) {
stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
} else {
uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
stackForCall += ComputeByteAlignment(
stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
}
*stackAdjust = stackForCall;
reserveStack(stackForCall);
// Save $ra because call is going to clobber it. Restore it in
// callWithABIPost. NOTE: This is needed for calls from SharedIC.
// Maybe we can do this differently.
storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t)));
// Position all arguments.
{
enoughMemory_ &= moveResolver_.resolve();
if (!enoughMemory_) {
return;
}
MoveEmitter emitter(asMasm());
emitter.emit(moveResolver_);
emitter.finish();
}
assertStackAlignment(ABIStackAlignment);
}
void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) {
// Restore ra value (as stored in callWithABIPre()).
loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra);
if (dynamicAlignment_) {
// Restore sp value from stack (as stored in setupUnalignedABICall()).
loadPtr(Address(StackPointer, stackAdjust), StackPointer);
// Use adjustFrame instead of freeStack because we already restored sp.
adjustFrame(-stackAdjust);
} else {
freeStack(stackAdjust);
}
#ifdef DEBUG
MOZ_ASSERT(inCall_);
inCall_ = false;
#endif
}
void MacroAssembler::callWithABINoProfiler(Register fun, ABIType result) {
// Load the callee in scratch2, no instruction between the movePtr and
// call should clobber it. Note that we can't use fun because it may be
// one of the IntArg registers clobbered before the call.
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
movePtr(fun, CallReg);
uint32_t stackAdjust;
callWithABIPre(&stackAdjust);
call(CallReg);
callWithABIPost(stackAdjust, result);
}
void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) {
// Load the callee in scratch2, as above.
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
loadPtr(fun, CallReg);
uint32_t stackAdjust;
callWithABIPre(&stackAdjust);
call(CallReg);
callWithABIPost(stackAdjust, result);
}
void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Round toward positive infinity.
Ceil_l_d(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// We have to check for (-1, -0] when the result is zero.
Label notZero;
ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
{
fmv_x_d(scratch, src);
ma_b(scratch, scratch, fail, Assembler::Signed);
}
bind(&notZero);
}
void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Round toward positive infinity.
Ceil_l_s(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// We have to check for (-1, -0] when the result is zero.
Label notZero;
ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
{
fmv_x_w(scratch, src);
ma_b(scratch, scratch, fail, Assembler::Signed);
}
bind(&notZero);
}
void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
template <typename T>
static void CompareExchange64(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Synchronization sync, const T& mem,
Register64 expect, Register64 replace,
Register64 output) {
MOZ_ASSERT(expect != output && replace != output);
UseScratchRegisterScope temps(&masm);
Register scratch = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch);
Register scratch2 = temps.Acquire();
Label tryAgain;
Label exit;
masm.memoryBarrierBefore(sync);
masm.bind(&tryAgain);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_d(true, true, output.reg, scratch);
masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump);
masm.movePtr(replace.reg, scratch2);
masm.sc_d(true, true, scratch2, scratch, scratch2);
masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::NonZero,
ShortJump);
masm.memoryBarrierAfter(sync);
masm.bind(&exit);
}
void MacroAssembler::compareExchange64(Synchronization sync, const Address& mem,
Register64 expect, Register64 replace,
Register64 output) {
CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
}
void MacroAssembler::compareExchange64(Synchronization sync,
const BaseIndex& mem, Register64 expect,
Register64 replace, Register64 output) {
CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
}
void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
Synchronization sync, const Address& mem,
Register expected, Register replacement,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register temp,
AnyRegister output) {
CompareExchangeJS(*this, arrayType, sync, mem, expected, replacement,
valueTemp, offsetTemp, maskTemp, temp, output);
}
void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
Synchronization sync,
const BaseIndex& mem, Register expected,
Register replacement, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register temp, AnyRegister output) {
CompareExchangeJS(*this, arrayType, sync, mem, expected, replacement,
valueTemp, offsetTemp, maskTemp, temp, output);
}
void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
fcvt_d_l(dest, src.scratchReg());
}
void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
fcvt_s_l(dest, src.scratchReg());
}
void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
fcvt_d_l(dest, src);
}
void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
Register temp) {
fcvt_d_lu(dest, src.scratchReg());
}
void MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest,
Register temp) {
fcvt_s_lu(dest, src.scratchReg());
}
void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
FloatRegister output) {
fsgnj_d(output, lhs, rhs);
}
void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs,
FloatRegister output) {
fsgnj_s(output, lhs, rhs);
}
void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
ExitFrameType type) {
enterFakeExitFrame(cxreg, scratch, type);
}
CodeOffset MacroAssembler::sub32FromMemAndBranchIfNegativeWithPatch(
Address address, Label* label) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(scratch != address.base);
ma_load(scratch, address);
addiw(scratch, scratch, 128);
CodeOffset patchPoint = CodeOffset(currentOffset());
ma_store(scratch, address);
ma_b(scratch, scratch, label, Assembler::Signed);
return patchPoint;
}
void MacroAssembler::patchSub32FromMemAndBranchIfNegative(CodeOffset offset,
Imm32 imm) {
int32_t val = imm.value;
MOZ_RELEASE_ASSERT(val >= 1 && val <= 127);
auto* inst = m_buffer.getInst(BufferOffset(offset.offset() - 4));
inst->InstructionOpcodeType();
MOZ_ASSERT(IsAddiw(inst->InstructionBits()));
/*
* | imm[11:0] | rs1 | 000 | rd | 0011011 |
*/
inst->SetInstructionBits(((uint32_t)inst->InstructionBits() & ~kImm12Mask) |
(((uint32_t)(-val) & 0xfff) << kImm12Shift));
}
void MacroAssembler::flexibleDivMod32(Register lhs, Register rhs,
Register divOutput, Register remOutput,
bool isUnsigned, const LiveRegisterSet&) {
MOZ_ASSERT(lhs != divOutput && lhs != remOutput, "lhs is preserved");
MOZ_ASSERT(rhs != divOutput && rhs != remOutput, "rhs is preserved");
// The recommended code sequence to obtain both the quotient and remainder
// is div[u] followed by mod[u].
if (isUnsigned) {
ma_divu32(divOutput, lhs, rhs);
ma_modu32(remOutput, lhs, rhs);
} else {
ma_div32(divOutput, lhs, rhs);
ma_mod32(remOutput, lhs, rhs);
}
}
void MacroAssembler::flexibleQuotient32(Register lhs, Register rhs,
Register dest, bool isUnsigned,
const LiveRegisterSet&) {
quotient32(lhs, rhs, dest, isUnsigned);
}
void MacroAssembler::flexibleQuotientPtr(Register lhs, Register rhs,
Register dest, bool isUnsigned,
const LiveRegisterSet&) {
quotient64(lhs, rhs, dest, isUnsigned);
}
void MacroAssembler::flexibleRemainder32(Register lhs, Register rhs,
Register dest, bool isUnsigned,
const LiveRegisterSet&) {
remainder32(lhs, rhs, dest, isUnsigned);
}
void MacroAssembler::flexibleRemainderPtr(Register lhs, Register rhs,
Register dest, bool isUnsigned,
const LiveRegisterSet&) {
remainder64(lhs, rhs, dest, isUnsigned);
}
void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Round toward negative infinity.
Floor_l_d(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// Fail if the input is negative zero.
{
fclass_d(scratch, src);
ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
}
}
void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Round toward negative infinity.
Floor_l_s(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// Fail if the input is negative zero.
{
fclass_s(scratch, src);
ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
}
}
void MacroAssembler::flush() {}
void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask)));
loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
}
void MacroAssembler::moveValue(const ValueOperand& src,
const ValueOperand& dest) {
if (src == dest) {
return;
}
movePtr(src.valueReg(), dest.valueReg());
}
void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
if (!src.isGCThing()) {
ma_li(dest.valueReg(), ImmWord(src.asRawBits()));
return;
}
writeDataRelocation(src);
movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
}
void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
FloatRegister dest) {
MOZ_ASSERT(HasRoundInstruction(mode));
ScratchDoubleScope2 fscratch(*this);
switch (mode) {
case RoundingMode::Down:
Floor_d_d(dest, src, fscratch);
break;
case RoundingMode::Up:
Ceil_d_d(dest, src, fscratch);
break;
case RoundingMode::NearestTiesToEven:
Round_d_d(dest, src, fscratch);
break;
case RoundingMode::TowardsZero:
Trunc_d_d(dest, src, fscratch);
break;
}
}
void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
FloatRegister dest) {
MOZ_ASSERT(HasRoundInstruction(mode));
ScratchFloat32Scope2 fscratch(*this);
switch (mode) {
case RoundingMode::Down:
Floor_s_s(dest, src, fscratch);
break;
case RoundingMode::Up:
Ceil_s_s(dest, src, fscratch);
break;
case RoundingMode::NearestTiesToEven:
Round_s_s(dest, src, fscratch);
break;
case RoundingMode::TowardsZero:
Trunc_s_s(dest, src, fscratch);
break;
}
}
void MacroAssembler::oolWasmTruncateCheckF32ToI32(
FloatRegister input, Register output, TruncFlags flags,
const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
MOZ_ASSERT(!(flags & TRUNC_SATURATING));
Label notNaN;
BranchFloat32(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
bind(&notNaN);
wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
}
void MacroAssembler::oolWasmTruncateCheckF64ToI32(
FloatRegister input, Register output, TruncFlags flags,
const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
MOZ_ASSERT(!(flags & TRUNC_SATURATING));
Label notNaN;
BranchFloat64(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
bind(&notNaN);
wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
}
void MacroAssembler::oolWasmTruncateCheckF32ToI64(
FloatRegister input, Register64 output, TruncFlags flags,
const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
MOZ_ASSERT(!(flags & TRUNC_SATURATING));
Label notNaN;
BranchFloat32(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
bind(&notNaN);
wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
}
void MacroAssembler::oolWasmTruncateCheckF64ToI64(
FloatRegister input, Register64 output, TruncFlags flags,
const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
MOZ_ASSERT(!(flags & TRUNC_SATURATING));
Label notNaN;
BranchFloat64(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
bind(&notNaN);
wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
}
void MacroAssembler::patchCallToNop(uint8_t* call) {
uint32_t* p = reinterpret_cast<uint32_t*>(call) - 7;
*reinterpret_cast<Instr*>(p) = kNopByte;
*reinterpret_cast<Instr*>(p + 1) = kNopByte;
*reinterpret_cast<Instr*>(p + 2) = kNopByte;
*reinterpret_cast<Instr*>(p + 3) = kNopByte;
*reinterpret_cast<Instr*>(p + 4) = kNopByte;
*reinterpret_cast<Instr*>(p + 5) = kNopByte;
*reinterpret_cast<Instr*>(p + 6) = kNopByte;
}
CodeOffset MacroAssembler::callWithPatch() {
BlockTrampolinePoolScope block_trampoline_pool(this, 2);
DEBUG_PRINTF("\tcallWithPatch\n");
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
int32_t imm32 = 1 * sizeof(uint32_t);
int32_t Hi20 = ((imm32 + 0x800) >> 12);
int32_t Lo12 = imm32 << 20 >> 20;
auipc(scratch, Hi20); // Read PC + Hi20 into scratch.
jalr(scratch, Lo12); // jump PC + Hi20 + Lo12
DEBUG_PRINTF("\tret %d\n", currentOffset());
return CodeOffset(currentOffset());
}
void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
DEBUG_PRINTF("\tpatchCall\n");
BufferOffset call(callerOffset - 2 * sizeof(uint32_t));
DEBUG_PRINTF("\tcallerOffset %d\n", callerOffset);
int32_t offset = BufferOffset(calleeOffset).getOffset() - call.getOffset();
if (is_int32(offset)) {
Instruction* auipc_ = (Instruction*)editSrc(call);
Instruction* jalr_ = (Instruction*)editSrc(
BufferOffset(callerOffset - 1 * sizeof(uint32_t)));
DEBUG_PRINTF("\t%p %zu\n\t", auipc_, callerOffset - 2 * sizeof(uint32_t));
#ifdef JS_DISASM_RISCV64
disassembleInstr(auipc_->InstructionBits());
#endif /* JS_DISASM_RISCV64 */
DEBUG_PRINTF("\t%p %zu\n\t", jalr_, callerOffset - 1 * sizeof(uint32_t));
#ifdef JS_DISASM_RISCV64
disassembleInstr(jalr_->InstructionBits());
#endif /* JS_DISASM_RISCV64 */
DEBUG_PRINTF("\t\n");
MOZ_ASSERT(IsJalr(jalr_->InstructionBits()) &&
IsAuipc(auipc_->InstructionBits()));
MOZ_ASSERT(auipc_->RdValue() == jalr_->Rs1Value());
int32_t Hi20 = (((int32_t)offset + 0x800) >> 12);
int32_t Lo12 = (int32_t)offset << 20 >> 20;
putInstrAt(call, SetAuipcOffset(Hi20, auipc_->InstructionBits()));
putInstrAt(BufferOffset(callerOffset - 1 * sizeof(uint32_t)),
SetJalrOffset(Lo12, jalr_->InstructionBits()));
} else {
MOZ_CRASH();
}
}
void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
uint32_t* u32 = reinterpret_cast<uint32_t*>(
editSrc(BufferOffset(farJump.offset() + 4 * kInstrSize)));
MOZ_ASSERT(*u32 == UINT32_MAX);
*u32 = targetOffset - farJump.offset();
}
void MacroAssembler::patchFarJump(uint8_t* farJump, uint8_t* target) {
uint32_t* u32 = reinterpret_cast<uint32_t*>(farJump + 4 * kInstrSize);
MOZ_ASSERT(*u32 == UINT32_MAX);
*u32 = (int64_t)target - (int64_t)farJump;
}
void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
CodeLocationLabel target) {
PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
}
void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
uint32_t* p = reinterpret_cast<uint32_t*>(call) - 7;
Assembler::WriteLoad64Instructions((Instruction*)p, SavedScratchRegister,
(uint64_t)target);
DEBUG_PRINTF("\tpatchNopToCall %" PRIu64 " %" PRIu64 "\n", (uint64_t)target,
ExtractLoad64Value((Instruction*)p));
MOZ_ASSERT(ExtractLoad64Value((Instruction*)p) == (uint64_t)target);
Instr jalr_ = JALR | (ra.code() << kRdShift) | (0x0 << kFunct3Shift) |
(SavedScratchRegister.code() << kRs1Shift) |
(0x0 << kImm12Shift);
*reinterpret_cast<Instr*>(p + 6) = jalr_;
}
void MacroAssembler::Pop(Register reg) {
pop(reg);
adjustFrame(-int32_t(sizeof(intptr_t)));
}
void MacroAssembler::Pop(FloatRegister t) {
pop(t);
// See MacroAssemblerRiscv64::ma_pop(FloatRegister) for why we use
// sizeof(double).
adjustFrame(-int32_t(sizeof(double)));
}
void MacroAssembler::Pop(const ValueOperand& val) {
popValue(val);
adjustFrame(-int32_t(sizeof(Value)));
}
void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
LiveRegisterSet ignore) {
int32_t diff =
set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
const int32_t reserved = diff;
for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
diff -= sizeof(intptr_t);
if (!ignore.has(*iter)) {
loadPtr(Address(StackPointer, diff), *iter);
}
}
#ifdef ENABLE_WASM_SIMD
# error "Needs more careful logic if SIMD is enabled"
#endif
for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
iter.more(); ++iter) {
diff -= sizeof(double);
if (!ignore.has(*iter)) {
loadDouble(Address(StackPointer, diff), *iter);
}
}
MOZ_ASSERT(diff == 0);
freeStack(reserved);
}
CodeOffset MacroAssembler::move32WithPatch(Register dest) {
CodeOffset offs = CodeOffset(currentOffset());
ma_liPatchable(dest, Imm32(0));
return offs;
}
void MacroAssembler::patchMove32(CodeOffset offset, Imm32 n) {
patchSub32FromStackPtr(offset, n);
}
void MacroAssembler::pushReturnAddress() { push(ra); }
void MacroAssembler::popReturnAddress() { pop(ra); }
void MacroAssembler::PopStackPtr() {
loadPtr(Address(StackPointer, 0), StackPointer);
adjustFrame(-int32_t(sizeof(intptr_t)));
}
void MacroAssembler::freeStackTo(uint32_t framePushed) {
MOZ_ASSERT(framePushed <= framePushed_);
ma_sub64(StackPointer, FramePointer, Imm32(framePushed));
framePushed_ = framePushed;
}
void MacroAssembler::PushBoxed(FloatRegister reg) {
subFromStackPtr(Imm32(sizeof(double)));
boxDouble(reg, Address(getStackPointer(), 0));
adjustFrame(sizeof(double));
}
void MacroAssembler::Push(Register reg) {
push(reg);
adjustFrame(int32_t(sizeof(intptr_t)));
}
void MacroAssembler::Push(const Imm32 imm) {
push(imm);
adjustFrame(int32_t(sizeof(intptr_t)));
}
void MacroAssembler::Push(const ImmWord imm) {
push(imm);
adjustFrame(int32_t(sizeof(intptr_t)));
}
void MacroAssembler::Push(const ImmPtr imm) {
Push(ImmWord(uintptr_t(imm.value)));
}
void MacroAssembler::Push(const ImmGCPtr ptr) {
push(ptr);
adjustFrame(int32_t(sizeof(intptr_t)));
}
void MacroAssembler::Push(FloatRegister reg) {
push(reg);
// See MacroAssemblerRiscv64::ma_push(FloatRegister) for why we use
// sizeof(double).
adjustFrame(int32_t(sizeof(double)));
}
void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
int32_t diff =
set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
const int32_t reserved = diff;
reserveStack(reserved);
for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
diff -= sizeof(intptr_t);
storePtr(*iter, Address(StackPointer, diff));
}
#ifdef ENABLE_WASM_SIMD
# error "Needs more careful logic if SIMD is enabled"
#endif
for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
iter.more(); ++iter) {
diff -= sizeof(double);
storeDouble(*iter, Address(StackPointer, diff));
}
MOZ_ASSERT(diff == 0);
}
void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
FloatRegister temp, Label* fail) {
JitSpew(JitSpew_Codegen, "[ %s", __FUNCTION__);
Label negative, done;
// Branch to a slow path if input < 0.0 due to complicated rounding rules.
{
loadConstantFloat32(0.0f, temp);
BranchFloat32(Assembler::DoubleLessThan, src, temp, &negative, ShortJump);
}
// Fail if the input is negative zero.
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
fclass_s(scratch, src);
ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Assembler::Equal);
}
// Handle the simple case of a positive input and NaN.
// Rounding proceeds with consideration of the fractional part of the input:
// 1. If > 0.5, round to integer with higher absolute value (so, up).
// 2. If < 0.5, round to integer with lower absolute value (so, down).
// 3. If = 0.5, round to +Infinity (so, up).
{
// Round, ties away from zero.
RoundMaxMag_l_s(dest, src);
ma_branch(&done);
}
// Handle the complicated case of a negative input.
// Rounding proceeds with consideration of the fractional part of the input:
// 1. If > 0.5, round to integer with higher absolute value (so, down).
// 2. If < 0.5, round to integer with lower absolute value (so, up).
// 3. If = 0.5, round to +Infinity (so, up).
bind(&negative);
{
// Inputs in [-0.5, 0) are rounded to -0. Fail.
loadConstantFloat32(-0.5f, temp);
branchFloat(Assembler::DoubleGreaterThanOrEqual, src, temp, fail);
// Other negative inputs need the biggest float less than 0.5 added.
loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
fadd_s(temp, src, temp);
// Round toward negative infinity.
Floor_l_s(dest, temp);
}
// Sign extend lower 32 bits to test if the result isn't an Int32.
bind(&done);
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
JitSpew(JitSpew_Codegen, "]");
}
void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
FloatRegister temp, Label* fail) {
JitSpew(JitSpew_Codegen, "[ %s", __FUNCTION__);
Label negative, done;
// Branch to a slow path if input < 0.0 due to complicated rounding rules.
{
loadConstantDouble(0.0, temp);
BranchFloat64(Assembler::DoubleLessThan, src, temp, &negative, ShortJump);
}
// Fail if the input is negative zero.
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
fclass_d(scratch, src);
ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
}
// Handle the simple case of a positive input and NaN.
// Rounding proceeds with consideration of the fractional part of the input:
// 1. If > 0.5, round to integer with higher absolute value (so, up).
// 2. If < 0.5, round to integer with lower absolute value (so, down).
// 3. If = 0.5, round to +Infinity (so, up).
{
// Round, ties away from zero.
RoundMaxMag_l_d(dest, src);
ma_branch(&done);
}
// Handle the complicated case of a negative input.
// Rounding proceeds with consideration of the fractional part of the input:
// 1. If > 0.5, round to integer with higher absolute value (so, down).
// 2. If < 0.5, round to integer with lower absolute value (so, up).
// 3. If = 0.5, round to +Infinity (so, up).
bind(&negative);
{
// Inputs in [-0.5, 0) are rounded to -0. Fail.
loadConstantDouble(-0.5, temp);
branchDouble(Assembler::DoubleGreaterThanOrEqual, src, temp, fail);
// Other negative inputs need the biggest double less than 0.5 added.
loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
fadd_d(temp, src, temp);
// Round toward negative infinity.
Floor_l_d(dest, temp);
}
// Sign extend lower 32 bits to test if the result isn't an Int32.
bind(&done);
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
JitSpew(JitSpew_Codegen, "]");
}
void MacroAssembler::setupUnalignedABICall(Register scratch) {
MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
setupNativeABICall();
dynamicAlignment_ = true;
or_(scratch, StackPointer, zero);
// Force sp to be aligned
asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
storePtr(scratch, Address(StackPointer, 0));
}
void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
Register pointer) {
if (IsShiftInScaleRange(shift)) {
computeEffectiveAddress(
BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
return;
}
lshift32(Imm32(shift), indexTemp32);
addPtr(indexTemp32, pointer);
}
void MacroAssembler::speculationBarrier() { MOZ_CRASH(); }
void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
Register) {
FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
mozilla::DebugOnly<unsigned> numFpu = fpuSet.size();
int32_t diffF = fpuSet.getPushSizeInBytes();
mozilla::DebugOnly<int32_t> diffG = set.gprs().size() * sizeof(intptr_t);
MOZ_ASSERT(dest.offset >= diffG + diffF);
for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
diffG -= sizeof(intptr_t);
dest.offset -= sizeof(intptr_t);
storePtr(*iter, dest);
}
MOZ_ASSERT(diffG == 0);
#ifdef ENABLE_WASM_SIMD
# error "Needs more careful logic if SIMD is enabled"
#endif
for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
FloatRegister reg = *iter;
diffF -= reg.size();
numFpu -= 1;
dest.offset -= reg.size();
if (reg.isDouble()) {
storeDouble(reg, dest);
} else if (reg.isSingle()) {
storeFloat32(reg, dest);
} else {
MOZ_CRASH("Unknown register type.");
}
}
MOZ_ASSERT(numFpu == 0);
diffF -= diffF % sizeof(uintptr_t);
MOZ_ASSERT(diffF == 0);
}
void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(*this);
Register scratch = temps.Acquire();
// Round toward zero.
Trunc_l_d(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// We have to check for (-1, -0] when the result is zero.
Label notZero;
ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
{
fmv_x_d(scratch, src);
ma_b(scratch, scratch, fail, Assembler::Signed);
}
bind(&notZero);
}
void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
Label* fail) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
// Round toward zero.
Trunc_l_s(dest, src);
// Sign extend lower 32 bits to test if the result isn't an Int32.
{
move32SignExtendToPtr(dest, scratch);
branchPtr(Assembler::NotEqual, dest, scratch, fail);
}
// We have to check for (-1, -0] when the result is zero.
Label notZero;
ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
{
fmv_x_w(scratch, src);
ma_b(scratch, scratch, fail, Assembler::Signed);
}
bind(&notZero);
}
void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register value,
const Address& mem, Register valueTemp,
Register offsetTemp,
Register maskTemp) {
AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
valueTemp, offsetTemp, maskTemp);
}
void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register value,
const BaseIndex& mem,
Register valueTemp, Register offsetTemp,
Register maskTemp) {
AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
valueTemp, offsetTemp, maskTemp);
}
template <typename T>
static void WasmAtomicExchange64(MacroAssembler& masm,
const wasm::MemoryAccessDesc& access,
const T& mem, Register64 value,
Register64 output) {
AtomicExchange64(masm, &access, access.sync(), mem, value, output);
}
void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
const Address& mem, Register64 value,
Register64 output) {
WasmAtomicExchange64(*this, access, mem, value, output);
}
void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
const BaseIndex& mem,
Register64 value, Register64 output) {
WasmAtomicExchange64(*this, access, mem, value, output);
}
void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
const Address& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register output) {
AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
const BaseIndex& mem, Register value,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register output) {
AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register64 value,
const Address& mem, Register64 temp,
Register64 output) {
AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
}
void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register64 value,
const BaseIndex& mem, Register64 temp,
Register64 output) {
AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
}
void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op,
Register64 value, const Address& mem,
Register64 temp, Register64 output) {
AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
}
void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op,
Register64 value, const BaseIndex& mem,
Register64 temp, Register64 output) {
AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
}
void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
Register64 value, const Address& mem,
Register64 temp) {
AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
}
void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
Register64 value, const BaseIndex& mem,
Register64 temp) {
AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
}
void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register value,
const Address& mem, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
AtomicOp op, Register value,
const BaseIndex& mem, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
Register boundsCheckLimit,
Label* label) {
ma_b(index, boundsCheckLimit, label, cond);
}
void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
Address boundsCheckLimit, Label* label) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
load32(boundsCheckLimit, scratch2);
ma_b(index, Register(scratch2), label, cond);
}
void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
Register64 boundsCheckLimit,
Label* label) {
ma_b(index.reg, boundsCheckLimit.reg, label, cond);
}
void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
Address boundsCheckLimit, Label* label) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
loadPtr(boundsCheckLimit, scratch2);
ma_b(index.reg, scratch2, label, cond);
}
void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
const Address& mem,
Register64 expect,
Register64 replace,
Register64 output) {
CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
output);
}
void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
const BaseIndex& mem,
Register64 expect,
Register64 replace,
Register64 output) {
CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
output);
}
template <typename T>
static void CompareExchange(MacroAssembler& masm,
const wasm::MemoryAccessDesc* access,
Scalar::Type type, Synchronization sync,
const T& mem, Register oldval, Register newval,
Register valueTemp, Register offsetTemp,
Register maskTemp, Register output) {
bool signExtend = Scalar::isSignedIntType(type);
unsigned nbytes = Scalar::byteSize(type);
switch (nbytes) {
case 1:
case 2:
break;
case 4:
MOZ_ASSERT(valueTemp == InvalidReg);
MOZ_ASSERT(offsetTemp == InvalidReg);
MOZ_ASSERT(maskTemp == InvalidReg);
break;
default:
MOZ_CRASH();
}
Label again, end;
UseScratchRegisterScope temps(&masm);
Register scratch1 = temps.Acquire();
Register scratch2 = temps.Acquire();
masm.computeEffectiveAddress(mem, scratch2);
if (nbytes == 4) {
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, output, scratch2);
masm.SignExtendWord(scratch1, oldval);
masm.ma_b(output, scratch1, &end, Assembler::NotEqual, ShortJump);
masm.mv(scratch1, newval);
masm.sc_w(true, true, scratch1, scratch2, scratch1);
masm.ma_b(scratch1, scratch1, &again, Assembler::NonZero, ShortJump);
masm.memoryBarrierAfter(sync);
masm.bind(&end);
return;
}
masm.andi(offsetTemp, scratch2, 3);
masm.subPtr(offsetTemp, scratch2);
if constexpr (std::endian::native != std::endian::little) {
masm.as_xori(offsetTemp, offsetTemp, 3);
}
masm.slli(offsetTemp, offsetTemp, 3);
masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
masm.sll(maskTemp, maskTemp, offsetTemp);
masm.not_(maskTemp, maskTemp);
masm.memoryBarrierBefore(sync);
masm.bind(&again);
if (access) {
masm.append(*access, wasm::TrapMachineInsn::Atomic,
FaultingCodeOffset(masm.currentOffset()));
}
masm.lr_w(true, true, scratch1, scratch2);
masm.srl(output, scratch1, offsetTemp);
switch (nbytes) {
case 1:
if (signExtend) {
masm.SignExtendByte(valueTemp, oldval);
masm.SignExtendByte(output, output);
masm.SignExtendByte(newval, newval);
} else {
masm.andi(valueTemp, oldval, 0xff);
masm.andi(output, output, 0xff);
masm.andi(newval, newval, 0xff);
}
break;
case 2:
if (signExtend) {
masm.SignExtendShort(valueTemp, oldval);
masm.SignExtendShort(output, output);
masm.SignExtendShort(newval, newval);
} else {
UseScratchRegisterScope temps(&masm);
Register mask = temps.Acquire();
masm.ma_li(mask, Imm32(0xffff));
masm.and_(valueTemp, oldval, mask);
masm.and_(output, output, mask);
masm.and_(newval, newval, mask);
}
break;
}
masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump);
masm.sllw(valueTemp, newval, offsetTemp);
masm.and_(scratch1, scratch1, maskTemp);
masm.or_(scratch1, scratch1, valueTemp);
masm.sc_w(true, true, scratch1, scratch2, scratch1);
masm.ma_b(scratch1, scratch1, &again, Assembler::NonZero, ShortJump);
masm.memoryBarrierAfter(sync);
masm.bind(&end);
}
void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync,
const Address& mem, Register expected,
Register replacement, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
CompareExchange(*this, nullptr, type, sync, mem, expected, replacement,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync,
const BaseIndex& mem, Register expected,
Register replacement, Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
CompareExchange(*this, nullptr, type, sync, mem, expected, replacement,
valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
const Address& mem, Register expected,
Register replacement,
Register valueTemp,
Register offsetTemp, Register maskTemp,
Register output) {
CompareExchange(*this, &access, access.type(), access.sync(), mem, expected,
replacement, valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmCompareExchange(
const wasm::MemoryAccessDesc& access, const BaseIndex& mem,
Register expected, Register replacement, Register valueTemp,
Register offsetTemp, Register maskTemp, Register output) {
CompareExchange(*this, &access, access.type(), access.sync(), mem, expected,
replacement, valueTemp, offsetTemp, maskTemp, output);
}
void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
Register memoryBase, Register ptr,
Register ptrScratch, AnyRegister output) {
wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
}
void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
Register memoryBase, Register ptr,
Register ptrScratch, Register64 output) {
wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
}
void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
AnyRegister value, Register memoryBase,
Register ptr, Register ptrScratch) {
wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
}
void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
Register64 value, Register memoryBase,
Register ptr, Register ptrScratch) {
wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
}
void MacroAssemblerRiscv64::Clear_if_nan_d(Register rd, FPURegister fs) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
feq_d(scratch, fs, fs);
neg(scratch, scratch);
and_(rd, rd, scratch);
}
void MacroAssemblerRiscv64::Clear_if_nan_s(Register rd, FPURegister fs) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
feq_s(scratch, fs, fs);
neg(scratch, scratch);
and_(rd, rd, scratch);
}
void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
Register output,
bool isSaturating,
Label* oolEntry) {
if (isSaturating) {
Trunc_w_d(output, input);
Clear_if_nan_d(output, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_l_d(output, input);
// Sign extend lower 32 bits to test if the result isn't an Int32.
move32SignExtendToPtr(output, scratch);
branchPtr(Assembler::NotEqual, output, scratch, oolEntry);
}
}
void MacroAssembler::wasmTruncateDoubleToInt64(
FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
Label* oolRejoin, FloatRegister tempDouble) {
if (isSaturating) {
Trunc_l_d(output.reg, input);
Clear_if_nan_d(output.reg, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_l_d(output.reg, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
Register output,
bool isSaturating,
Label* oolEntry) {
if (isSaturating) {
Trunc_uw_d(output, input);
Clear_if_nan_d(output, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_uw_d(output, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
void MacroAssembler::wasmTruncateDoubleToUInt64(
FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
Label* oolRejoin, FloatRegister tempDouble) {
if (isSaturating) {
Trunc_ul_d(output.reg, input);
Clear_if_nan_d(output.reg, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_ul_d(output.reg, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
Register output,
bool isSaturating,
Label* oolEntry) {
if (isSaturating) {
Trunc_w_s(output, input);
Clear_if_nan_s(output, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_l_s(output, input, scratch);
// Sign extend lower 32 bits to test if the result isn't an Int32.
move32SignExtendToPtr(output, scratch);
branchPtr(Assembler::NotEqual, output, scratch, oolEntry);
}
}
void MacroAssembler::wasmTruncateFloat32ToInt64(
FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
Label* oolRejoin, FloatRegister tempDouble) {
if (isSaturating) {
Trunc_l_s(output.reg, input);
Clear_if_nan_s(output.reg, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_l_s(output.reg, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
Register output,
bool isSaturating,
Label* oolEntry) {
if (isSaturating) {
Trunc_uw_s(output, input);
Clear_if_nan_s(output, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_uw_s(output, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
void MacroAssembler::wasmTruncateFloat32ToUInt64(
FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
Label* oolRejoin, FloatRegister tempDouble) {
if (isSaturating) {
Trunc_ul_s(output.reg, input);
Clear_if_nan_s(output.reg, input);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Trunc_ul_s(output.reg, input, scratch);
ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
}
}
// TODO(riscv64): widenInt32 should be nop?
void MacroAssembler::widenInt32(Register r) {
move32To64SignExtend(r, Register64(r));
}
void MacroAssembler::wasmMarkCallAsSlow() { mv(ra, ra); }
const int32_t SlowCallMarker = 0x8093; // addi ra, ra, 0
void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow,
Register temp1, Register temp2) {
MOZ_ASSERT(ra_ != temp2);
UseScratchRegisterScope temps(*this);
// temp1 aliases ra_, so allocating a new register.
const Register scratchMarker = temps.Acquire();
move32(Imm32(SlowCallMarker), scratchMarker);
Label slow;
// Handle `jalr; (ra_ here) marker`.
load32(Address(ra_, 0), temp2);
branch32(Assembler::Equal, temp2, scratchMarker, &slow);
// Handle `jal; (ra_ here) nop; marker`.
// See also: AssemblerRISCVI::jal(Register rd, int32_t imm21); Bug 1996840
branch32(Assembler::NotEqual, temp2, Imm32(kNopByte), notSlow);
load32(Address(ra_, 4), temp2);
branch32(Assembler::NotEqual, temp2, scratchMarker, notSlow);
bind(&slow);
}
CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
const Register reg) {
BlockTrampolinePoolScope block_trampoline_pool(this, 2);
CodeOffset offset = call(desc, reg);
wasmMarkCallAsSlow();
return offset;
}
//}}} check_macroassembler_style
// This method generates lui, dsll and ori instruction block that can be
// modified by UpdateLoad64Value, either during compilation (eg.
// Assembler::bind), or during execution (eg. jit::PatchJump).
void MacroAssemblerRiscv64::ma_liPatchable(Register dest, Imm32 imm) {
m_buffer.ensureSpace(2 * sizeof(uint32_t));
int64_t value = imm.value;
int64_t high_20 = ((value + 0x800) >> 12);
int64_t low_12 = value << 52 >> 52;
lui(dest, high_20);
addi(dest, dest, low_12);
}
void MacroAssemblerRiscv64::ma_liPatchable(Register dest, ImmPtr imm) {
return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
}
void MacroAssemblerRiscv64::ma_liPatchable(Register dest, ImmWord imm,
LiFlags flags) {
DEBUG_PRINTF("\tma_liPatchable\n");
if (Li64 == flags) {
li_constant(dest, imm.value);
} else {
li_ptr(dest, imm.value);
}
}
void MacroAssemblerRiscv64::ma_li(Register dest, ImmGCPtr ptr) {
BlockTrampolinePoolScope block_trampoline_pool(this, 6);
writeDataRelocation(ptr);
ma_liPatchable(dest, ImmPtr(ptr.value));
}
void MacroAssemblerRiscv64::ma_li(Register dest, Imm32 imm) {
RV_li(dest, imm.value);
}
void MacroAssemblerRiscv64::ma_li(Register dest, Imm64 imm) {
RV_li(dest, imm.value);
}
void MacroAssemblerRiscv64::ma_li(Register dest, CodeLabel* label) {
DEBUG_PRINTF("[ %s\n", __FUNCTION__);
BlockTrampolinePoolScope block_trampoline_pool(this, 7);
BufferOffset bo = m_buffer.nextOffset();
JitSpew(JitSpew_Codegen, ".load CodeLabel %p", label);
ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
label->patchAt()->bind(bo.getOffset());
label->setLinkMode(CodeLabel::MoveImmediate);
DEBUG_PRINTF("]\n");
}
void MacroAssemblerRiscv64::ma_li(Register dest, ImmWord imm) {
RV_li(dest, imm.value);
}
// Shortcut for when we know we're transferring 32 bits of data.
void MacroAssemblerRiscv64::ma_pop(Register r) {
ld(r, StackPointer, 0);
addi(StackPointer, StackPointer, sizeof(intptr_t));
}
void MacroAssemblerRiscv64::ma_push(Register r) {
UseScratchRegisterScope temps(this);
if (r == sp) {
Register scratch = temps.Acquire();
// Pushing sp requires one more instruction.
mv(scratch, sp);
r = scratch;
}
addi(StackPointer, StackPointer, (int32_t)-sizeof(intptr_t));
sd(r, StackPointer, 0);
}
void MacroAssemblerRiscv64::ma_mul32TestOverflow(Register rd, Register rj,
Register rk, Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(rd != scratch);
mul(rd, rj, rk);
sext_w(scratch, rd);
ma_b(scratch, rd, overflow, Assembler::NotEqual);
}
void MacroAssemblerRiscv64::ma_mul32TestOverflow(Register rd, Register rj,
Imm32 imm, Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(rd != scratch && rj != scratch);
ma_li(scratch, imm);
mul(rd, rj, scratch);
sext_w(scratch, rd);
ma_b(scratch, rd, overflow, Assembler::NotEqual);
}
void MacroAssemblerRiscv64::ma_mulPtrTestOverflow(Register rd, Register rj,
Register rk,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Register scratch2 = temps.Acquire();
MOZ_ASSERT(rd != scratch);
if (rd == rj) {
or_(scratch, rj, zero);
rj = scratch;
rk = (rd == rk) ? rj : rk;
} else if (rd == rk) {
or_(scratch, rk, zero);
rk = scratch;
}
mul(rd, rj, rk);
mulh(scratch, rj, rk);
srai(scratch2, rd, 63);
ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
}
int32_t MacroAssemblerRiscv64::GetOffset(int32_t offset, Label* L,
OffsetSize bits) {
if (L) {
offset = branchOffsetHelper(L, bits);
} else {
MOZ_ASSERT(is_intn(offset, bits));
}
return offset;
}
bool MacroAssemblerRiscv64::CalculateOffset(Label* L, OffsetSize bits,
int32_t* offset) {
if (!isNear(L, bits)) return false;
*offset = GetOffset(*offset, L, bits);
return true;
}
void MacroAssemblerRiscv64::BranchShortHelper(int32_t offset, Label* L) {
MOZ_ASSERT(L == nullptr || offset == 0);
BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
offset = GetOffset(offset, L, OffsetSize::kOffset21);
Assembler::j(offset);
}
bool MacroAssemblerRiscv64::BranchShortHelper(int32_t offset, Label* L,
Condition cond, Register rs,
const Operand& rt) {
MOZ_ASSERT(L == nullptr || offset == 0);
MOZ_ASSERT(rt.is_reg() || rt.is_imm());
UseScratchRegisterScope temps(this);
Register scratch = Register();
if (rt.is_imm()) {
if (rt.immediate() == 0) {
scratch = zero;
} else {
scratch = temps.Acquire();
ma_li(scratch, Imm64(rt.immediate()));
}
} else {
MOZ_ASSERT(rt.is_reg());
scratch = rt.rm();
}
{
BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
switch (cond) {
case Always:
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
break;
case Equal:
// rs == rt
if (rt.is_reg() && rs == rt.rm()) {
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::beq(rs, scratch, offset);
}
break;
case NotEqual:
// rs != rt
if (rt.is_reg() && rs == rt.rm()) {
break; // No code needs to be emitted
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bne(rs, scratch, offset);
}
break;
// Signed comparison.
case GreaterThan:
// rs > rt
if (rt.is_reg() && rs == rt.rm()) {
break; // No code needs to be emitted.
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bgt(rs, scratch, offset);
}
break;
case GreaterThanOrEqual:
// rs >= rt
if (rt.is_reg() && rs == rt.rm()) {
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bge(rs, scratch, offset);
}
break;
case LessThan:
// rs < rt
if (rt.is_reg() && rs == rt.rm()) {
break; // No code needs to be emitted.
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::blt(rs, scratch, offset);
}
break;
case LessThanOrEqual:
// rs <= rt
if (rt.is_reg() && rs == rt.rm()) {
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::ble(rs, scratch, offset);
}
break;
// Unsigned comparison.
case Above:
// rs > rt
if (rt.is_reg() && rs == rt.rm()) {
break; // No code needs to be emitted.
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bgtu(rs, scratch, offset);
}
break;
case AboveOrEqual:
// rs >= rt
if (rt.is_reg() && rs == rt.rm()) {
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bgeu(rs, scratch, offset);
}
break;
case Below:
// rs < rt
if (rt.is_reg() && rs == rt.rm()) {
break; // No code needs to be emitted.
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
bltu(rs, scratch, offset);
}
break;
case BelowOrEqual:
// rs <= rt
if (rt.is_reg() && rs == rt.rm()) {
if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
Assembler::j(offset);
} else {
if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
Assembler::bleu(rs, scratch, offset);
}
break;
default:
MOZ_CRASH("UNREACHABLE");
}
}
return true;
}
// BRANCH_ARGS_CHECK checks that conditional jump arguments are correct.
#define BRANCH_ARGS_CHECK(cond, rs, rt) \
MOZ_ASSERT(((cond) == Always && (rs) == zero && (rt).rm() == zero) || \
((cond) != Always && ((rs) != zero || (rt).rm() != zero)))
bool MacroAssemblerRiscv64::BranchShortCheck(int32_t offset, Label* L,
Condition cond, Register rs,
const Operand& rt) {
BRANCH_ARGS_CHECK(cond, rs, rt);
if (!L) {
MOZ_ASSERT(is_int13(offset));
return BranchShortHelper(offset, nullptr, cond, rs, rt);
}
MOZ_ASSERT(offset == 0);
return BranchShortHelper(0, L, cond, rs, rt);
}
void MacroAssemblerRiscv64::BranchShort(Label* L) { BranchShortHelper(0, L); }
bool MacroAssemblerRiscv64::BranchShort(int32_t offset, Condition cond,
Register rs, const Operand& rt) {
return BranchShortCheck(offset, nullptr, cond, rs, rt);
}
bool MacroAssemblerRiscv64::BranchShort(Label* L, Condition cond, Register rs,
const Operand& rt) {
return BranchShortCheck(0, L, cond, rs, rt);
}
void MacroAssemblerRiscv64::BranchLong(Label* L) {
// Generate position independent long branch.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
int32_t imm = branchLongOffsetHelper(L);
GenPCRelativeJump(scratch, imm);
}
CodeOffset MacroAssemblerRiscv64::BranchAndLinkLong(Label* L) {
// Generate position independent long branch and link.
int32_t imm = branchLongOffsetHelper(L);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
GenPCRelativeJumpAndLink(scratch, imm);
return CodeOffset(currentOffset());
}
void MacroAssemblerRiscv64::ma_branch(Label* target, Condition cond,
Register r1, const Operand& r2,
JumpKind jumpKind) {
// Always prefer short jumps when the label is already bound. (If the label is
// bound, BranchShort can cheaply determine if short jumps are possible.)
if (target->bound()) {
jumpKind = ShortJump;
}
if (jumpKind == ShortJump && BranchShort(target, cond, r1, r2)) {
return;
}
if (cond != Always) {
Label skip;
Condition neg_cond = InvertCondition(cond);
MOZ_ALWAYS_TRUE(
BranchShort(&skip, neg_cond, r1, r2)); // Guaranteed to be short.
BranchLong(target);
bind(&skip);
} else {
BranchLong(target);
}
}
// Branches when done from within riscv code.
void MacroAssemblerRiscv64::ma_b(Register lhs, Address addr, Label* label,
Condition c, JumpKind jumpKind) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(lhs != scratch);
ma_load(scratch, addr, SizeDouble);
ma_b(lhs, Register(scratch), label, c, jumpKind);
}
void MacroAssemblerRiscv64::ma_b(Register lhs, ImmPtr imm, Label* l,
Condition c, JumpKind jumpKind) {
ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind);
}
// Branches when done from within riscv code.
void MacroAssemblerRiscv64::ma_b(Register lhs, ImmWord imm, Label* label,
Condition c, JumpKind jumpKind) {
switch (c) {
case Always:
ma_branch(label, c, zero, Operand(zero), jumpKind);
break;
case Zero:
case NonZero:
case Signed:
case NotSigned:
MOZ_ASSERT(imm.value == 0);
ma_b(lhs, lhs, label, c, jumpKind);
break;
default:
ma_branch(label, c, lhs, Operand(imm.value), jumpKind);
break;
}
}
void MacroAssemblerRiscv64::ma_b(Register lhs, Imm32 imm, Label* label,
Condition c, JumpKind jumpKind) {
switch (c) {
case Always:
ma_branch(label, c, zero, Operand(zero), jumpKind);
break;
case Zero:
case NonZero:
case Signed:
case NotSigned:
MOZ_ASSERT(imm.value == 0);
ma_b(lhs, lhs, label, c, jumpKind);
break;
default:
ma_branch(label, c, lhs, Operand(imm.value), jumpKind);
break;
}
}
void MacroAssemblerRiscv64::ma_b(Address addr, Imm32 imm, Label* label,
Condition c, JumpKind jumpKind) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_load(scratch2, addr);
ma_b(Register(scratch2), imm, label, c, jumpKind);
}
void MacroAssemblerRiscv64::ma_b(Register lhs, Register rhs, Label* label,
Condition c, JumpKind jumpKind) {
switch (c) {
case Always:
ma_branch(label, c, zero, Operand(zero), jumpKind);
break;
case Zero:
MOZ_ASSERT(lhs == rhs);
ma_branch(label, Equal, lhs, Operand(zero), jumpKind);
break;
case NonZero:
MOZ_ASSERT(lhs == rhs);
ma_branch(label, NotEqual, lhs, Operand(zero), jumpKind);
break;
case Signed:
MOZ_ASSERT(lhs == rhs);
ma_branch(label, LessThan, lhs, Operand(zero), jumpKind);
break;
case NotSigned:
MOZ_ASSERT(lhs == rhs);
ma_branch(label, GreaterThanOrEqual, lhs, Operand(zero), jumpKind);
break;
default:
ma_branch(label, c, lhs, rhs, jumpKind);
break;
}
}
void MacroAssemblerRiscv64::ExtractBits(Register rt, Register rs, uint16_t pos,
uint16_t size, bool sign_extend) {
#if JS_CODEGEN_RISCV64
constexpr uint16_t MaxBits = 64;
#elif JS_CODEGEN_RISCV32
constexpr uint16_t MaxBits = 32;
#endif
MOZ_ASSERT(pos < MaxBits);
MOZ_ASSERT(size > 0);
MOZ_ASSERT(size <= MaxBits);
MOZ_ASSERT((pos + size) > 0);
MOZ_ASSERT((pos + size) <= MaxBits);
Register src;
if (uint16_t shift = MaxBits - (pos + size)) {
slli(rt, rs, shift);
src = rt;
} else {
src = rs;
}
if (sign_extend) {
srai(rt, src, MaxBits - size);
} else {
srli(rt, src, MaxBits - size);
}
}
void MacroAssemblerRiscv64::InsertBits(Register dest, Register source, int pos,
int size) {
#if JS_CODEGEN_RISCV64
MOZ_ASSERT(size < 64);
#elif JS_CODEGEN_RISCV32
MOZ_ASSERT(size < 32);
#endif
UseScratchRegisterScope temps(this);
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
Register source_ = temps.Acquire();
if (pos != 0) {
Register mask = temps.Acquire();
// Create a mask of the length=size.
ma_li(mask, Imm32(1));
slli(mask, mask, size);
addi(mask, mask, -1);
and_(source_, mask, source);
slli(source_, source_, pos);
// Make a mask containing 0's. 0's start at "pos" with length=size.
slli(mask, mask, pos);
not_(mask, mask);
// cut area for insertion of source.
and_(dest, mask, dest);
} else {
// clear top bits from source and bottom bits from dest.
slli(source_, source, 64 - size);
srli(source_, source_, 64 - size);
srli(dest, dest, size);
slli(dest, dest, size);
}
// insert source
or_(dest, dest, source_);
}
void MacroAssemblerRiscv64::InsertBits(Register dest, Register source,
Register pos, int size) {
#if JS_CODEGEN_RISCV64
MOZ_ASSERT(size < 64);
#elif JS_CODEGEN_RISCV32
MOZ_ASSERT(size < 32);
#endif
UseScratchRegisterScope temps(this);
Register mask = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
Register source_ = temps.Acquire();
// Create a mask of the length=size.
ma_li(mask, Imm32(1));
slli(mask, mask, size);
addi(mask, mask, -1);
and_(source_, mask, source);
sll(source_, source_, pos);
// Make a mask containing 0's. 0's start at "pos" with length=size.
sll(mask, mask, pos);
not_(mask, mask);
// cut area for insertion of source.
and_(dest, mask, dest);
// insert source
or_(dest, dest, source_);
}
void MacroAssemblerRiscv64::ma_add32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(rt.immediate())) {
addiw(rd, rs, static_cast<int32_t>(rt.immediate()));
} else if ((-4096 <= rt.immediate() && rt.immediate() <= -2049) ||
(2048 <= rt.immediate() && rt.immediate() <= 4094)) {
addiw(rd, rs, rt.immediate() / 2);
addiw(rd, rd, rt.immediate() - (rt.immediate() / 2));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
addw(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
addw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_add64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(rt.immediate())) {
addi(rd, rs, static_cast<int32_t>(rt.immediate()));
} else if ((-4096 <= rt.immediate() && rt.immediate() <= -2049) ||
(2048 <= rt.immediate() && rt.immediate() <= 4094)) {
addi(rd, rs, rt.immediate() / 2);
addi(rd, rd, rt.immediate() - (rt.immediate() / 2));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
add(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
add(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_sub32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(-rt.immediate())) {
addiw(rd, rs,
static_cast<int32_t>(
-rt.immediate())); // No subi instr, use addi(x, y, -imm).
} else if ((-4096 <= -rt.immediate() && -rt.immediate() <= -2049) ||
(2048 <= -rt.immediate() && -rt.immediate() <= 4094)) {
addiw(rd, rs, -rt.immediate() / 2);
addiw(rd, rd, -rt.immediate() - (-rt.immediate() / 2));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
subw(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
subw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_sub64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(-rt.immediate())) {
addi(rd, rs,
static_cast<int32_t>(
-rt.immediate())); // No subi instr, use addi(x, y, -imm).
} else if ((-4096 <= -rt.immediate() && -rt.immediate() <= -2049) ||
(2048 <= -rt.immediate() && -rt.immediate() <= 4094)) {
addi(rd, rs, -rt.immediate() / 2);
addi(rd, rd, -rt.immediate() - (-rt.immediate() / 2));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
sub(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
sub(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_and(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(rt.immediate())) {
andi(rd, rs, rt.immediate());
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
and_(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
and_(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_or(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(rt.immediate())) {
ori(rd, rs, rt.immediate());
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
or_(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
or_(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_xor(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
if (is_int12(rt.immediate())) {
xori(rd, rs, rt.immediate());
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
xor_(rd, rs, scratch);
}
} else {
MOZ_ASSERT(rt.is_reg());
xor_(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_nor(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
nor(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
nor(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_div32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
divw(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
divw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_divu32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
divuw(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
divuw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_div64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
div(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
div(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_divu64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
divu(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
divu(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_mod32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
remw(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
remw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_modu32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
remuw(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
remuw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_mod64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
rem(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
rem(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_modu64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
remu(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
remu(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_mul32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
mulw(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
mulw(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_mulh32(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
mul(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
mul(rd, rs, rt.rm());
}
srai(rd, rd, 32);
}
void MacroAssemblerRiscv64::ma_mul64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
mul(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
mul(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_mulh64(Register rd, Register rs, Operand rt) {
if (rt.is_imm()) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, rt.immediate());
mulh(rd, rs, scratch);
} else {
MOZ_ASSERT(rt.is_reg());
mulh(rd, rs, rt.rm());
}
}
void MacroAssemblerRiscv64::ma_sll64(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sll(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
slli(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_sll32(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sllw(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
slliw(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_sra64(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sra(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
srai(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_sra32(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sraw(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
sraiw(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_srl64(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
srl(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
srli(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_srl32(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
srlw(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
uint8_t shamt = static_cast<uint8_t>(rt.immediate());
srliw(rd, rs, shamt);
}
}
void MacroAssemblerRiscv64::ma_slt(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
slt(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
if (is_int12(rt.immediate())) {
slti(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
slt(rd, rs, scratch);
}
}
}
void MacroAssemblerRiscv64::ma_sltu(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sltu(rd, rs, rt.rm());
} else {
MOZ_ASSERT(rt.is_imm());
if (is_int12(rt.immediate())) {
sltiu(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
sltu(rd, rs, scratch);
}
}
}
void MacroAssemblerRiscv64::ma_sle(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
slt(rd, rt.rm(), rs);
} else {
MOZ_ASSERT(rt.is_imm());
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
slt(rd, scratch, rs);
}
xori(rd, rd, 1);
}
void MacroAssemblerRiscv64::ma_sleu(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sltu(rd, rt.rm(), rs);
} else {
MOZ_ASSERT(rt.is_imm());
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
sltu(rd, scratch, rs);
}
xori(rd, rd, 1);
}
void MacroAssemblerRiscv64::ma_sgt(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
slt(rd, rt.rm(), rs);
} else {
MOZ_ASSERT(rt.is_imm());
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
slt(rd, scratch, rs);
}
}
void MacroAssemblerRiscv64::ma_sgtu(Register rd, Register rs, Operand rt) {
if (rt.is_reg()) {
sltu(rd, rt.rm(), rs);
} else {
MOZ_ASSERT(rt.is_imm());
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
BlockTrampolinePoolScope block_trampoline_pool(this, 9);
ma_li(scratch, rt.immediate());
sltu(rd, scratch, rs);
}
}
void MacroAssemblerRiscv64::ma_sge(Register rd, Register rs, Operand rt) {
ma_slt(rd, rs, rt);
xori(rd, rd, 1);
}
void MacroAssemblerRiscv64::ma_sgeu(Register rd, Register rs, Operand rt) {
ma_sltu(rd, rs, rt);
xori(rd, rd, 1);
}
static inline bool IsZero(const Operand& rt) {
if (rt.is_reg()) {
return rt.rm() == zero_reg;
}
MOZ_ASSERT(rt.is_imm());
return rt.immediate() == 0;
}
void MacroAssemblerRiscv64::ma_seq(Register rd, Register rs, Operand rt) {
if (rs == zero_reg) {
ma_seqz(rd, rt);
} else if (IsZero(rt)) {
seqz(rd, rs);
} else {
ma_sub64(rd, rs, rt);
seqz(rd, rd);
}
}
void MacroAssemblerRiscv64::ma_sne(Register rd, Register rs, Operand rt) {
if (rs == zero_reg) {
ma_snez(rd, rt);
} else if (IsZero(rt)) {
snez(rd, rs);
} else {
ma_sub64(rd, rs, rt);
snez(rd, rd);
}
}
void MacroAssemblerRiscv64::ma_seqz(Register rd, const Operand& rt) {
if (rt.is_reg()) {
seqz(rd, rt.rm());
} else {
ma_li(rd, rt.immediate() == 0);
}
}
void MacroAssemblerRiscv64::ma_snez(Register rd, const Operand& rt) {
if (rt.is_reg()) {
snez(rd, rt.rm());
} else {
ma_li(rd, rt.immediate() != 0);
}
}
void MacroAssemblerRiscv64::ma_neg(Register rd, const Operand& rt) {
MOZ_ASSERT(rt.is_reg());
neg(rd, rt.rm());
}
void MacroAssemblerRiscv64::ma_jump(ImmPtr dest) {
DEBUG_PRINTF("[ %s\n", __FUNCTION__);
BlockTrampolinePoolScope block_trampoline_pool(this, 8);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_liPatchable(scratch, dest);
jr(scratch, 0);
DEBUG_PRINTF("]\n");
}
// fp instructions
void MacroAssemblerRiscv64::ma_lid(FloatRegister dest, double value) {
ImmWord imm(mozilla::BitwiseCast<uint64_t>(value));
if (imm.value != 0) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
fmv_d_x(dest, scratch);
} else {
fmv_d_x(dest, zero);
}
}
// fp instructions
void MacroAssemblerRiscv64::ma_lis(FloatRegister dest, float value) {
Imm32 imm(mozilla::BitwiseCast<uint32_t>(value));
if (imm.value != 0) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, imm);
fmv_w_x(dest, scratch);
} else {
fmv_w_x(dest, zero);
}
}
void MacroAssemblerRiscv64::ma_sub32TestOverflow(Register rd, Register rj,
Register rk, Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
sub(scratch, rj, rk);
subw(rd, rj, rk);
ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
}
void MacroAssemblerRiscv64::ma_sub32TestOverflow(Register rd, Register rj,
Imm32 imm, Label* overflow) {
if (imm.value != INT32_MIN) {
ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(rj != scratch);
ma_li(scratch, Imm32(imm.value));
ma_sub32TestOverflow(rd, rj, scratch, overflow);
}
}
void MacroAssemblerRiscv64::ma_add32TestOverflow(Register rd, Register rj,
Register rk, Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
add(scratch, rj, rk);
addw(rd, rj, rk);
ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
}
void MacroAssemblerRiscv64::ma_add32TestOverflow(Register rd, Register rj,
Imm32 imm, Label* overflow) {
// Check for signed range because of addi
if (is_intn(imm.value, 12)) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
addi(scratch, rj, imm.value);
addiw(rd, rj, imm.value);
ma_b(rd, scratch, overflow, Assembler::NotEqual);
} else {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
ma_li(scratch2, imm);
ma_add32TestOverflow(rd, rj, scratch2, overflow);
}
}
void MacroAssemblerRiscv64::ma_subPtrTestOverflow(Register rd, Register rj,
Register rk,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
MOZ_ASSERT_IF(rj == rd, rj != rk);
MOZ_ASSERT(rj != scratch2);
MOZ_ASSERT(rk != scratch2);
MOZ_ASSERT(rd != scratch2);
Register rj_copy = rj;
if (rj == rd) {
ma_or(scratch2, rj, zero);
rj_copy = scratch2;
}
{
Register scratch = temps.Acquire();
MOZ_ASSERT(rd != scratch);
sub(rd, rj, rk);
// If the sign of rj and rk are the same, no overflow
ma_xor(scratch, rj_copy, rk);
// Check if the sign of rd and rj are the same
ma_xor(scratch2, rd, rj_copy);
ma_and(scratch2, scratch2, scratch);
}
ma_b(scratch2, zero, overflow, Assembler::LessThan);
}
void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
Register rk,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(rd != scratch);
if (rj == rk) {
if (rj == rd) {
ma_or(scratch, rj, zero);
rj = scratch;
}
add(rd, rj, rj);
ma_xor(scratch, rj, rd);
ma_b(scratch, zero, overflow, Assembler::LessThan);
} else {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
MOZ_ASSERT(rj != scratch);
MOZ_ASSERT(rd != scratch2);
if (rj == rd) {
ma_or(scratch2, rj, zero);
rj = scratch2;
}
add(rd, rj, rk);
slti(scratch, rj, 0);
slt(scratch2, rd, rk);
ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
}
}
void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
Imm32 imm, Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
if (imm.value == 0) {
ori(rd, rj, 0);
return;
}
if (rj == rd) {
ori(scratch2, rj, 0);
rj = scratch2;
}
ma_add64(rd, rj, imm);
if (imm.value > 0) {
ma_b(rd, rj, overflow, Assembler::LessThan);
} else {
MOZ_ASSERT(imm.value < 0);
ma_b(rd, rj, overflow, Assembler::GreaterThan);
}
}
void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
ImmWord imm,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
if (imm.value == 0) {
ori(rd, rj, 0);
return;
}
if (rj == rd) {
MOZ_ASSERT(rj != scratch2);
ori(scratch2, rj, 0);
rj = scratch2;
}
ma_li(rd, imm);
add(rd, rj, rd);
if (imm.value > 0) {
ma_b(rd, rj, overflow, Assembler::LessThan);
} else {
MOZ_ASSERT(imm.value < 0);
ma_b(rd, rj, overflow, Assembler::GreaterThan);
}
}
void MacroAssemblerRiscv64::ma_add32TestCarry(Condition cond, Register rd,
Register rj, Register rk,
Label* overflow) {
MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
MOZ_ASSERT_IF(rd == rj, rk != rd);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
addw(rd, rj, rk);
sltu(scratch, rd, rd == rj ? rk : rj);
ma_b(Register(scratch), Register(scratch), overflow,
cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
}
void MacroAssemblerRiscv64::ma_add32TestCarry(Condition cond, Register rd,
Register rj, Imm32 imm,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
MOZ_ASSERT(rj != scratch2);
ma_li(scratch2, imm);
ma_add32TestCarry(cond, rd, rj, scratch2, overflow);
}
void MacroAssemblerRiscv64::ma_subPtrTestOverflow(Register rd, Register rj,
Imm32 imm, Label* overflow) {
// TODO(riscv): Check subPtrTestOverflow
MOZ_ASSERT(imm.value != INT32_MIN);
ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow);
}
void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
Register rj, Register rk,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(rd != rk);
MOZ_ASSERT(rd != scratch);
add(rd, rj, rk);
sltu(scratch, rd, rk);
ma_b(scratch, Register(scratch), overflow,
cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
}
void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
Register rj, Imm32 imm,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
// Check for signed range because of addi
if (is_intn(imm.value, 12)) {
addi(rd, rj, imm.value);
sltiu(scratch2, rd, imm.value);
ma_b(scratch2, scratch2, overflow,
cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
} else {
ma_li(scratch2, imm);
ma_addPtrTestCarry(cond, rd, rj, scratch2, overflow);
}
}
void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
Register rj, ImmWord imm,
Label* overflow) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
// Check for signed range because of addi_d
if (is_intn(imm.value, 12)) {
uint32_t value = imm.value;
addi(rd, rj, value);
ma_sltu(scratch2, rd, Operand(value));
ma_b(scratch2, scratch2, overflow,
cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
} else {
ma_li(scratch2, imm);
ma_addPtrTestCarry(cond, rd, rj, scratch2, overflow);
}
}
void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
Register rj, Register rk,
Label* taken) {
MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
ma_add64(rd, rj, rk);
ma_b(rd, rd, taken, cond);
}
void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
Register rj, Imm32 imm,
Label* taken) {
MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
ma_add64(rd, rj, imm);
ma_b(rd, rd, taken, cond);
}
void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
Register rj, ImmWord imm,
Label* taken) {
MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
ma_add64(rd, rj, Operand(imm.value));
ma_b(rd, rd, taken, cond);
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_load(
Register dest, const BaseIndex& src, LoadStoreSize size,
LoadStoreExtension extension) {
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
computeScaledAddress(src, scratch2);
return ma_load(dest, Address(scratch2, src.offset), size, extension);
}
void MacroAssemblerRiscv64::ma_pop(FloatRegister f) {
if (f.isDouble()) {
fld(f, StackPointer, 0);
} else {
MOZ_ASSERT(f.isSingle(), "simd128 is not supported");
flw(f, StackPointer, 0);
}
// See also MacroAssemblerRiscv64::ma_push -- Free space for double even when
// storing a float.
addi(StackPointer, StackPointer, sizeof(double));
}
void MacroAssemblerRiscv64::ma_push(FloatRegister f) {
// We allocate space for double even when storing a float.
addi(StackPointer, StackPointer, (int32_t)-sizeof(double));
if (f.isDouble()) {
fsd(f, StackPointer, 0);
} else {
MOZ_ASSERT(f.isSingle(), "simd128 is not supported");
fsw(f, StackPointer, 0);
}
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fld_s(FloatRegister ft,
Address address) {
int32_t offset = address.offset;
Register base = address.base;
FaultingCodeOffset fco;
if (is_intn(offset, 12)) {
fco = FaultingCodeOffset(currentOffset());
flw(ft, base, offset);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(base != scratch);
ma_li(scratch, Imm32(offset));
ma_add64(scratch, base, scratch);
fco = FaultingCodeOffset(currentOffset());
flw(ft, scratch, 0);
}
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fld_d(FloatRegister ft,
Address address) {
int32_t offset = address.offset;
Register base = address.base;
FaultingCodeOffset fco;
if (is_intn(offset, 12)) {
fco = FaultingCodeOffset(currentOffset());
fld(ft, base, offset);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(base != scratch);
ma_li(scratch, Imm32(offset));
ma_add64(scratch, base, scratch);
fco = FaultingCodeOffset(currentOffset());
fld(ft, scratch, 0);
}
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_d(FloatRegister ft,
Address address) {
int32_t offset = address.offset;
Register base = address.base;
FaultingCodeOffset fco;
if (is_intn(offset, 12)) {
fco = FaultingCodeOffset(currentOffset());
fsd(ft, base, offset);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(base != scratch);
ma_li(scratch, Imm32(offset));
ma_add64(scratch, base, scratch);
fco = FaultingCodeOffset(currentOffset());
fsd(ft, scratch, 0);
}
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_s(FloatRegister ft,
Address address) {
int32_t offset = address.offset;
Register base = address.base;
FaultingCodeOffset fco;
if (is_intn(offset, 12)) {
fco = FaultingCodeOffset(currentOffset());
fsw(ft, base, offset);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
MOZ_ASSERT(base != scratch);
ma_li(scratch, Imm32(offset));
ma_add64(scratch, base, scratch);
fco = FaultingCodeOffset(currentOffset());
fsw(ft, scratch, 0);
}
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_d(FloatRegister src,
BaseIndex address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(address, scratch);
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
ma_fst_d(src, Address(scratch, address.offset));
return fco;
}
FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_s(FloatRegister src,
BaseIndex address) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(address, scratch);
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
ma_fst_s(src, Address(scratch, address.offset));
return fco;
}
void MacroAssemblerRiscv64::ma_fld_d(FloatRegister dest, const BaseIndex& src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(src, scratch);
ma_fld_d(dest, Address(scratch, src.offset));
}
void MacroAssemblerRiscv64::ma_fld_s(FloatRegister dest, const BaseIndex& src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
computeScaledAddress(src, scratch);
ma_fld_s(dest, Address(scratch, src.offset));
}
void MacroAssemblerRiscv64::ma_call(ImmPtr dest) {
DEBUG_PRINTF("[ %s\n", __FUNCTION__);
BlockTrampolinePoolScope block_trampoline_pool(this, 8);
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
ma_liPatchable(CallReg, dest);
jalr(CallReg, 0);
DEBUG_PRINTF("]\n");
}
void MacroAssemblerRiscv64::CompareIsNotNanF32(Register rd, FPURegister cmp1,
FPURegister cmp2) {
feq_s(rd, cmp1, cmp1); // rd <- !isNan(cmp1)
if (cmp1 != cmp2) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
feq_s(scratch, cmp2, cmp2); // scratch <- !isNaN(cmp2)
ma_and(rd, rd, scratch); // rd <- !isNan(cmp1) && !isNan(cmp2)
}
}
void MacroAssemblerRiscv64::CompareIsNotNanF64(Register rd, FPURegister cmp1,
FPURegister cmp2) {
feq_d(rd, cmp1, cmp1); // rd <- !isNan(cmp1)
if (cmp1 != cmp2) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
feq_d(scratch, cmp2, cmp2); // scratch <- !isNaN(cmp2)
ma_and(rd, rd, scratch); // rd <- !isNan(cmp1) && !isNan(cmp2)
}
}
void MacroAssemblerRiscv64::CompareIsNanF32(Register rd, FPURegister cmp1,
FPURegister cmp2) {
CompareIsNotNanF32(rd, cmp1, cmp2); // rd <- !isNan(cmp1) && !isNan(cmp2)
ma_xor(rd, rd, Operand(1)); // rd <- isNan(cmp1) || isNan(cmp2)
}
void MacroAssemblerRiscv64::CompareIsNanF64(Register rd, FPURegister cmp1,
FPURegister cmp2) {
CompareIsNotNanF64(rd, cmp1, cmp2); // rd <- !isNan(cmp1) && !isNan(cmp2)
ma_xor(rd, rd, Operand(1)); // rd <- isNan(cmp1) || isNan(cmp2)
}
void MacroAssemblerRiscv64::BranchFloat32(DoubleCondition cc,
FloatRegister frs1,
FloatRegister frs2, Label* L,
JumpKind jumpKind) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_compareF32(scratch, cc, frs1, frs2);
ma_b(scratch, Imm32(0), L, NotEqual, jumpKind);
}
void MacroAssemblerRiscv64::BranchFloat64(DoubleCondition cc,
FloatRegister frs1,
FloatRegister frs2, Label* L,
JumpKind jumpKind) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_compareF64(scratch, cc, frs1, frs2);
ma_b(scratch, Imm32(0), L, NotEqual, jumpKind);
}
void MacroAssemblerRiscv64::Clz32(Register rd, Register rs) {
if (HasZbbExtension()) {
#if JS_CODEGEN_RISCV64
clzw(rd, rs);
#else
clz(rd, rs);
#endif
return;
}
// 32 bit unsigned in lower word: count number of leading zeros.
// int n = 32;
// unsigned y;
// y = x >>16; if (y != 0) { n = n -16; x = y; }
// y = x >> 8; if (y != 0) { n = n - 8; x = y; }
// y = x >> 4; if (y != 0) { n = n - 4; x = y; }
// y = x >> 2; if (y != 0) { n = n - 2; x = y; }
// y = x >> 1; if (y != 0) {rd = n - 2; return;}
// rd = n - x;
Label L0, L1, L2, L3, L4;
UseScratchRegisterScope temps(this);
Register x = rd;
Register y = temps.Acquire();
Register n = temps.Acquire();
MOZ_ASSERT(rs != y && rs != n);
mv(x, rs);
ma_li(n, Imm32(32));
#if JS_CODEGEN_RISCV64
srliw(y, x, 16);
ma_branch(&L0, Equal, y, Operand(zero_reg));
mv(x, y);
addiw(n, n, -16);
bind(&L0);
srliw(y, x, 8);
ma_branch(&L1, Equal, y, Operand(zero_reg));
addiw(n, n, -8);
mv(x, y);
bind(&L1);
srliw(y, x, 4);
ma_branch(&L2, Equal, y, Operand(zero_reg));
addiw(n, n, -4);
mv(x, y);
bind(&L2);
srliw(y, x, 2);
ma_branch(&L3, Equal, y, Operand(zero_reg));
addiw(n, n, -2);
mv(x, y);
bind(&L3);
srliw(y, x, 1);
subw(rd, n, x);
ma_branch(&L4, Equal, y, Operand(zero_reg));
addiw(rd, n, -2);
bind(&L4);
#elif JS_CODEGEN_RISCV32
srli(y, x, 16);
ma_branch(&L0, Equal, y, Operand(zero_reg));
mv(x, y);
addi(n, n, -16);
bind(&L0);
srli(y, x, 8);
ma_branch(&L1, Equal, y, Operand(zero_reg));
addi(n, n, -8);
mv(x, y);
bind(&L1);
srli(y, x, 4);
ma_branch(&L2, Equal, y, Operand(zero_reg));
addi(n, n, -4);
mv(x, y);
bind(&L2);
srli(y, x, 2);
ma_branch(&L3, Equal, y, Operand(zero_reg));
addi(n, n, -2);
mv(x, y);
bind(&L3);
srli(y, x, 1);
sub(rd, n, x);
ma_branch(&L4, Equal, y, Operand(zero_reg));
addi(rd, n, -2);
bind(&L4);
#endif
}
#if JS_CODEGEN_RISCV64
void MacroAssemblerRiscv64::Clz64(Register rd, Register rs) {
if (HasZbbExtension()) {
clz(rd, rs);
return;
}
// 64 bit: count number of leading zeros.
// int n = 64;
// unsigned y;
// y = x >>32; if (y != 0) { n = n - 32; x = y; }
// y = x >>16; if (y != 0) { n = n - 16; x = y; }
// y = x >> 8; if (y != 0) { n = n - 8; x = y; }
// y = x >> 4; if (y != 0) { n = n - 4; x = y; }
// y = x >> 2; if (y != 0) { n = n - 2; x = y; }
// y = x >> 1; if (y != 0) {rd = n - 2; return;}
// rd = n - x;
Label L0, L1, L2, L3, L4, L5;
UseScratchRegisterScope temps(this);
Register x = rd;
Register y = temps.Acquire();
Register n = temps.Acquire();
MOZ_ASSERT(rs != y && rs != n);
mv(x, rs);
ma_li(n, Imm32(64));
srli(y, x, 32);
ma_branch(&L0, Equal, y, Operand(zero_reg));
addiw(n, n, -32);
mv(x, y);
bind(&L0);
srli(y, x, 16);
ma_branch(&L1, Equal, y, Operand(zero_reg));
addiw(n, n, -16);
mv(x, y);
bind(&L1);
srli(y, x, 8);
ma_branch(&L2, Equal, y, Operand(zero_reg));
addiw(n, n, -8);
mv(x, y);
bind(&L2);
srli(y, x, 4);
ma_branch(&L3, Equal, y, Operand(zero_reg));
addiw(n, n, -4);
mv(x, y);
bind(&L3);
srli(y, x, 2);
ma_branch(&L4, Equal, y, Operand(zero_reg));
addiw(n, n, -2);
mv(x, y);
bind(&L4);
srli(y, x, 1);
subw(rd, n, x);
ma_branch(&L5, Equal, y, Operand(zero_reg));
addiw(rd, n, -2);
bind(&L5);
}
#endif
void MacroAssemblerRiscv64::Ctz32(Register rd, Register rs) {
if (HasZbbExtension()) {
#if JS_CODEGEN_RISCV64
ctzw(rd, rs);
#else
ctz(rd, rs);
#endif
return;
}
// Convert trailing zeroes to trailing ones, and bits to their left
// to zeroes.
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_add64(scratch, rs, Operand(-1));
ma_xor(rd, scratch, rs);
ma_and(rd, rd, scratch);
// Count number of leading zeroes.
}
Clz32(rd, rd);
{
// Subtract number of leading zeroes from 32 to get number of trailing
// ones. Remember that the trailing ones were formerly trailing zeroes.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, Imm32(32));
ma_sub32(rd, scratch, rd);
}
}
#if JS_CODEGEN_RISCV64
void MacroAssemblerRiscv64::Ctz64(Register rd, Register rs) {
if (HasZbbExtension()) {
ctz(rd, rs);
return;
}
// Convert trailing zeroes to trailing ones, and bits to their left
// to zeroes.
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_add64(scratch, rs, Operand(-1));
ma_xor(rd, scratch, rs);
ma_and(rd, rd, scratch);
// Count number of leading zeroes.
}
Clz64(rd, rd);
{
// Subtract number of leading zeroes from 64 to get number of trailing
// ones. Remember that the trailing ones were formerly trailing zeroes.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ma_li(scratch, 64);
ma_sub64(rd, scratch, rd);
}
}
#endif
void MacroAssemblerRiscv64::Popcnt32(Register rd, Register rs,
Register scratch) {
if (HasZbbExtension()) {
#if JS_CODEGEN_RISCV64
cpopw(rd, rs);
#else
cpop(rd, rs);
#endif
return;
}
MOZ_ASSERT(scratch != rs);
MOZ_ASSERT(scratch != rd);
//
// A generalization of the best bit counting method to integers of
// bit-widths up to 128 (parameterized by type T) is this:
//
// v = v - ((v >> 1) & (T)~(T)0/3); // temp
// v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3); // temp
// v = (v + (v >> 4)) & (T)~(T)0/255*15; // temp
// c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; //count
//
// There are algorithms which are faster in the cases where very few
// bits are set but the algorithm here attempts to minimize the total
// number of instructions executed even when a large number of bits
// are set.
// The number of instruction is 20.
// uint32_t B0 = 0x55555555; // (T)~(T)0/3
// uint32_t B1 = 0x33333333; // (T)~(T)0/15*3
// uint32_t B2 = 0x0F0F0F0F; // (T)~(T)0/255*15
// uint32_t value = 0x01010101; // (T)~(T)0/255
uint32_t shift = 24;
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
Register value = temps.Acquire();
MOZ_ASSERT((rd != value) && (rs != value));
ma_li(value, 0x01010101); // value = 0x01010101;
ma_li(scratch2, 0x55555555); // B0 = 0x55555555;
ma_srl32(scratch, rs, Operand(1));
ma_and(scratch, scratch, scratch2);
ma_sub32(scratch, rs, scratch);
ma_li(scratch2, 0x33333333); // B1 = 0x33333333;
slli(rd, scratch2, 4);
or_(scratch2, scratch2, rd);
ma_and(rd, scratch, scratch2);
ma_srl32(scratch, scratch, Operand(2));
ma_and(scratch, scratch, scratch2);
ma_add32(scratch, rd, scratch);
ma_srl32(rd, scratch, Operand(4));
ma_add32(rd, rd, scratch);
ma_li(scratch2, 0xF);
ma_mul32(scratch2, value, scratch2); // B2 = 0x0F0F0F0F;
ma_and(rd, rd, scratch2);
ma_mul32(rd, rd, value);
ma_srl32(rd, rd, Operand(shift));
}
#if JS_CODEGEN_RISCV64
void MacroAssemblerRiscv64::Popcnt64(Register rd, Register rs,
Register scratch) {
if (HasZbbExtension()) {
cpop(rd, rs);
return;
}
MOZ_ASSERT(scratch != rs);
MOZ_ASSERT(scratch != rd);
// uint64_t B0 = 0x5555555555555555l; // (T)~(T)0/3
// uint64_t B1 = 0x3333333333333333l; // (T)~(T)0/15*3
// uint64_t B2 = 0x0F0F0F0F0F0F0F0Fl; // (T)~(T)0/255*15
// uint64_t value = 0x0101010101010101l; // (T)~(T)0/255
// uint64_t shift = 24; // (sizeof(T) - 1) * BITS_PER_BYTE
uint64_t shift = 24;
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
Register value = temps.Acquire();
MOZ_ASSERT((rd != value) && (rs != value));
ma_li(value, 0x1111111111111111l); // value = 0x1111111111111111l;
ma_li(scratch2, 5);
ma_mul64(scratch2, value, scratch2); // B0 = 0x5555555555555555l;
ma_srl64(scratch, rs, Operand(1));
ma_and(scratch, scratch, scratch2);
ma_sub64(scratch, rs, scratch);
ma_li(scratch2, 3);
ma_mul64(scratch2, value, scratch2); // B1 = 0x3333333333333333l;
ma_and(rd, scratch, scratch2);
ma_srl64(scratch, scratch, Operand(2));
ma_and(scratch, scratch, scratch2);
ma_add64(scratch, rd, scratch);
ma_srl64(rd, scratch, Operand(4));
ma_add64(rd, rd, scratch);
ma_li(scratch2, 0xF);
ma_li(value, 0x0101010101010101l); // value = 0x0101010101010101l;
ma_mul64(scratch2, value, scratch2); // B2 = 0x0F0F0F0F0F0F0F0Fl;
ma_and(rd, rd, scratch2);
ma_mul64(rd, rd, value);
srli(rd, rd, 32 + shift);
}
#endif
void MacroAssemblerRiscv64::ma_mod_mask(Register src, Register dest,
Register hold, Register remain,
int32_t shift, Label* negZero) {
// MATH:
// We wish to compute x % (1<<y) - 1 for a known constant, y.
// First, let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit
// dividend as a number in base b, namely
// c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
// now, since both addition and multiplication commute with modulus,
// x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
// (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
// now, since b == C + 1, b % C == 1, and b^n % C == 1
// this means that the whole thing simplifies to:
// c_0 + c_1 + c_2 ... c_n % C
// each c_n can easily be computed by a shift/bitextract, and the modulus
// can be maintained by simply subtracting by C whenever the number gets
// over C.
int32_t mask = (1 << shift) - 1;
Label head, negative, sumSigned, done;
// hold holds -1 if the value was negative, 1 otherwise.
// remain holds the remaining bits that have not been processed
// scratch2 serves as a temporary location to store extracted bits
// into as well as holding the trial subtraction as a temp value dest is
// the accumulator (and holds the final result)
// move the whole value into the remain.
or_(remain, src, zero);
// Zero out the dest.
ma_li(dest, Imm32(0));
// Set the hold appropriately.
ma_b(remain, remain, &negative, Signed, ShortJump);
ma_li(hold, Imm32(1));
ma_branch(&head, ShortJump);
bind(&negative);
ma_li(hold, Imm32(-1));
negw(remain, remain);
// Begin the main loop.
bind(&head);
UseScratchRegisterScope temps(this);
Register scratch2 = temps.Acquire();
// Extract the bottom bits into scratch2.
ma_and(scratch2, remain, Imm32(mask));
// Add those bits to the accumulator.
addw(dest, dest, scratch2);
// Do a trial subtraction
ma_sub32(scratch2, dest, Imm32(mask));
// If (sum - C) > 0, store sum - C back into sum, thus performing a
// modulus.
ma_b(scratch2, Register(scratch2), &sumSigned, Signed, ShortJump);
or_(dest, scratch2, zero);
bind(&sumSigned);
// Get rid of the bits that we extracted before.
srliw(remain, remain, shift);
// If the shift produced zero, finish, otherwise, continue in the loop.
ma_b(remain, remain, &head, NonZero, ShortJump);
// Check the hold to see if we need to negate the result.
ma_b(hold, hold, &done, NotSigned, ShortJump);
if (negZero != nullptr) {
// Jump out in case of negative zero.
ma_b(dest, dest, negZero, Zero);
}
// If the hold was non-zero, negate the result to be in line with
// what JS wants
negw(dest, dest);
bind(&done);
}
void MacroAssemblerRiscv64::ma_fmovz(FloatFormat fmt, FloatRegister fd,
FloatRegister fj, Register rk) {
Label done;
ma_b(rk, zero, &done, Assembler::NotEqual);
if (fmt == SingleFloat) {
fmv_s(fd, fj);
} else {
fmv_d(fd, fj);
}
bind(&done);
}
void MacroAssemblerRiscv64::ByteSwap(Register dest, Register src,
int operand_size, Register scratch) {
MOZ_ASSERT(operand_size == 4 || operand_size == 8);
#if JS_CODEGEN_RISCV64
if (HasZbbExtension()) {
rev8(dest, src);
if (operand_size == 4) {
srai(dest, dest, 32);
}
return;
}
#endif
MOZ_ASSERT(scratch != src);
MOZ_ASSERT(scratch != dest);
if (operand_size == 4) {
// Uint32_t x1 = 0x00FF00FF;
// x0 = (x0 << 16 | x0 >> 16);
// x0 = (((x0 & x1) << 8) | ((x0 & (x1 << 8)) >> 8));
UseScratchRegisterScope temps(this);
BlockTrampolinePoolScope block_trampoline_pool(this, 17);
MOZ_ASSERT((dest != t6) && (src != t6));
Register x0 = temps.Acquire();
Register x1 = temps.Acquire();
Register x2 = scratch;
RV_li(x1, 0x00FF00FF);
slliw(x0, src, 16);
srliw(dest, src, 16);
or_(x0, dest, x0); // x0 <- x0 << 16 | x0 >> 16
and_(x2, x0, x1); // x2 <- x0 & 0x00FF00FF
slliw(x2, x2, 8); // x2 <- (x0 & x1) << 8
slliw(x1, x1, 8); // x1 <- 0xFF00FF00
and_(dest, x0, x1); // x0 & 0xFF00FF00
srliw(dest, dest, 8);
or_(dest, dest, x2); // (((x0 & x1) << 8) | ((x0 & (x1 << 8)) >> 8))
} else {
// uinx24_t x1 = 0x0000FFFF0000FFFFl;
// uinx24_t x1 = 0x00FF00FF00FF00FFl;
// x0 = (x0 << 32 | x0 >> 32);
// x0 = (x0 & x1) << 16 | (x0 & (x1 << 16)) >> 16;
// x0 = (x0 & x1) << 8 | (x0 & (x1 << 8)) >> 8;
UseScratchRegisterScope temps(this);
BlockTrampolinePoolScope block_trampoline_pool(this, 30);
MOZ_ASSERT((dest != t6) && (src != t6));
Register x0 = temps.Acquire();
Register x1 = temps.Acquire();
Register x2 = scratch;
RV_li(x1, 0x0000FFFF0000FFFFl);
slli(x0, src, 32);
srli(dest, src, 32);
or_(x0, dest, x0); // x0 <- x0 << 32 | x0 >> 32
and_(x2, x0, x1); // x2 <- x0 & 0x0000FFFF0000FFFF
slli(x2, x2, 16); // x2 <- (x0 & 0x0000FFFF0000FFFF) << 16
slli(x1, x1, 16); // x1 <- 0xFFFF0000FFFF0000
and_(dest, x0, x1); // rd <- x0 & 0xFFFF0000FFFF0000
srli(dest, dest, 16); // rd <- x0 & (x1 << 16)) >> 16
or_(x0, dest, x2); // (x0 & x1) << 16 | (x0 & (x1 << 16)) >> 16;
RV_li(x1, 0x00FF00FF00FF00FFl);
and_(x2, x0, x1); // x2 <- x0 & 0x00FF00FF00FF00FF
slli(x2, x2, 8); // x2 <- (x0 & x1) << 8
slli(x1, x1, 8); // x1 <- 0xFF00FF00FF00FF00
and_(dest, x0, x1);
srli(dest, dest, 8); // rd <- (x0 & (x1 << 8)) >> 8
or_(dest, dest, x2); // (((x0 & x1) << 8) | ((x0 & (x1 << 8)) >> 8))
}
}
template <typename F_TYPE>
void MacroAssemblerRiscv64::FloatMinMaxHelper(FPURegister dst, FPURegister src1,
FPURegister src2,
MaxMinKind kind) {
MOZ_ASSERT((std::is_same<F_TYPE, float>::value) ||
(std::is_same<F_TYPE, double>::value));
if (src1 == src2) {
if (dst != src1) {
if (std::is_same<float, F_TYPE>::value) {
fmv_s(dst, src1);
} else {
fmv_d(dst, src1);
}
}
return;
}
Label done, nan;
// For RISCV, fmin_s returns the other non-NaN operand as result if only one
// operand is NaN; but for JS, if any operand is NaN, result is Nan. The
// following handles the discrepency between handling of NaN between ISA and
// JS semantics
if (std::is_same<float, F_TYPE>::value) {
BranchFloat32(Assembler::DoubleUnordered, src1, src2, &nan, ShortJump);
} else {
BranchFloat64(Assembler::DoubleUnordered, src1, src2, &nan, ShortJump);
}
if (kind == MaxMinKind::kMax) {
if (std::is_same<float, F_TYPE>::value) {
fmax_s(dst, src1, src2);
} else {
fmax_d(dst, src1, src2);
}
} else {
if (std::is_same<float, F_TYPE>::value) {
fmin_s(dst, src1, src2);
} else {
fmin_d(dst, src1, src2);
}
}
jump(&done);
bind(&nan);
// if any operand is NaN, return NaN (fadd returns NaN if any operand is NaN)
if (std::is_same<float, F_TYPE>::value) {
fadd_s(dst, src1, src2);
} else {
fadd_d(dst, src1, src2);
}
bind(&done);
}
void MacroAssemblerRiscv64::Float32Max(FPURegister dst, FPURegister src1,
FPURegister src2) {
comment(__FUNCTION__);
FloatMinMaxHelper<float>(dst, src1, src2, MaxMinKind::kMax);
}
void MacroAssemblerRiscv64::Float32Min(FPURegister dst, FPURegister src1,
FPURegister src2) {
comment(__FUNCTION__);
FloatMinMaxHelper<float>(dst, src1, src2, MaxMinKind::kMin);
}
void MacroAssemblerRiscv64::Float64Max(FPURegister dst, FPURegister src1,
FPURegister src2) {
comment(__FUNCTION__);
FloatMinMaxHelper<double>(dst, src1, src2, MaxMinKind::kMax);
}
void MacroAssemblerRiscv64::Float64Min(FPURegister dst, FPURegister src1,
FPURegister src2) {
comment(__FUNCTION__);
FloatMinMaxHelper<double>(dst, src1, src2, MaxMinKind::kMin);
}
void MacroAssemblerRiscv64::Rol(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (HasZbbExtension()) {
rolw(rd, rs, rt.rm());
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
negw(scratch, rt.rm());
srlw(scratch, rs, scratch);
sllw(rd, rs, rt.rm());
or_(rd, scratch, rd);
sext_w(rd, rd);
} else {
Ror(rd, rs, Operand(32 - (rt.immediate() & 0x1f)));
}
}
void MacroAssemblerRiscv64::Ror(Register rd, Register rs, const Operand& rt) {
if (HasZbbExtension()) {
if (rt.is_reg()) {
rorw(rd, rs, rt.rm());
} else {
int64_t ror_value = rt.immediate() % 32;
if (ror_value < 0) {
ror_value += 32;
}
roriw(rd, rs, ror_value);
}
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (rt.is_reg()) {
negw(scratch, rt.rm());
sllw(scratch, rs, scratch);
srlw(rd, rs, rt.rm());
or_(rd, scratch, rd);
sext_w(rd, rd);
} else {
int64_t ror_value = rt.immediate() & 0x1f;
if (ror_value == 0) {
mv(rd, rs);
return;
}
srliw(scratch, rs, ror_value);
slliw(rd, rs, 32 - ror_value);
or_(rd, scratch, rd);
sext_w(rd, rd);
}
}
void MacroAssemblerRiscv64::Drol(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (HasZbbExtension()) {
rol(rd, rs, rt.rm());
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
negw(scratch, rt.rm());
srl(scratch, rs, scratch);
sll(rd, rs, rt.rm());
or_(rd, scratch, rd);
} else {
Dror(rd, rs, Operand(64 - (rt.immediate() & 0x3f)));
}
}
void MacroAssemblerRiscv64::Dror(Register rd, Register rs, const Operand& rt) {
if (HasZbbExtension()) {
if (rt.is_reg()) {
ror(rd, rs, rt.rm());
} else {
int64_t dror_value = rt.immediate() % 64;
if (dror_value < 0) {
dror_value += 64;
}
rori(rd, rs, dror_value);
}
return;
}
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (rt.is_reg()) {
negw(scratch, rt.rm());
sll(scratch, rs, scratch);
srl(rd, rs, rt.rm());
or_(rd, scratch, rd);
} else {
int64_t dror_value = rt.immediate() & 0x3f;
if (dror_value == 0) {
mv(rd, rs);
return;
}
srli(scratch, rs, dror_value);
slli(rd, rs, 64 - dror_value);
or_(rd, scratch, rd);
}
}
void MacroAssemblerRiscv64::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
Register memoryBase, Register ptr,
Register ptrScratch,
AnyRegister output, Register tmp) {
access.assertOffsetInGuardPages();
uint32_t offset = access.offset32();
MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
// Maybe add the offset.
if (offset) {
asMasm().addPtr(ImmWord(offset), ptrScratch);
ptr = ptrScratch;
}
asMasm().memoryBarrierBefore(access.sync());
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
FaultingCodeOffset fco;
switch (access.type()) {
case Scalar::Int8:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lb(output.gpr(), scratch, 0);
break;
case Scalar::Uint8:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lbu(output.gpr(), scratch, 0);
break;
case Scalar::Int16:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lh(output.gpr(), scratch, 0);
break;
case Scalar::Uint16:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lhu(output.gpr(), scratch, 0);
break;
case Scalar::Int32:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lw(output.gpr(), scratch, 0);
break;
case Scalar::Uint32:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
lwu(output.gpr(), scratch, 0);
break;
case Scalar::Float64:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
fld(output.fpu(), scratch, 0);
break;
case Scalar::Float32:
add(scratch, memoryBase, ptr);
fco = FaultingCodeOffset(currentOffset());
flw(output.fpu(), scratch, 0);
break;
default:
MOZ_CRASH("unexpected array type");
}
append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
asMasm().memoryBarrierAfter(access.sync());
}
void MacroAssemblerRiscv64::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
AnyRegister value,
Register memoryBase, Register ptr,
Register ptrScratch, Register tmp) {
access.assertOffsetInGuardPages();
uint32_t offset = access.offset32();
MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
// Maybe add the offset.
if (offset) {
asMasm().addPtr(ImmWord(offset), ptrScratch);
ptr = ptrScratch;
}
unsigned byteSize = access.byteSize();
bool isSigned = Scalar::isSignedIntType(access.type());
bool isFloat = Scalar::isFloatingType(access.type());
BaseIndex address(memoryBase, ptr, TimesOne);
asMasm().memoryBarrierBefore(access.sync());
FaultingCodeOffset fco;
if (isFloat) {
if (byteSize == 4) {
fco = ma_fst_s(value.fpu(), address);
} else {
fco = ma_fst_d(value.fpu(), address);
}
} else {
fco =
ma_store(value.gpr(), address, static_cast<LoadStoreSize>(8 * byteSize),
isSigned ? SignExtend : ZeroExtend);
}
// Only the last emitted instruction is a memory access.
append(access, js::wasm::TrapMachineInsnForStore(access.byteSize()), fco);
asMasm().memoryBarrierAfter(access.sync());
}
void MacroAssemblerRiscv64::GenPCRelativeJumpAndLink(Register rd,
int32_t imm32) {
MOZ_ASSERT(is_int32(imm32 + 0x800));
int32_t Hi20 = ((imm32 + 0x800) >> 12);
int32_t Lo12 = imm32 << 20 >> 20;
auipc(rd, Hi20); // Read PC + Hi20 into scratch.
jalr(rd, Lo12); // jump PC + Hi20 + Lo12
}
CodeOffset MacroAssemblerRiscv64::BranchAndLinkShortHelper(int32_t offset,
Label* L) {
MOZ_ASSERT(L == nullptr || offset == 0);
BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
offset = GetOffset(offset, L, OffsetSize::kOffset21);
return jal(offset);
}
CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(int32_t offset) {
MOZ_ASSERT(is_int21(offset));
return BranchAndLinkShortHelper(offset, nullptr);
}
CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(Label* L) {
return BranchAndLinkShortHelper(0, L);
}
CodeOffset MacroAssemblerRiscv64::BranchAndLink(Label* L) {
if (L->bound() && !isNear(L)) {
return BranchAndLinkLong(L);
}
return BranchAndLinkShort(L);
}
void MacroAssemblerRiscv64::ma_fmv_d(FloatRegister src, ValueOperand dest) {
fmv_x_d(dest.valueReg(), src);
}
void MacroAssemblerRiscv64::ma_fmv_d(ValueOperand src, FloatRegister dest) {
fmv_d_x(dest, src.valueReg());
}
void MacroAssemblerRiscv64::ma_fmv_w(FloatRegister src, ValueOperand dest) {
fmv_x_w(dest.valueReg(), src);
}
void MacroAssemblerRiscv64::ma_fmv_w(ValueOperand src, FloatRegister dest) {
fmv_w_x(dest, src.valueReg());
}
} // namespace jit
} // namespace js