Lines Matching refs:codegen
88 #define __ codegen->GetVIXLAssembler()->
100 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in); in EmitNativeCode() local
120 codegen->GetAssembler()->MaybeUnpoisonHeapReference(tmp_reg); in EmitNativeCode()
144 codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); in EmitNativeCode()
145 codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg); in EmitNativeCode()
702 CodeGeneratorARM64* codegen) { in GenUnsafeGet() argument
717 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenUnsafeGet()
720 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenUnsafeGet()
730 codegen->LoadAcquire(invoke, type, trg, mem_op, /* needs_null_check= */ true); in GenUnsafeGet()
732 codegen->Load(type, trg, mem_op); in GenUnsafeGet()
737 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0u, offset_loc); in GenUnsafeGet()
844 CodeGeneratorARM64* codegen) { in GenUnsafePut() argument
846 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenUnsafePut()
863 codegen->GetAssembler()->PoisonHeapReference(temp.W()); in GenUnsafePut()
868 codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check= */ false); in GenUnsafePut()
870 codegen->Store(type, source, mem_op); in GenUnsafePut()
876 codegen->MarkGCCard(base, value, value_can_be_null); in GenUnsafePut()
965 static void EmitLoadExclusive(CodeGeneratorARM64* codegen, in EmitLoadExclusive() argument
970 Arm64Assembler* assembler = codegen->GetAssembler(); in EmitLoadExclusive()
1018 static void EmitStoreExclusive(CodeGeneratorARM64* codegen, in EmitStoreExclusive() argument
1024 Arm64Assembler* assembler = codegen->GetAssembler(); in EmitStoreExclusive()
1065 static void GenerateCompareAndSet(CodeGeneratorARM64* codegen, in GenerateCompareAndSet() argument
1087 Arm64Assembler* assembler = codegen->GetAssembler(); in GenerateCompareAndSet()
1114 EmitLoadExclusive(codegen, type, ptr, old_value, use_load_acquire); in GenerateCompareAndSet()
1123 EmitStoreExclusive(codegen, type, ptr, store_result, new_value, use_store_release); in GenerateCompareAndSet()
1182 void EmitNativeCode(CodeGenerator* codegen) override { in EmitNativeCode() argument
1183 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); in EmitNativeCode()
1273 static void GenUnsafeCas(HInvoke* invoke, DataType::Type type, CodeGeneratorARM64* codegen) { in GenUnsafeCas() argument
1274 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenUnsafeCas()
1287 codegen->MarkGCCard(base, new_value, new_value_can_be_null); in GenUnsafeCas()
1306 new (codegen->GetScopedAllocator()) ReadBarrierCasSlowPathARM64( in GenUnsafeCas()
1318 codegen); in GenUnsafeCas()
1319 codegen->AddSlowPath(slow_path); in GenUnsafeCas()
1328 GenerateCompareAndSet(codegen, in GenUnsafeCas()
1394 static void GenerateGetAndUpdate(CodeGeneratorARM64* codegen, in GenerateGetAndUpdate() argument
1401 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateGetAndUpdate()
1437 EmitLoadExclusive(codegen, load_store_type, ptr, old_value_reg, use_load_acquire); in GenerateGetAndUpdate()
1471 EmitStoreExclusive(codegen, load_store_type, ptr, store_result, new_value, use_store_release); in GenerateGetAndUpdate()
1898 CodeGeneratorARM64* codegen, in GenerateVisitStringIndexOf() argument
1913 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
1914 codegen->AddSlowPath(slow_path); in GenerateVisitStringIndexOf()
1922 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
1923 codegen->AddSlowPath(slow_path); in GenerateVisitStringIndexOf()
1933 codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); in GenerateVisitStringIndexOf()
2076 CodeGeneratorARM64* codegen, in GenFPToFPCall() argument
2078 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); in GenFPToFPCall()
3738 CodeGeneratorARM64* codegen, in GenerateFP16Compare() argument
3741 DCHECK(codegen->GetInstructionSetFeatures().HasFP16()); in GenerateFP16Compare()
3752 CodeGeneratorARM64* codegen, in GenerateFP16Compare() argument
3759 GenerateFP16Compare(invoke, codegen, masm, compareOp); in GenerateFP16Compare()
3822 static void GenerateDivideUnsigned(HInvoke* invoke, CodeGeneratorARM64* codegen) { in GenerateDivideUnsigned() argument
3824 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateDivideUnsigned()
3834 new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateDivideUnsigned()
3835 codegen->AddSlowPath(slow_path); in GenerateDivideUnsigned()
3939 static void GenerateSubTypeObjectCheckNoReadBarrier(CodeGeneratorARM64* codegen, in GenerateSubTypeObjectCheckNoReadBarrier() argument
3944 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateSubTypeObjectCheckNoReadBarrier()
3958 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); in GenerateSubTypeObjectCheckNoReadBarrier()
3964 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); in GenerateSubTypeObjectCheckNoReadBarrier()
3974 CodeGeneratorARM64* codegen, in GenerateVarHandleAccessModeAndVarTypeChecks() argument
3981 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleAccessModeAndVarTypeChecks()
3997 codegen->GetAssembler()->MaybeUnpoisonHeapReference(var_type_no_rb); in GenerateVarHandleAccessModeAndVarTypeChecks()
4022 GenerateSubTypeObjectCheckNoReadBarrier(codegen, slow_path, arg_reg, var_type_no_rb); in GenerateVarHandleAccessModeAndVarTypeChecks()
4029 CodeGeneratorARM64* codegen, in GenerateVarHandleStaticFieldCheck() argument
4031 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleStaticFieldCheck()
4046 CodeGeneratorARM64* codegen, in GenerateVarHandleInstanceFieldChecks() argument
4048 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleInstanceFieldChecks()
4067 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); in GenerateVarHandleInstanceFieldChecks()
4075 codegen, slow_path, object, temp, /*object_can_be_null=*/ false); in GenerateVarHandleInstanceFieldChecks()
4091 CodeGeneratorARM64* codegen, in GenerateVarHandleArrayChecks() argument
4093 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleArrayChecks()
4120 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp); in GenerateVarHandleArrayChecks()
4135 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp2); in GenerateVarHandleArrayChecks()
4143 codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp2); in GenerateVarHandleArrayChecks()
4157 codegen->GetCompilerOptions().IsBootImage() || in GenerateVarHandleArrayChecks()
4159 DCHECK(boot_image_available || codegen->GetCompilerOptions().IsJitCompiler()); in GenerateVarHandleArrayChecks()
4174 CodeGeneratorARM64* codegen, in GenerateVarHandleCoordinateChecks() argument
4178 GenerateVarHandleStaticFieldCheck(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4180 GenerateVarHandleInstanceFieldChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4183 GenerateVarHandleArrayChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4188 CodeGeneratorARM64* codegen, in GenerateVarHandleChecks() argument
4192 new (codegen->GetScopedAllocator()) VarHandleSlowPathARM64(invoke, order); in GenerateVarHandleChecks()
4193 codegen->AddSlowPath(slow_path); in GenerateVarHandleChecks()
4195 GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen, slow_path, type); in GenerateVarHandleChecks()
4196 GenerateVarHandleCoordinateChecks(invoke, codegen, slow_path); in GenerateVarHandleChecks()
4222 CodeGeneratorARM64* codegen) { in GenerateVarHandleTarget() argument
4223 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleTarget()
4241 codegen->GenerateGcRootFieldLoad(invoke, in GenerateVarHandleTarget()
4420 CodeGeneratorARM64* codegen, in GenerateVarHandleGet() argument
4427 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleGet()
4433 slow_path = GenerateVarHandleChecks(invoke, codegen, order, type); in GenerateVarHandleGet()
4434 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGet()
4449 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenerateVarHandleGet()
4474 codegen->LoadAcquire(invoke, load_type, load_reg, address, /*needs_null_check=*/ false); in GenerateVarHandleGet()
4476 codegen->Load(load_type, load_reg, address); in GenerateVarHandleGet()
4484 codegen->MaybeGenerateReadBarrierSlow(invoke, out_loc, out_loc, object_loc, 0u, offset_loc); in GenerateVarHandleGet()
4536 CodeGeneratorARM64* codegen, in GenerateVarHandleSet() argument
4542 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleSet()
4548 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleSet()
4549 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleSet()
4566 codegen->GetAssembler()->PoisonHeapReference(temp); in GenerateVarHandleSet()
4585 codegen->StoreRelease(invoke, value_type, source, address, /*needs_null_check=*/ false); in GenerateVarHandleSet()
4587 codegen->Store(value_type, source, address); in GenerateVarHandleSet()
4592 codegen->MarkGCCard(target.object, Register(value), /*value_can_be_null=*/ true); in GenerateVarHandleSet()
4720 CodeGeneratorARM64* codegen, in GenerateVarHandleCompareAndSetOrExchange() argument
4732 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleCompareAndSetOrExchange()
4741 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleCompareAndSetOrExchange()
4743 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleCompareAndSetOrExchange()
4751 codegen->MarkGCCard(target.object, new_value.W(), new_value_can_be_null); in GenerateVarHandleCompareAndSetOrExchange()
4851 new (codegen->GetScopedAllocator()) ReadBarrierCasSlowPathARM64( in GenerateVarHandleCompareAndSetOrExchange()
4863 codegen); in GenerateVarHandleCompareAndSetOrExchange()
4864 codegen->AddSlowPath(rb_slow_path); in GenerateVarHandleCompareAndSetOrExchange()
4869 GenerateCompareAndSet(codegen, in GenerateVarHandleCompareAndSetOrExchange()
5025 CodeGeneratorARM64* codegen, in GenerateVarHandleGetAndUpdate() argument
5032 MacroAssembler* masm = codegen->GetVIXLAssembler(); in GenerateVarHandleGetAndUpdate()
5040 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleGetAndUpdate()
5042 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGetAndUpdate()
5051 codegen->MarkGCCard(target.object, arg.W(), new_value_can_be_null); in GenerateVarHandleGetAndUpdate()
5115 GenerateGetAndUpdate(codegen, get_and_update_op, load_store_type, order, tmp_ptr, arg, old_value); in GenerateVarHandleGetAndUpdate()
5138 codegen->GenerateIntrinsicCasMoveWithBakerReadBarrier(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5140 codegen->GenerateReadBarrierSlow( in GenerateVarHandleGetAndUpdate()
5277 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in); in EmitByteArrayViewCode() local
5278 MacroAssembler* masm = codegen->GetVIXLAssembler(); in EmitByteArrayViewCode()
5307 codegen->LoadClassRootForIntrinsic(temp2, ClassRoot::kJavaLangInvokeByteArrayViewVarHandle); in EmitByteArrayViewCode()
5343 GenerateVarHandleGet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5346 GenerateVarHandleSet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5351 invoke, codegen, order_, return_success_, strong_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5355 invoke, codegen, get_and_update_op_, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()