1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_UTILS_X86_ASSEMBLER_X86_H_
18 #define ART_COMPILER_UTILS_X86_ASSEMBLER_X86_H_
19 
20 #include <vector>
21 
22 #include "arch/x86/instruction_set_features_x86.h"
23 #include "base/arena_containers.h"
24 #include "base/array_ref.h"
25 #include "base/bit_utils.h"
26 #include "base/enums.h"
27 #include "base/globals.h"
28 #include "base/macros.h"
29 #include "constants_x86.h"
30 #include "heap_poisoning.h"
31 #include "managed_register_x86.h"
32 #include "offsets.h"
33 #include "utils/assembler.h"
34 
35 namespace art {
36 namespace x86 {
37 
38 class Immediate : public ValueObject {
39  public:
Immediate(int32_t value_in)40   explicit Immediate(int32_t value_in) : value_(value_in) {}
41 
value()42   int32_t value() const { return value_; }
43 
is_int8()44   bool is_int8() const { return IsInt<8>(value_); }
is_uint8()45   bool is_uint8() const { return IsUint<8>(value_); }
is_int16()46   bool is_int16() const { return IsInt<16>(value_); }
is_uint16()47   bool is_uint16() const { return IsUint<16>(value_); }
48 
49  private:
50   const int32_t value_;
51 };
52 
53 
54 class Operand : public ValueObject {
55  public:
mod()56   uint8_t mod() const {
57     return (encoding_at(0) >> 6) & 3;
58   }
59 
rm()60   Register rm() const {
61     return static_cast<Register>(encoding_at(0) & 7);
62   }
63 
scale()64   ScaleFactor scale() const {
65     return static_cast<ScaleFactor>((encoding_at(1) >> 6) & 3);
66   }
67 
index()68   Register index() const {
69     return static_cast<Register>((encoding_at(1) >> 3) & 7);
70   }
71 
base()72   Register base() const {
73     return static_cast<Register>(encoding_at(1) & 7);
74   }
75 
disp()76   int32_t disp() const {
77     return disp_;
78   }
79 
disp8()80   int8_t disp8() const {
81     CHECK_GE(length_, 2);
82     return static_cast<int8_t>(encoding_[length_ - 1]);
83   }
84 
disp32()85   int32_t disp32() const {
86     CHECK_GE(length_, 5);
87     int32_t value;
88     memcpy(&value, &encoding_[length_ - 4], sizeof(value));
89     return value;
90   }
91 
IsRegister(Register reg)92   bool IsRegister(Register reg) const {
93     return ((encoding_[0] & 0xF8) == 0xC0)  // Addressing mode is register only.
94         && ((encoding_[0] & 0x07) == reg);  // Register codes match.
95   }
96 
97  protected:
98   // Operand can be sub classed (e.g: Address).
Operand()99   Operand() : length_(0), disp_(0), fixup_(nullptr) { }
100 
SetModRM(int mod_in,Register rm_in)101   void SetModRM(int mod_in, Register rm_in) {
102     CHECK_EQ(mod_in & ~3, 0);
103     encoding_[0] = (mod_in << 6) | rm_in;
104     length_ = 1;
105   }
106 
SetSIB(ScaleFactor scale_in,Register index_in,Register base_in)107   void SetSIB(ScaleFactor scale_in, Register index_in, Register base_in) {
108     CHECK_EQ(length_, 1);
109     CHECK_EQ(scale_in & ~3, 0);
110     encoding_[1] = (scale_in << 6) | (index_in << 3) | base_in;
111     length_ = 2;
112   }
113 
SetDisp8(int8_t disp)114   void SetDisp8(int8_t disp) {
115     CHECK(length_ == 1 || length_ == 2);
116     encoding_[length_++] = static_cast<uint8_t>(disp);
117     disp_ = disp;
118   }
119 
SetDisp32(int32_t disp)120   void SetDisp32(int32_t disp) {
121     CHECK(length_ == 1 || length_ == 2);
122     int disp_size = sizeof(disp);
123     memmove(&encoding_[length_], &disp, disp_size);
124     length_ += disp_size;
125     disp_ = disp;
126   }
127 
GetFixup()128   AssemblerFixup* GetFixup() const {
129     return fixup_;
130   }
131 
SetFixup(AssemblerFixup * fixup)132   void SetFixup(AssemblerFixup* fixup) {
133     fixup_ = fixup;
134   }
135 
136  private:
137   uint8_t length_;
138   uint8_t encoding_[6];
139   int32_t disp_;
140 
141   // A fixup can be associated with the operand, in order to be applied after the
142   // code has been generated. This is used for constant area fixups.
143   AssemblerFixup* fixup_;
144 
Operand(Register reg)145   explicit Operand(Register reg) : disp_(0), fixup_(nullptr) { SetModRM(3, reg); }
146 
147   // Get the operand encoding byte at the given index.
encoding_at(int index_in)148   uint8_t encoding_at(int index_in) const {
149     CHECK_GE(index_in, 0);
150     CHECK_LT(index_in, length_);
151     return encoding_[index_in];
152   }
153 
154   friend class X86Assembler;
155 };
156 
157 
158 class Address : public Operand {
159  public:
Address(Register base_in,int32_t disp)160   Address(Register base_in, int32_t disp) {
161     Init(base_in, disp);
162   }
163 
Address(Register base_in,int32_t disp,AssemblerFixup * fixup)164   Address(Register base_in, int32_t disp, AssemblerFixup *fixup) {
165     Init(base_in, disp);
166     SetFixup(fixup);
167   }
168 
Address(Register base_in,Offset disp)169   Address(Register base_in, Offset disp) {
170     Init(base_in, disp.Int32Value());
171   }
172 
Address(Register base_in,FrameOffset disp)173   Address(Register base_in, FrameOffset disp) {
174     CHECK_EQ(base_in, ESP);
175     Init(ESP, disp.Int32Value());
176   }
177 
Address(Register base_in,MemberOffset disp)178   Address(Register base_in, MemberOffset disp) {
179     Init(base_in, disp.Int32Value());
180   }
181 
Address(Register index_in,ScaleFactor scale_in,int32_t disp)182   Address(Register index_in, ScaleFactor scale_in, int32_t disp) {
183     CHECK_NE(index_in, ESP);  // Illegal addressing mode.
184     SetModRM(0, ESP);
185     SetSIB(scale_in, index_in, EBP);
186     SetDisp32(disp);
187   }
188 
Address(Register base_in,Register index_in,ScaleFactor scale_in,int32_t disp)189   Address(Register base_in, Register index_in, ScaleFactor scale_in, int32_t disp) {
190     Init(base_in, index_in, scale_in, disp);
191   }
192 
Address(Register base_in,Register index_in,ScaleFactor scale_in,int32_t disp,AssemblerFixup * fixup)193   Address(Register base_in,
194           Register index_in,
195           ScaleFactor scale_in,
196           int32_t disp, AssemblerFixup *fixup) {
197     Init(base_in, index_in, scale_in, disp);
198     SetFixup(fixup);
199   }
200 
displaceBy(int offset)201   Address displaceBy(int offset) {
202     if (rm() == ESP) {
203       // SIB addressing mode
204       return Address(base(), index(), scale(), disp() + offset, GetFixup());
205     } else {
206       return Address(rm(), disp() + offset, GetFixup());
207     }
208   }
209 
GetBaseRegister()210   Register GetBaseRegister() {
211     if (rm() == ESP) {
212       return base();
213     } else {
214       return rm();
215     }
216   }
217 
Absolute(uintptr_t addr)218   static Address Absolute(uintptr_t addr) {
219     Address result;
220     result.SetModRM(0, EBP);
221     result.SetDisp32(addr);
222     return result;
223   }
224 
Absolute(ThreadOffset32 addr)225   static Address Absolute(ThreadOffset32 addr) {
226     return Absolute(addr.Int32Value());
227   }
228 
229  private:
Address()230   Address() {}
231 
Init(Register base_in,int32_t disp)232   void Init(Register base_in, int32_t disp) {
233     if (disp == 0 && base_in != EBP) {
234       SetModRM(0, base_in);
235       if (base_in == ESP) SetSIB(TIMES_1, ESP, base_in);
236     } else if (disp >= -128 && disp <= 127) {
237       SetModRM(1, base_in);
238       if (base_in == ESP) SetSIB(TIMES_1, ESP, base_in);
239       SetDisp8(disp);
240     } else {
241       SetModRM(2, base_in);
242       if (base_in == ESP) SetSIB(TIMES_1, ESP, base_in);
243       SetDisp32(disp);
244     }
245   }
246 
Init(Register base_in,Register index_in,ScaleFactor scale_in,int32_t disp)247   void Init(Register base_in, Register index_in, ScaleFactor scale_in, int32_t disp) {
248     CHECK_NE(index_in, ESP);  // Illegal addressing mode.
249     if (disp == 0 && base_in != EBP) {
250       SetModRM(0, ESP);
251       SetSIB(scale_in, index_in, base_in);
252     } else if (disp >= -128 && disp <= 127) {
253       SetModRM(1, ESP);
254       SetSIB(scale_in, index_in, base_in);
255       SetDisp8(disp);
256     } else {
257       SetModRM(2, ESP);
258       SetSIB(scale_in, index_in, base_in);
259       SetDisp32(disp);
260     }
261   }
262 };
263 
264 std::ostream& operator<<(std::ostream& os, const Address& addr);
265 
266 // This is equivalent to the Label class, used in a slightly different context. We
267 // inherit the functionality of the Label class, but prevent unintended
268 // derived-to-base conversions by making the base class private.
269 class NearLabel : private Label {
270  public:
NearLabel()271   NearLabel() : Label() {}
272 
273   // Expose the Label routines that we need.
274   using Label::Position;
275   using Label::LinkPosition;
276   using Label::IsBound;
277   using Label::IsUnused;
278   using Label::IsLinked;
279 
280  private:
281   using Label::BindTo;
282   using Label::LinkTo;
283 
284   friend class x86::X86Assembler;
285 
286   DISALLOW_COPY_AND_ASSIGN(NearLabel);
287 };
288 
289 /**
290  * Class to handle constant area values.
291  */
292 class ConstantArea {
293  public:
ConstantArea(ArenaAllocator * allocator)294   explicit ConstantArea(ArenaAllocator* allocator)
295       : buffer_(allocator->Adapter(kArenaAllocAssembler)) {}
296 
297   // Add a double to the constant area, returning the offset into
298   // the constant area where the literal resides.
299   size_t AddDouble(double v);
300 
301   // Add a float to the constant area, returning the offset into
302   // the constant area where the literal resides.
303   size_t AddFloat(float v);
304 
305   // Add an int32_t to the constant area, returning the offset into
306   // the constant area where the literal resides.
307   size_t AddInt32(int32_t v);
308 
309   // Add an int32_t to the end of the constant area, returning the offset into
310   // the constant area where the literal resides.
311   size_t AppendInt32(int32_t v);
312 
313   // Add an int64_t to the constant area, returning the offset into
314   // the constant area where the literal resides.
315   size_t AddInt64(int64_t v);
316 
IsEmpty()317   bool IsEmpty() const {
318     return buffer_.size() == 0;
319   }
320 
GetSize()321   size_t GetSize() const {
322     return buffer_.size() * elem_size_;
323   }
324 
GetBuffer()325   ArrayRef<const int32_t> GetBuffer() const {
326     return ArrayRef<const int32_t>(buffer_);
327   }
328 
329  private:
330   static constexpr size_t elem_size_ = sizeof(int32_t);
331   ArenaVector<int32_t> buffer_;
332 };
333 
334 class X86Assembler final : public Assembler {
335  public:
336   explicit X86Assembler(ArenaAllocator* allocator,
337                         const X86InstructionSetFeatures* instruction_set_features = nullptr)
338                 : Assembler(allocator),
339                   constant_area_(allocator),
340                   has_AVX_(instruction_set_features != nullptr ? instruction_set_features->HasAVX() : false),
341                   has_AVX2_(instruction_set_features != nullptr ? instruction_set_features->HasAVX2() :false) {}
~X86Assembler()342   virtual ~X86Assembler() {}
343 
344   /*
345    * Emit Machine Instructions.
346    */
347   void call(Register reg);
348   void call(const Address& address);
349   void call(Label* label);
350   void call(const ExternalLabel& label);
351 
352   void pushl(Register reg);
353   void pushl(const Address& address);
354   void pushl(const Immediate& imm);
355 
356   void popl(Register reg);
357   void popl(const Address& address);
358 
359   void movl(Register dst, const Immediate& src);
360   void movl(Register dst, Register src);
361 
362   void movl(Register dst, const Address& src);
363   void movl(const Address& dst, Register src);
364   void movl(const Address& dst, const Immediate& imm);
365   void movl(const Address& dst, Label* lbl);
366 
367   void movntl(const Address& dst, Register src);
368 
369   void blsi(Register dst, Register src);  // no addr variant (for now)
370   void blsmsk(Register dst, Register src);  // no addr variant (for now)
371   void blsr(Register dst, Register src);  // no addr varianr (for now)
372 
373   void bswapl(Register dst);
374 
375   void bsfl(Register dst, Register src);
376   void bsfl(Register dst, const Address& src);
377   void bsrl(Register dst, Register src);
378   void bsrl(Register dst, const Address& src);
379 
380   void popcntl(Register dst, Register src);
381   void popcntl(Register dst, const Address& src);
382 
383   void rorl(Register reg, const Immediate& imm);
384   void rorl(Register operand, Register shifter);
385   void roll(Register reg, const Immediate& imm);
386   void roll(Register operand, Register shifter);
387 
388   void movzxb(Register dst, ByteRegister src);
389   void movzxb(Register dst, const Address& src);
390   void movsxb(Register dst, ByteRegister src);
391   void movsxb(Register dst, const Address& src);
392   void movb(Register dst, const Address& src);
393   void movb(const Address& dst, ByteRegister src);
394   void movb(const Address& dst, const Immediate& imm);
395 
396   void movzxw(Register dst, Register src);
397   void movzxw(Register dst, const Address& src);
398   void movsxw(Register dst, Register src);
399   void movsxw(Register dst, const Address& src);
400   void movw(Register dst, const Address& src);
401   void movw(const Address& dst, Register src);
402   void movw(const Address& dst, const Immediate& imm);
403 
404   void leal(Register dst, const Address& src);
405 
406   void cmovl(Condition condition, Register dst, Register src);
407   void cmovl(Condition condition, Register dst, const Address& src);
408 
409   void setb(Condition condition, Register dst);
410 
411   void movaps(XmmRegister dst, XmmRegister src);     // move
412   void movaps(XmmRegister dst, const Address& src);  // load aligned
413   void movups(XmmRegister dst, const Address& src);  // load unaligned
414   void movaps(const Address& dst, XmmRegister src);  // store aligned
415   void movups(const Address& dst, XmmRegister src);  // store unaligned
416 
417   void vmovaps(XmmRegister dst, XmmRegister src);     // move
418   void vmovaps(XmmRegister dst, const Address& src);  // load aligned
419   void vmovups(XmmRegister dst, const Address& src);  // load unaligned
420   void vmovaps(const Address& dst, XmmRegister src);  // store aligned
421   void vmovups(const Address& dst, XmmRegister src);  // store unaligned
422 
423   void movss(XmmRegister dst, const Address& src);
424   void movss(const Address& dst, XmmRegister src);
425   void movss(XmmRegister dst, XmmRegister src);
426 
427   void movd(XmmRegister dst, Register src);
428   void movd(Register dst, XmmRegister src);
429 
430   void addss(XmmRegister dst, XmmRegister src);
431   void addss(XmmRegister dst, const Address& src);
432   void subss(XmmRegister dst, XmmRegister src);
433   void subss(XmmRegister dst, const Address& src);
434   void mulss(XmmRegister dst, XmmRegister src);
435   void mulss(XmmRegister dst, const Address& src);
436   void divss(XmmRegister dst, XmmRegister src);
437   void divss(XmmRegister dst, const Address& src);
438 
439   void addps(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
440   void subps(XmmRegister dst, XmmRegister src);
441   void mulps(XmmRegister dst, XmmRegister src);
442   void divps(XmmRegister dst, XmmRegister src);
443 
444   void vmulps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
445   void vmulpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
446   void vdivps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
447   void vdivpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
448 
449   void vaddps(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
450   void vsubps(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
451   void vsubpd(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
452   void vaddpd(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
453 
454   void movapd(XmmRegister dst, XmmRegister src);     // move
455   void movapd(XmmRegister dst, const Address& src);  // load aligned
456   void movupd(XmmRegister dst, const Address& src);  // load unaligned
457   void movapd(const Address& dst, XmmRegister src);  // store aligned
458   void movupd(const Address& dst, XmmRegister src);  // store unaligned
459 
460   void vmovapd(XmmRegister dst, XmmRegister src);     // move
461   void vmovapd(XmmRegister dst, const Address& src);  // load aligned
462   void vmovupd(XmmRegister dst, const Address& src);  // load unaligned
463   void vmovapd(const Address& dst, XmmRegister src);  // store aligned
464   void vmovupd(const Address& dst, XmmRegister src);  // store unaligned
465 
466   void movsd(XmmRegister dst, const Address& src);
467   void movsd(const Address& dst, XmmRegister src);
468   void movsd(XmmRegister dst, XmmRegister src);
469 
470   void movhpd(XmmRegister dst, const Address& src);
471   void movhpd(const Address& dst, XmmRegister src);
472 
473   void addsd(XmmRegister dst, XmmRegister src);
474   void addsd(XmmRegister dst, const Address& src);
475   void subsd(XmmRegister dst, XmmRegister src);
476   void subsd(XmmRegister dst, const Address& src);
477   void mulsd(XmmRegister dst, XmmRegister src);
478   void mulsd(XmmRegister dst, const Address& src);
479   void divsd(XmmRegister dst, XmmRegister src);
480   void divsd(XmmRegister dst, const Address& src);
481 
482   void addpd(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
483   void subpd(XmmRegister dst, XmmRegister src);
484   void mulpd(XmmRegister dst, XmmRegister src);
485   void divpd(XmmRegister dst, XmmRegister src);
486 
487   void movdqa(XmmRegister dst, XmmRegister src);     // move
488   void movdqa(XmmRegister dst, const Address& src);  // load aligned
489   void movdqu(XmmRegister dst, const Address& src);  // load unaligned
490   void movdqa(const Address& dst, XmmRegister src);  // store aligned
491   void movdqu(const Address& dst, XmmRegister src);  // store unaligned
492 
493   void vmovdqa(XmmRegister dst, XmmRegister src);     // move
494   void vmovdqa(XmmRegister dst, const Address& src);  // load aligned
495   void vmovdqu(XmmRegister dst, const Address& src);  // load unaligned
496   void vmovdqa(const Address& dst, XmmRegister src);  // store aligned
497   void vmovdqu(const Address& dst, XmmRegister src);  // store unaligned
498 
499   void paddb(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
500   void psubb(XmmRegister dst, XmmRegister src);
501 
502   void vpaddb(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
503   void vpaddw(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
504 
505   void paddw(XmmRegister dst, XmmRegister src);
506   void psubw(XmmRegister dst, XmmRegister src);
507   void pmullw(XmmRegister dst, XmmRegister src);
508   void vpmullw(XmmRegister dst, XmmRegister src1, XmmRegister src2);
509 
510   void vpsubb(XmmRegister dst, XmmRegister src1, XmmRegister src2);
511   void vpsubw(XmmRegister dst, XmmRegister src1, XmmRegister src2);
512   void vpsubd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
513 
514   void paddd(XmmRegister dst, XmmRegister src);
515   void psubd(XmmRegister dst, XmmRegister src);
516   void pmulld(XmmRegister dst, XmmRegister src);
517 
518   void vpmulld(XmmRegister dst, XmmRegister src1, XmmRegister src2);
519 
520   void vpaddd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
521 
522   void paddq(XmmRegister dst, XmmRegister src);
523   void psubq(XmmRegister dst, XmmRegister src);
524 
525   void vpaddq(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
526   void vpsubq(XmmRegister dst, XmmRegister add_left, XmmRegister add_right);
527 
528   void paddusb(XmmRegister dst, XmmRegister src);
529   void paddsb(XmmRegister dst, XmmRegister src);
530   void paddusw(XmmRegister dst, XmmRegister src);
531   void paddsw(XmmRegister dst, XmmRegister src);
532   void psubusb(XmmRegister dst, XmmRegister src);
533   void psubsb(XmmRegister dst, XmmRegister src);
534   void psubusw(XmmRegister dst, XmmRegister src);
535   void psubsw(XmmRegister dst, XmmRegister src);
536 
537   void cvtsi2ss(XmmRegister dst, Register src);
538   void cvtsi2sd(XmmRegister dst, Register src);
539 
540   void cvtss2si(Register dst, XmmRegister src);
541   void cvtss2sd(XmmRegister dst, XmmRegister src);
542 
543   void cvtsd2si(Register dst, XmmRegister src);
544   void cvtsd2ss(XmmRegister dst, XmmRegister src);
545 
546   void cvttss2si(Register dst, XmmRegister src);
547   void cvttsd2si(Register dst, XmmRegister src);
548 
549   void cvtdq2ps(XmmRegister dst, XmmRegister src);
550   void cvtdq2pd(XmmRegister dst, XmmRegister src);
551 
552   void comiss(XmmRegister a, XmmRegister b);
553   void comiss(XmmRegister a, const Address& b);
554   void comisd(XmmRegister a, XmmRegister b);
555   void comisd(XmmRegister a, const Address& b);
556   void ucomiss(XmmRegister a, XmmRegister b);
557   void ucomiss(XmmRegister a, const Address& b);
558   void ucomisd(XmmRegister a, XmmRegister b);
559   void ucomisd(XmmRegister a, const Address& b);
560 
561   void roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm);
562   void roundss(XmmRegister dst, XmmRegister src, const Immediate& imm);
563 
564   void sqrtsd(XmmRegister dst, XmmRegister src);
565   void sqrtss(XmmRegister dst, XmmRegister src);
566 
567   void xorpd(XmmRegister dst, const Address& src);
568   void xorpd(XmmRegister dst, XmmRegister src);
569   void xorps(XmmRegister dst, const Address& src);
570   void xorps(XmmRegister dst, XmmRegister src);
571   void pxor(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
572   void vpxor(XmmRegister dst, XmmRegister src1, XmmRegister src2);
573   void vxorps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
574   void vxorpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
575 
576   void andpd(XmmRegister dst, XmmRegister src);
577   void andpd(XmmRegister dst, const Address& src);
578   void andps(XmmRegister dst, XmmRegister src);
579   void andps(XmmRegister dst, const Address& src);
580   void pand(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
581   void vpand(XmmRegister dst, XmmRegister src1, XmmRegister src2);
582   void vandps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
583   void vandpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
584 
585   void andn(Register dst, Register src1, Register src2);  // no addr variant (for now)
586   void andnpd(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
587   void andnps(XmmRegister dst, XmmRegister src);
588   void pandn(XmmRegister dst, XmmRegister src);
589   void vpandn(XmmRegister dst, XmmRegister src1, XmmRegister src2);
590   void vandnps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
591   void vandnpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
592 
593   void orpd(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
594   void orps(XmmRegister dst, XmmRegister src);
595   void por(XmmRegister dst, XmmRegister src);
596   void vpor(XmmRegister dst, XmmRegister src1, XmmRegister src2);
597   void vorps(XmmRegister dst, XmmRegister src1, XmmRegister src2);
598   void vorpd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
599 
600   void pavgb(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
601   void pavgw(XmmRegister dst, XmmRegister src);
602   void psadbw(XmmRegister dst, XmmRegister src);
603   void pmaddwd(XmmRegister dst, XmmRegister src);
604   void vpmaddwd(XmmRegister dst, XmmRegister src1, XmmRegister src2);
605   void phaddw(XmmRegister dst, XmmRegister src);
606   void phaddd(XmmRegister dst, XmmRegister src);
607   void haddps(XmmRegister dst, XmmRegister src);
608   void haddpd(XmmRegister dst, XmmRegister src);
609   void phsubw(XmmRegister dst, XmmRegister src);
610   void phsubd(XmmRegister dst, XmmRegister src);
611   void hsubps(XmmRegister dst, XmmRegister src);
612   void hsubpd(XmmRegister dst, XmmRegister src);
613 
614   void pminsb(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
615   void pmaxsb(XmmRegister dst, XmmRegister src);
616   void pminsw(XmmRegister dst, XmmRegister src);
617   void pmaxsw(XmmRegister dst, XmmRegister src);
618   void pminsd(XmmRegister dst, XmmRegister src);
619   void pmaxsd(XmmRegister dst, XmmRegister src);
620 
621   void pminub(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
622   void pmaxub(XmmRegister dst, XmmRegister src);
623   void pminuw(XmmRegister dst, XmmRegister src);
624   void pmaxuw(XmmRegister dst, XmmRegister src);
625   void pminud(XmmRegister dst, XmmRegister src);
626   void pmaxud(XmmRegister dst, XmmRegister src);
627 
628   void minps(XmmRegister dst, XmmRegister src);  // no addr variant (for now)
629   void maxps(XmmRegister dst, XmmRegister src);
630   void minpd(XmmRegister dst, XmmRegister src);
631   void maxpd(XmmRegister dst, XmmRegister src);
632 
633   void pcmpeqb(XmmRegister dst, XmmRegister src);
634   void pcmpeqw(XmmRegister dst, XmmRegister src);
635   void pcmpeqd(XmmRegister dst, XmmRegister src);
636   void pcmpeqq(XmmRegister dst, XmmRegister src);
637 
638   void pcmpgtb(XmmRegister dst, XmmRegister src);
639   void pcmpgtw(XmmRegister dst, XmmRegister src);
640   void pcmpgtd(XmmRegister dst, XmmRegister src);
641   void pcmpgtq(XmmRegister dst, XmmRegister src);  // SSE4.2
642 
643   void shufpd(XmmRegister dst, XmmRegister src, const Immediate& imm);
644   void shufps(XmmRegister dst, XmmRegister src, const Immediate& imm);
645   void pshufd(XmmRegister dst, XmmRegister src, const Immediate& imm);
646 
647   void punpcklbw(XmmRegister dst, XmmRegister src);
648   void punpcklwd(XmmRegister dst, XmmRegister src);
649   void punpckldq(XmmRegister dst, XmmRegister src);
650   void punpcklqdq(XmmRegister dst, XmmRegister src);
651 
652   void punpckhbw(XmmRegister dst, XmmRegister src);
653   void punpckhwd(XmmRegister dst, XmmRegister src);
654   void punpckhdq(XmmRegister dst, XmmRegister src);
655   void punpckhqdq(XmmRegister dst, XmmRegister src);
656 
657   void psllw(XmmRegister reg, const Immediate& shift_count);
658   void pslld(XmmRegister reg, const Immediate& shift_count);
659   void psllq(XmmRegister reg, const Immediate& shift_count);
660 
661   void psraw(XmmRegister reg, const Immediate& shift_count);
662   void psrad(XmmRegister reg, const Immediate& shift_count);
663   // no psraq
664 
665   void psrlw(XmmRegister reg, const Immediate& shift_count);
666   void psrld(XmmRegister reg, const Immediate& shift_count);
667   void psrlq(XmmRegister reg, const Immediate& shift_count);
668   void psrldq(XmmRegister reg, const Immediate& shift_count);
669 
670   void flds(const Address& src);
671   void fstps(const Address& dst);
672   void fsts(const Address& dst);
673 
674   void fldl(const Address& src);
675   void fstpl(const Address& dst);
676   void fstl(const Address& dst);
677 
678   void fstsw();
679 
680   void fucompp();
681 
682   void fnstcw(const Address& dst);
683   void fldcw(const Address& src);
684 
685   void fistpl(const Address& dst);
686   void fistps(const Address& dst);
687   void fildl(const Address& src);
688   void filds(const Address& src);
689 
690   void fincstp();
691   void ffree(const Immediate& index);
692 
693   void fsin();
694   void fcos();
695   void fptan();
696   void fprem();
697 
698   void xchgb(ByteRegister reg, const Address& address);
699   void xchgb(Register reg, const Address& address);
700   void xchgw(Register reg, const Address& address);
701 
702   void xchgl(Register dst, Register src);
703   void xchgl(Register reg, const Address& address);
704 
705   void cmpb(const Address& address, const Immediate& imm);
706   void cmpw(const Address& address, const Immediate& imm);
707 
708   void cmpl(Register reg, const Immediate& imm);
709   void cmpl(Register reg0, Register reg1);
710   void cmpl(Register reg, const Address& address);
711 
712   void cmpl(const Address& address, Register reg);
713   void cmpl(const Address& address, const Immediate& imm);
714 
715   void testl(Register reg1, Register reg2);
716   void testl(Register reg, const Immediate& imm);
717   void testl(Register reg1, const Address& address);
718 
719   void testb(const Address& dst, const Immediate& imm);
720   void testl(const Address& dst, const Immediate& imm);
721 
722   void andl(Register dst, const Immediate& imm);
723   void andl(Register dst, Register src);
724   void andl(Register dst, const Address& address);
725   void andw(const Address& address, const Immediate& imm);
726 
727   void orl(Register dst, const Immediate& imm);
728   void orl(Register dst, Register src);
729   void orl(Register dst, const Address& address);
730 
731   void xorl(Register dst, Register src);
732   void xorl(Register dst, const Immediate& imm);
733   void xorl(Register dst, const Address& address);
734 
735   void addl(Register dst, Register src);
736   void addl(Register reg, const Immediate& imm);
737   void addl(Register reg, const Address& address);
738 
739   void addl(const Address& address, Register reg);
740   void addl(const Address& address, const Immediate& imm);
741   void addw(const Address& address, const Immediate& imm);
742 
743   void adcl(Register dst, Register src);
744   void adcl(Register reg, const Immediate& imm);
745   void adcl(Register dst, const Address& address);
746 
747   void subl(Register dst, Register src);
748   void subl(Register reg, const Immediate& imm);
749   void subl(Register reg, const Address& address);
750   void subl(const Address& address, Register src);
751 
752   void cdq();
753 
754   void idivl(Register reg);
755   void divl(Register reg);
756 
757   void imull(Register dst, Register src);
758   void imull(Register reg, const Immediate& imm);
759   void imull(Register dst, Register src, const Immediate& imm);
760   void imull(Register reg, const Address& address);
761 
762   void imull(Register reg);
763   void imull(const Address& address);
764 
765   void mull(Register reg);
766   void mull(const Address& address);
767 
768   void sbbl(Register dst, Register src);
769   void sbbl(Register reg, const Immediate& imm);
770   void sbbl(Register reg, const Address& address);
771   void sbbl(const Address& address, Register src);
772 
773   void incl(Register reg);
774   void incl(const Address& address);
775 
776   void decl(Register reg);
777   void decl(const Address& address);
778 
779   void shll(Register reg, const Immediate& imm);
780   void shll(Register operand, Register shifter);
781   void shll(const Address& address, const Immediate& imm);
782   void shll(const Address& address, Register shifter);
783   void shrl(Register reg, const Immediate& imm);
784   void shrl(Register operand, Register shifter);
785   void shrl(const Address& address, const Immediate& imm);
786   void shrl(const Address& address, Register shifter);
787   void sarl(Register reg, const Immediate& imm);
788   void sarl(Register operand, Register shifter);
789   void sarl(const Address& address, const Immediate& imm);
790   void sarl(const Address& address, Register shifter);
791   void shld(Register dst, Register src, Register shifter);
792   void shld(Register dst, Register src, const Immediate& imm);
793   void shrd(Register dst, Register src, Register shifter);
794   void shrd(Register dst, Register src, const Immediate& imm);
795 
796   void negl(Register reg);
797   void notl(Register reg);
798 
799   void enter(const Immediate& imm);
800   void leave();
801 
802   void ret();
803   void ret(const Immediate& imm);
804 
805   void nop();
806   void int3();
807   void hlt();
808 
809   void j(Condition condition, Label* label);
810   void j(Condition condition, NearLabel* label);
811   void jecxz(NearLabel* label);
812 
813   void jmp(Register reg);
814   void jmp(const Address& address);
815   void jmp(Label* label);
816   void jmp(NearLabel* label);
817 
818   void repne_scasb();
819   void repne_scasw();
820   void repe_cmpsb();
821   void repe_cmpsw();
822   void repe_cmpsl();
823   void rep_movsb();
824   void rep_movsw();
825 
826   X86Assembler* lock();
827   void cmpxchgb(const Address& address, ByteRegister reg);
828   void cmpxchgw(const Address& address, Register reg);
829   void cmpxchgl(const Address& address, Register reg);
830   void cmpxchg8b(const Address& address);
831 
832   void xaddb(const Address& address, ByteRegister reg);
833   void xaddw(const Address& address, Register reg);
834   void xaddl(const Address& address, Register reg);
835 
836   void mfence();
837 
838   X86Assembler* fs();
839   X86Assembler* gs();
840 
841   //
842   // Macros for High-level operations.
843   //
844 
845   void AddImmediate(Register reg, const Immediate& imm);
846 
847   void LoadLongConstant(XmmRegister dst, int64_t value);
848   void LoadDoubleConstant(XmmRegister dst, double value);
849 
LockCmpxchgb(const Address & address,Register reg)850   void LockCmpxchgb(const Address& address, Register reg) {
851     // For testing purpose
852     lock()->cmpxchgb(address, static_cast<ByteRegister>(reg));
853   }
854 
LockCmpxchgb(const Address & address,ByteRegister reg)855   void LockCmpxchgb(const Address& address, ByteRegister reg) {
856     lock()->cmpxchgb(address, reg);
857   }
858 
LockCmpxchgw(const Address & address,Register reg)859   void LockCmpxchgw(const Address& address, Register reg) {
860     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
861     // We make sure that the operand size override bytecode is emited before the lock bytecode.
862     // We test against clang which enforces this bytecode order.
863     EmitOperandSizeOverride();
864     EmitUint8(0xF0);
865     EmitUint8(0x0F);
866     EmitUint8(0xB1);
867     EmitOperand(reg, address);
868   }
869 
LockCmpxchgl(const Address & address,Register reg)870   void LockCmpxchgl(const Address& address, Register reg) {
871     lock()->cmpxchgl(address, reg);
872   }
873 
LockCmpxchg8b(const Address & address)874   void LockCmpxchg8b(const Address& address) {
875     lock()->cmpxchg8b(address);
876   }
877 
LockXaddb(const Address & address,Register reg)878   void LockXaddb(const Address& address, Register reg) {
879     // For testing purpose
880     lock()->xaddb(address, static_cast<ByteRegister>(reg));
881   }
882 
LockXaddb(const Address & address,ByteRegister reg)883   void LockXaddb(const Address& address, ByteRegister reg) {
884     lock()->xaddb(address, reg);
885   }
886 
LockXaddw(const Address & address,Register reg)887   void LockXaddw(const Address& address, Register reg) {
888     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
889     // We make sure that the operand size override bytecode is emited before the lock bytecode.
890     // We test against clang which enforces this bytecode order.
891     EmitOperandSizeOverride();
892     EmitUint8(0xF0);
893     EmitUint8(0x0F);
894     EmitUint8(0xC1);
895     EmitOperand(reg, address);
896   }
897 
LockXaddl(const Address & address,Register reg)898   void LockXaddl(const Address& address, Register reg) {
899     lock()->xaddl(address, reg);
900   }
901 
902   //
903   // Misc. functionality
904   //
PreferredLoopAlignment()905   int PreferredLoopAlignment() { return 16; }
906   void Align(int alignment, int offset);
907   void Bind(Label* label) override;
Jump(Label * label)908   void Jump(Label* label) override {
909     jmp(label);
910   }
911   void Bind(NearLabel* label);
912 
913   //
914   // Heap poisoning.
915   //
916 
917   // Poison a heap reference contained in `reg`.
PoisonHeapReference(Register reg)918   void PoisonHeapReference(Register reg) { negl(reg); }
919   // Unpoison a heap reference contained in `reg`.
UnpoisonHeapReference(Register reg)920   void UnpoisonHeapReference(Register reg) { negl(reg); }
921   // Poison a heap reference contained in `reg` if heap poisoning is enabled.
MaybePoisonHeapReference(Register reg)922   void MaybePoisonHeapReference(Register reg) {
923     if (kPoisonHeapReferences) {
924       PoisonHeapReference(reg);
925     }
926   }
927   // Unpoison a heap reference contained in `reg` if heap poisoning is enabled.
MaybeUnpoisonHeapReference(Register reg)928   void MaybeUnpoisonHeapReference(Register reg) {
929     if (kPoisonHeapReferences) {
930       UnpoisonHeapReference(reg);
931     }
932   }
933 
934   // Add a double to the constant area, returning the offset into
935   // the constant area where the literal resides.
AddDouble(double v)936   size_t AddDouble(double v) { return constant_area_.AddDouble(v); }
937 
938   // Add a float to the constant area, returning the offset into
939   // the constant area where the literal resides.
AddFloat(float v)940   size_t AddFloat(float v)   { return constant_area_.AddFloat(v); }
941 
942   // Add an int32_t to the constant area, returning the offset into
943   // the constant area where the literal resides.
AddInt32(int32_t v)944   size_t AddInt32(int32_t v) {
945     return constant_area_.AddInt32(v);
946   }
947 
948   // Add an int32_t to the end of the constant area, returning the offset into
949   // the constant area where the literal resides.
AppendInt32(int32_t v)950   size_t AppendInt32(int32_t v) {
951     return constant_area_.AppendInt32(v);
952   }
953 
954   // Add an int64_t to the constant area, returning the offset into
955   // the constant area where the literal resides.
AddInt64(int64_t v)956   size_t AddInt64(int64_t v) { return constant_area_.AddInt64(v); }
957 
958   // Add the contents of the constant area to the assembler buffer.
959   void AddConstantArea();
960 
961   // Is the constant area empty? Return true if there are no literals in the constant area.
IsConstantAreaEmpty()962   bool IsConstantAreaEmpty() const { return constant_area_.IsEmpty(); }
963 
964   // Return the current size of the constant area.
ConstantAreaSize()965   size_t ConstantAreaSize() const { return constant_area_.GetSize(); }
966 
967   bool CpuHasAVXorAVX2FeatureFlag();
968 
969  private:
970   inline void EmitUint8(uint8_t value);
971   inline void EmitInt32(int32_t value);
972   inline void EmitRegisterOperand(int rm, int reg);
973   inline void EmitXmmRegisterOperand(int rm, XmmRegister reg);
974   inline void EmitFixup(AssemblerFixup* fixup);
975   inline void EmitOperandSizeOverride();
976 
977   void EmitOperand(int rm, const Operand& operand);
978   void EmitImmediate(const Immediate& imm, bool is_16_op = false);
979   void EmitComplex(
980       int rm, const Operand& operand, const Immediate& immediate, bool is_16_op = false);
981   void EmitLabel(Label* label, int instruction_size);
982   void EmitLabelLink(Label* label);
983   void EmitLabelLink(NearLabel* label);
984 
985   void EmitGenericShift(int rm, const Operand& operand, const Immediate& imm);
986   void EmitGenericShift(int rm, const Operand& operand, Register shifter);
987 
988   uint8_t EmitVexPrefixByteZero(bool is_twobyte_form);
989   uint8_t EmitVexPrefixByteOne(bool R, bool X, bool B, int SET_VEX_M);
990   uint8_t EmitVexPrefixByteOne(bool R,
991                                X86ManagedRegister operand,
992                                int SET_VEX_L,
993                                int SET_VEX_PP);
994   uint8_t EmitVexPrefixByteTwo(bool W,
995                                X86ManagedRegister operand,
996                                int SET_VEX_L,
997                                int SET_VEX_PP);
998   uint8_t EmitVexPrefixByteTwo(bool W,
999                                int SET_VEX_L,
1000                                int SET_VEX_PP);
1001   ConstantArea constant_area_;
1002   bool has_AVX_;     // x86 256bit SIMD AVX.
1003   bool has_AVX2_;    // x86 256bit SIMD AVX 2.0.
1004 
1005   DISALLOW_COPY_AND_ASSIGN(X86Assembler);
1006 };
1007 
EmitUint8(uint8_t value)1008 inline void X86Assembler::EmitUint8(uint8_t value) {
1009   buffer_.Emit<uint8_t>(value);
1010 }
1011 
EmitInt32(int32_t value)1012 inline void X86Assembler::EmitInt32(int32_t value) {
1013   buffer_.Emit<int32_t>(value);
1014 }
1015 
EmitRegisterOperand(int rm,int reg)1016 inline void X86Assembler::EmitRegisterOperand(int rm, int reg) {
1017   CHECK_GE(rm, 0);
1018   CHECK_LT(rm, 8);
1019   buffer_.Emit<uint8_t>(0xC0 + (rm << 3) + reg);
1020 }
1021 
EmitXmmRegisterOperand(int rm,XmmRegister reg)1022 inline void X86Assembler::EmitXmmRegisterOperand(int rm, XmmRegister reg) {
1023   EmitRegisterOperand(rm, static_cast<Register>(reg));
1024 }
1025 
EmitFixup(AssemblerFixup * fixup)1026 inline void X86Assembler::EmitFixup(AssemblerFixup* fixup) {
1027   buffer_.EmitFixup(fixup);
1028 }
1029 
EmitOperandSizeOverride()1030 inline void X86Assembler::EmitOperandSizeOverride() {
1031   EmitUint8(0x66);
1032 }
1033 
1034 }  // namespace x86
1035 }  // namespace art
1036 
1037 #endif  // ART_COMPILER_UTILS_X86_ASSEMBLER_X86_H_
1038