35 void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
53 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N);
break;
54 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N);
break;
58 Res = PromoteIntRes_CONVERT_RNDSAT(N);
break;
60 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N);
break;
61 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N);
break;
63 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N);
break;
65 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N);
break;
66 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));
break;
67 case ISD::SELECT: Res = PromoteIntRes_SELECT(N);
break;
68 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N);
break;
70 case ISD::SETCC: Res = PromoteIntRes_SETCC(N);
break;
71 case ISD::SHL: Res = PromoteIntRes_SHL(N);
break;
73 Res = PromoteIntRes_SIGN_EXTEND_INREG(N);
break;
74 case ISD::SRA: Res = PromoteIntRes_SRA(N);
break;
75 case ISD::SRL: Res = PromoteIntRes_SRL(N);
break;
77 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N);
break;
78 case ISD::VAARG: Res = PromoteIntRes_VAARG(N);
break;
81 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N);
break;
83 Res = PromoteIntRes_VECTOR_SHUFFLE(N);
break;
85 Res = PromoteIntRes_INSERT_VECTOR_ELT(N);
break;
87 Res = PromoteIntRes_BUILD_VECTOR(N);
break;
89 Res = PromoteIntRes_SCALAR_TO_VECTOR(N);
break;
91 Res = PromoteIntRes_CONCAT_VECTORS(N);
break;
107 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N);
break;
110 case ISD::SREM: Res = PromoteIntRes_SDIV(N);
break;
113 case ISD::UREM: Res = PromoteIntRes_UDIV(N);
break;
116 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo);
break;
118 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo);
break;
120 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo);
break;
123 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N));
break;
136 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N));
break;
139 Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N));
break;
144 SetPromotedInteger(
SDValue(N, ResNo), Res);
147 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(
SDNode *N,
149 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
150 return GetPromotedInteger(Op);
153 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(
SDNode *N) {
160 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(
SDNode *N) {
214 switch (getTypeAction(InVT)) {
232 BitConvertToInteger(GetScalarizedVector(InOp)));
239 Lo = BitConvertToInteger(Lo);
240 Hi = BitConvertToInteger(Hi);
248 JoinIntegers(Lo, Hi));
260 CreateStackStoreLoad(InOp, OutVT));
269 unsigned DiffBits = NVT.
getSizeInBits() - OVT.getSizeInBits();
274 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(
SDNode *N) {
283 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(
SDNode *N) {
293 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
297 SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(
SDNode *N) {
302 "can only promote integers");
338 TopBit.
setBit(OVT.getSizeInBits());
344 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *N) {
351 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(
SDNode *N) {
375 SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(
SDNode *N) {
385 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(
SDNode *N) {
429 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(
SDNode *N) {
439 ReplaceValueWith(
SDValue(N, 0), Res);
444 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *N,
unsigned ResNo) {
446 return PromoteIntRes_Overflow(N);
468 ReplaceValueWith(
SDValue(N, 1), Ofl);
493 Mask = PromoteTargetBoolean(Mask, getSetCCResultType(OpTy));
500 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(
SDNode *N) {
520 "Vector compare must return a vector result!");
526 !LHS.getValueType().isVector())
527 LHS = GetPromotedInteger(LHS);
530 RHS = GetPromotedInteger(RHS);
537 assert(NVT.bitsLE(SVT) &&
"Integer type overpromoted?");
549 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *N) {
555 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *N) {
581 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(
SDNode *N) {
594 Res = GetPromotedInteger(InOp);
598 assert(InVT.
isVector() &&
"Cannot split scalar types");
601 "Dst and Src must have the same number of elements");
603 "Promoted vector type must be a power of two");
606 GetSplitVector(InOp, EOp1, EOp2);
620 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *N,
unsigned ResNo) {
622 return PromoteIntRes_Overflow(N);
643 ReplaceValueWith(
SDValue(N, 1), Ofl);
648 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *N,
unsigned ResNo) {
651 return PromoteIntRes_Overflow(N);
662 LHS = SExtPromotedInteger(LHS);
663 RHS = SExtPromotedInteger(RHS);
665 LHS = ZExtPromotedInteger(LHS);
666 RHS = ZExtPromotedInteger(RHS);
694 ReplaceValueWith(
SDValue(N, 1), Overflow);
722 for (
unsigned i = 0; i < NumRegs; ++i) {
730 std::reverse(Parts.begin(), Parts.end());
735 for (
unsigned i = 1; i < NumRegs; ++i) {
746 ReplaceValueWith(
SDValue(N, 1), Chain);
759 bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *N,
unsigned OpNo) {
769 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
776 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
778 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N);
break;
779 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo);
break;
780 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo);
break;
786 Res = PromoteIntOp_CONVERT_RNDSAT(N);
break;
788 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
break;
790 Res = PromoteIntOp_SCALAR_TO_VECTOR(N);
break;
792 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo);
break;
793 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo);
break;
794 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo);
break;
797 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
808 case ISD::ROTR: Res = PromoteIntOp_Shift(N);
break;
812 if (!Res.getNode())
return false;
816 if (Res.getNode() ==
N)
820 "Invalid operand expansion");
822 ReplaceValueWith(
SDValue(N, 0), Res);
828 void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &NewLHS,
SDValue &NewRHS,
844 NewLHS = ZExtPromotedInteger(NewLHS);
845 NewRHS = ZExtPromotedInteger(NewRHS);
851 NewLHS = SExtPromotedInteger(NewLHS);
852 NewRHS = SExtPromotedInteger(NewRHS);
857 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(
SDNode *N) {
875 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *N,
unsigned OpNo) {
876 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
880 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->
getOperand(1))->
get());
889 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *N,
unsigned OpNo) {
890 assert(OpNo == 1 &&
"only know how to promote condition");
901 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(
SDNode *N) {
914 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(
SDNode *N) {
920 assert(!((NumElts & 1) && (!TLI.
isTypeLegal(VecVT))) &&
921 "Legal vector of one illegal element?");
928 "Type of inserted value narrower than vector element type!");
931 for (
unsigned i = 0; i < NumElts; ++i)
937 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(
SDNode *N) {
938 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(
N)->getCvtCode();
942 "can only promote integer arguments");
949 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *N,
958 "Type of inserted value narrower than vector element type!");
965 assert(OpNo == 2 &&
"Different operand and result vector types?");
974 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(
SDNode *N) {
981 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *N,
unsigned OpNo) {
982 assert(OpNo == 0 &&
"Only know how to promote the condition!");
989 Cond = PromoteTargetBoolean(Cond, SVT);
995 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *N,
unsigned OpNo) {
996 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1000 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->
getOperand(4))->
get());
1007 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *N,
unsigned OpNo) {
1008 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1012 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->
getOperand(2))->
get());
1023 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(
SDNode *N) {
1031 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(
SDNode *N) {
1048 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(
SDNode *N) {
1053 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(
SDNode *N) {
1058 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(
SDNode *N) {
1075 void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *N,
unsigned ResNo) {
1087 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
1090 llvm_unreachable(
"Do not know how to expand the result of this operator!");
1093 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi);
break;
1095 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi);
break;
1097 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi);
break;
1101 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi);
break;
1106 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi);
break;
1107 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi);
break;
1109 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi);
break;
1110 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi);
break;
1112 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi);
break;
1115 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
break;
1116 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi);
break;
1117 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi);
break;
1120 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi);
break;
1121 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi);
break;
1122 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi);
break;
1123 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi);
break;
1139 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1140 SplitInteger(Tmp.first, Lo, Hi);
1141 ReplaceValueWith(
SDValue(N, 1), Tmp.second);
1147 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi);
break;
1150 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi);
break;
1153 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi);
break;
1156 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi);
break;
1160 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi);
break;
1163 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi);
break;
1165 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi);
break;
1167 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi);
break;
1176 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *Node) {
1266 return ExpandChainLibCall(LC, Node,
false);
1271 void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *N,
unsigned Amt,
1276 GetExpandedInteger(N->
getOperand(0), InL, InH);
1280 unsigned NVTBits = NVT.getSizeInBits();
1286 }
else if (Amt > NVTBits) {
1290 }
else if (Amt == NVTBits) {
1293 }
else if (Amt == 1 &&
1298 SDValue LoOps[2] = { InL, InL };
1317 }
else if (Amt > NVTBits) {
1321 }
else if (Amt == NVTBits) {
1339 }
else if (Amt > NVTBits) {
1344 }
else if (Amt == NVTBits) {
1362 bool DAGTypeLegalizer::
1370 "Expanded integer type size not a power of two!");
1374 APInt KnownZero, KnownOne;
1378 if (((KnownZero|KnownOne) & HighBitMask) == 0)
1383 GetExpandedInteger(N->
getOperand(0), InL, InH);
1412 if ((KnownZero & HighBitMask) == HighBitMask) {
1450 bool DAGTypeLegalizer::
1457 "Expanded integer type size not a power of two!");
1462 GetExpandedInteger(N->
getOperand(0), InL, InH);
1486 Lo = DAG.
getSelect(dl, NVT, isShort, LoS, LoL);
1487 Hi = DAG.
getSelect(dl, NVT, isShort, HiS, HiL);
1502 Lo = DAG.
getSelect(dl, NVT, isShort, LoS, LoL);
1503 Hi = DAG.
getSelect(dl, NVT, isShort, HiS, HiL);
1519 Lo = DAG.
getSelect(dl, NVT, isShort, LoS, LoL);
1520 Hi = DAG.
getSelect(dl, NVT, isShort, HiS, HiL);
1525 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *N,
1529 SDValue LHSL, LHSH, RHSL, RHSH;
1530 GetExpandedInteger(N->
getOperand(0), LHSL, LHSH);
1531 GetExpandedInteger(N->
getOperand(1), RHSL, RHSH);
1534 SDValue LoOps[2] = { LHSL, RHSL };
1535 SDValue HiOps[3] = { LHSH, RHSH };
1564 SDValue Cmp1 = DAG.
getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1569 SDValue Cmp2 = DAG.
getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1578 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1587 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *N,
1590 SDValue LHSL, LHSH, RHSL, RHSH;
1592 GetExpandedInteger(N->
getOperand(0), LHSL, LHSH);
1593 GetExpandedInteger(N->
getOperand(1), RHSL, RHSH);
1595 SDValue LoOps[2] = { LHSL, RHSL };
1596 SDValue HiOps[3] = { LHSH, RHSH };
1613 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *N,
1616 SDValue LHSL, LHSH, RHSL, RHSH;
1618 GetExpandedInteger(N->
getOperand(0), LHSL, LHSH);
1619 GetExpandedInteger(N->
getOperand(1), RHSL, RHSH);
1622 SDValue HiOps[3] = { LHSH, RHSH };
1633 void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(
SDNode *N,
unsigned ResNo,
1635 SDValue Res = DisintegrateMERGE_VALUES(N, ResNo);
1636 SplitInteger(Res, Lo, Hi);
1639 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *N,
1653 "Only know how to promote this result!");
1654 SDValue Res = GetPromotedInteger(Op);
1656 "Operand over promoted?");
1658 SplitInteger(Res, Lo, Hi);
1662 void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *N,
1668 unsigned NVTBits = NVT.getSizeInBits();
1671 if (NVTBits < EVTBits) {
1674 EVTBits - NVTBits)));
1683 void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *N,
1688 EVT EVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
1689 unsigned NVTBits = NVT.getSizeInBits();
1692 if (NVTBits < EVTBits) {
1695 EVTBits - NVTBits)));
1703 void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *N,
1711 void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *N,
1715 const APInt &Cst = cast<ConstantSDNode>(
N)->getAPIntValue();
1720 void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *N,
1733 Lo = DAG.
getSelect(dl, NVT, HiNotZero, HiLZ,
1739 void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *N,
1750 void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *N,
1763 Lo = DAG.
getSelect(dl, NVT, LoNotZero, LoLZ,
1769 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(
SDNode *N,
SDValue &Lo,
1776 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, &Op, 1,
true,
1781 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(
SDNode *N,
SDValue &Lo,
1788 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, &Op, 1,
false,
1793 void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *N,
1796 ExpandRes_NormalLoad(N, Lo, Hi);
1814 assert(NVT.
isByteSized() &&
"Expanded type not byte sized!");
1820 MemVT, isVolatile, isNonTemporal, Alignment, TBAAInfo);
1842 isVolatile, isNonTemporal, isInvariant, Alignment,
1845 unsigned ExcessBits =
1853 Hi = DAG.
getExtLoad(ExtType, dl, NVT, Ch, Ptr,
1855 isVolatile, isNonTemporal,
1856 MinAlign(Alignment, IncrementSize), TBAAInfo);
1868 unsigned ExcessBits = (EBytes - IncrementSize)*8;
1874 isVolatile, isNonTemporal, Alignment, TBAAInfo);
1883 isVolatile, isNonTemporal,
1884 MinAlign(Alignment, IncrementSize), TBAAInfo);
1907 ReplaceValueWith(
SDValue(N, 1), Ch);
1910 void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *N,
1914 GetExpandedInteger(N->
getOperand(0), LL, LH);
1915 GetExpandedInteger(N->
getOperand(1), RL, RH);
1920 void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *N,
1930 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
1932 GetExpandedInteger(N->
getOperand(0), LL, LH);
1933 GetExpandedInteger(N->
getOperand(1), RL, RH);
1956 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) {
2007 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2,
true,
2012 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *Node,
2023 SplitInteger(Sum, Lo, Hi);
2051 ReplaceValueWith(
SDValue(Node, 1), Cmp);
2054 void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *N,
2071 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2,
true, dl).first,
Lo,
Hi);
2074 void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *N,
2082 return ExpandShiftByConstant(N, CN->getZExtValue(),
Lo,
Hi);
2086 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2108 GetExpandedInteger(N->
getOperand(0), LHSL, LHSH);
2118 "ShiftAmountTy is too small to cover the range of this type!");
2122 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2166 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2, isSigned, dl).first,
Lo,
2171 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2175 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *N,
2192 "Only know how to promote this result!");
2193 SDValue Res = GetPromotedInteger(Op);
2195 "Operand over promoted?");
2197 SplitInteger(Res, Lo, Hi);
2198 unsigned ExcessBits =
2206 void DAGTypeLegalizer::
2210 EVT EVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
2225 unsigned ExcessBits =
2233 void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *N,
2250 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2,
true, dl).first,
Lo,
Hi);
2253 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *N,
2264 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *N,
2275 SplitInteger(Sum, Lo, Hi);
2284 ReplaceValueWith(
SDValue(N, 1), Ofl);
2287 void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *N,
2297 SplitInteger(MUL, Lo, Hi);
2311 ReplaceValueWith(
SDValue(N, 1), Overflow);
2344 Args.push_back(Entry);
2352 Args.push_back(Entry);
2360 Func, Args, DAG, dl);
2361 std::pair<SDValue, SDValue> CallInfo = TLI.
LowerCallTo(CLI);
2363 SplitInteger(CallInfo.first, Lo, Hi);
2364 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp,
2367 DAG.getConstant(0, PtrVT),
2370 ReplaceValueWith(
SDValue(N, 1), Ofl);
2373 void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *N,
2390 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2,
false, dl).first,
Lo,
Hi);
2393 void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *N,
2410 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, 2,
false, dl).first,
Lo,
Hi);
2413 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *N,
2421 Hi = DAG.getConstant(0, NVT);
2427 "Only know how to promote this result!");
2428 SDValue Res = GetPromotedInteger(Op);
2430 "Operand over promoted?");
2432 SplitInteger(Res, Lo, Hi);
2433 unsigned ExcessBits =
2435 Hi = DAG.getZeroExtendInReg(Hi, dl,
2441 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *N,
2444 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
2445 SDValue Zero = DAG.getConstant(0, VT);
2449 cast<AtomicSDNode>(
N)->getMemOperand(),
2450 cast<AtomicSDNode>(
N)->getOrdering(),
2451 cast<AtomicSDNode>(
N)->getSynchScope());
2464 bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *N,
unsigned OpNo) {
2474 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
2480 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N);
break;
2486 case ISD::SETCC: Res = ExpandIntOp_SETCC(N);
break;
2488 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo);
break;
2496 case ISD::ROTR: Res = ExpandIntOp_Shift(N);
break;
2504 if (!Res.getNode())
return false;
2508 if (Res.getNode() ==
N)
2512 "Invalid operand expansion");
2514 ReplaceValueWith(
SDValue(N, 0), Res);
2520 void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
2524 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2525 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2526 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2529 if (RHSLo == RHSHi) {
2531 if (RHSCST->isAllOnesValue()) {
2551 if ((CCCode ==
ISD::SETLT && CST->isNullValue()) ||
2552 (CCCode ==
ISD::SETGT && CST->isAllOnesValue())) {
2581 LHSLo, RHSLo, LowCC,
false, DagCombineInfo, dl);
2583 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.
getValueType()),
2584 LHSLo, RHSLo, LowCC);
2586 LHSHi, RHSHi, CCCode,
false, DagCombineInfo, dl);
2590 LHSHi, RHSHi, DAG.getCondCode(CCCode));
2595 (Tmp2C && Tmp2C->isNullValue() &&
2598 (Tmp2C && Tmp2C->getAPIntValue() == 1 &&
2611 DagCombineInfo, dl);
2613 NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.
getValueType()),
2616 NewLHS, Tmp1, Tmp2);
2623 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(N));
2634 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2638 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(
SDNode *N) {
2641 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(N));
2651 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2653 DAG.getCondCode(CCCode)), 0);
2659 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(N));
2664 "Unexpected setcc expansion!");
2669 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2670 DAG.getCondCode(CCCode)), 0);
2682 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(
SDNode *N) {
2688 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
2691 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(
SDNode *N) {
2696 "Don't know how to expand this SINT_TO_FP!");
2702 return ExpandOp_NormalStore(N, OpNo);
2705 assert(OpNo == 1 &&
"Can only expand the stored value so far");
2718 assert(NVT.
isByteSized() &&
"Expanded type not byte sized!");
2724 Alignment, TBAAInfo);
2732 isVolatile, isNonTemporal, Alignment, TBAAInfo);
2734 unsigned ExcessBits =
2742 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
2744 NEVT, isVolatile, isNonTemporal,
2745 MinAlign(Alignment, IncrementSize), TBAAInfo);
2754 unsigned EBytes = ExtVT.getStoreSize();
2756 unsigned ExcessBits = (EBytes - IncrementSize)*8;
2758 ExtVT.getSizeInBits() - ExcessBits);
2767 DAG.getConstant(ExcessBits,
2773 HiVT, isVolatile, isNonTemporal, Alignment, TBAAInfo);
2779 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
2782 isVolatile, isNonTemporal,
2783 MinAlign(Alignment, IncrementSize), TBAAInfo);
2789 GetExpandedInteger(N->
getOperand(0), InL, InH);
2794 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(
SDNode *N) {
2803 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
2814 const uint64_t F32TwoE32 = 0x4F800000ULL;
2815 const uint64_t F32TwoE64 = 0x5F800000ULL;
2816 const uint64_t F32TwoE128 = 0x7F800000ULL;
2820 FF =
APInt(32, F32TwoE32);
2822 FF =
APInt(32, F32TwoE64);
2824 FF =
APInt(32, F32TwoE128);
2830 GetExpandedInteger(Op, Lo, Hi);
2831 SDValue SignSet = DAG.getSetCC(dl,
2837 SDValue FudgePtr = DAG.getConstantPool(
2842 SDValue Zero = DAG.getIntPtrConstant(0);
2843 SDValue Four = DAG.getIntPtrConstant(4);
2847 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
2850 Alignment = std::min(Alignment, 4u);
2858 false,
false, Alignment);
2865 "Don't know how to expand this UINT_TO_FP!");
2866 return TLI.
makeLibCall(DAG, LC, DstVT, &Op, 1,
true, dl).first;
2869 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(
SDNode *N) {
2872 cast<AtomicSDNode>(N)->getMemoryVT(),
2875 cast<AtomicSDNode>(
N)->getMemOperand(),
2876 cast<AtomicSDNode>(
N)->getOrdering(),
2877 cast<AtomicSDNode>(
N)->getSynchScope());
2882 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *N) {
2888 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
2897 for (
unsigned i = 0; i != OutNumElems; ++i) {
2914 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *N) {
2919 unsigned NumElts = VT.getVectorNumElements();
2921 for (
unsigned i = 0; i != NumElts; ++i) {
2929 return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]);
2933 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(
SDNode *N) {
2936 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
2944 for (
unsigned i = 0; i != NumElems; ++i) {
2960 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(
SDNode *N) {
2965 "Input must be a scalar");
2969 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
2977 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *N) {
2982 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
2990 assert(NumElem * NumOperands == NumOutElem &&
2991 "Unexpected number of elements");
2995 for (
unsigned i = 0; i < NumOperands; ++i) {
2997 for (
unsigned j = 0; j < NumElem; ++j) {
2999 InElemTy, Op, DAG.getConstant(j,
3001 Ops[i * NumElem + j] = DAG.getNode(
ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3008 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *N) {
3011 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
3024 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
3034 return DAG.getAnyExtOrTrunc(Ext, dl, N->
getValueType(0));
3037 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *N) {
3047 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3052 for (
unsigned i=0; i<NumElem; ++i) {
3062 &NewOps[0], NewOps.
size());
void push_back(const T &Elt)
unsigned Log2_32_Ceil(uint32_t Value)
SDValue getConstant(uint64_t Val, EVT VT, bool isTarget=false)
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
SDValue getValue(unsigned R) const
LLVMContext * getContext() const
void dump() const
dump - Dump this node, for debugging.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
void setBit(unsigned bitPosition)
Set a given bit to 1.
enable_if_c<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
unsigned getOpcode() const
Type * getTypeForEVT(LLVMContext &Context) const
unsigned getSizeInBits() const
SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy)
unsigned getNumOperands() const
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
MDNode - a tuple of other values.
const SDValue & getOperand(unsigned Num) const
static MachinePointerInfo getConstantPool()
void ComputeMaskedBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
const SDValue & getBasePtr() const
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const MDNode *TBAAInfo=0, const MDNode *Ranges=0)
bool isVector() const
isVector - Return true if this is a vector value type.
EVT getShiftAmountTy(EVT LHSTy) const
bool isLittleEndian() const
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const MDNode *TBAAInfo=0)
bool isNormalStore(const SDNode *N)
#define llvm_unreachable(msg)
EVT getValueType(unsigned ResNo) const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, unsigned NumOps, bool isSigned, SDLoc dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
APInt LLVM_ATTRIBUTE_UNUSED_RESULT lshr(unsigned shiftAmt) const
Logical right-shift function.
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
virtual MVT getPointerTy(uint32_t=0) const
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, SDLoc dl) const
EVT getVectorElementType() const
SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
bool isUNINDEXEDStore(const SDNode *N)
unsigned getNumValues() const
Simple integer binary arithmetic operators.
AtomicOrdering getOrdering() const
static unsigned int semanticsPrecision(const fltSemantics &)
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
getUNDEF - Return an UNDEF node. UNDEF does not have a useful SDLoc.
EVT getMemoryVT() const
getMemoryVT - Return the type of the in-memory value.
virtual MVT getVectorIdxTy() const
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
UNDEF - An undefined node.
const SDValue & getBasePtr() const
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
SDNode * getNode() const
get the SDNode which holds the desired result
unsigned getStoreSize() const
bool isTypeLegal(EVT VT) const
bool isNormalLoad(const SDNode *N)
SynchronizationScope getSynchScope() const
bool intersects(const APInt &RHS) const
APInt LLVM_ATTRIBUTE_UNUSED_RESULT trunc(unsigned width) const
Truncate to new width.
Simple binary floating point operators.
bool isNonTemporal() const
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
std::vector< ArgListEntry > ArgListTy
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
PointerType * getPointerTo(unsigned AddrSpace=0)
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
uint64_t getConstantOperandVal(unsigned Num) const
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
const MachinePointerInfo & getPointerInfo() const
bool isUNINDEXEDLoad(const SDNode *N)
static bool isZero(Value *V, DataLayout *DL)
SDValue getConvertRndSat(EVT VT, SDLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code)
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, unsigned Alignment, const MDNode *TBAAInfo=0)
SDValue getIntPtrConstant(uint64_t Val, bool isTarget=false)
const SDValue & getChain() const
Byte Swap and Counting operators.
MachineMemOperand * getMemOperand() const
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
raw_ostream & dbgs()
dbgs - Return a circular-buffered debug stream.
unsigned Log2_32(uint32_t Value)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
const MDNode * getTBAAInfo() const
Returns the TBAAInfo that describes the dereference.
ISD::LoadExtType getExtensionType() const
Class for arbitrary precision integers.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT)
ANY_EXTEND - Used for integer types. The high bits are undefined.
uint64_t MinAlign(uint64_t A, uint64_t B)
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
Bitwise operators - logical and, logical or, logical xor.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const MDNode *TBAAInfo=0)
bool isNON_EXTLoad(const SDNode *N)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
SDValue getValueType(EVT)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
bool isPowerOf2_32(uint32_t Value)
SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
SDValue getEntryNode() const
TRUNCATE - Completely drop the high bits.
unsigned getAlignment() const
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
unsigned getVectorNumElements() const