CGClass.cpp

//===--- CGClass.cpp - Emit LLVM Code for C++ classes ---------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This contains code dealing with C++ code generation of classes
//
//===----------------------------------------------------------------------===//

#include "CGDebugInfo.h"
#include "CodeGenFunction.h"
#include "clang/AST/CXXInheritance.h"
#include "clang/AST/EvaluatedExprVisitor.h"
#include "clang/AST/RecordLayout.h"
#include "clang/AST/StmtCXX.h"
#include "clang/Frontend/CodeGenOptions.h"

using namespace clang;
using namespace CodeGen;

static CharUnits 
ComputeNonVirtualBaseClassOffset(ASTContext &Context, 
                                 const CXXRecordDecl *DerivedClass,
                                 CastExpr::path_const_iterator Start,
                                 CastExpr::path_const_iterator End) {
  CharUnits Offset = CharUnits::Zero();
  
  const CXXRecordDecl *RD = DerivedClass;
  
  for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
    const CXXBaseSpecifier *Base = *I;
    assert(!Base->isVirtual() && "Should not see virtual bases here!");

    // Get the layout.
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
    
    const CXXRecordDecl *BaseDecl = 
      cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
    
    // Add the offset.
    Offset += Layout.getBaseClassOffset(BaseDecl);
    
    RD = BaseDecl;
  }
  
  return Offset;
}

llvm::Constant *
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
                                   CastExpr::path_const_iterator PathBegin,
                                   CastExpr::path_const_iterator PathEnd) {
  assert(PathBegin != PathEnd && "Base path should not be empty!");

  CharUnits Offset = 
    ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl,
                                     PathBegin, PathEnd);
  if (Offset.isZero())
    return 0;
  
  llvm::Type *PtrDiffTy = 
  Types.ConvertType(getContext().getPointerDiffType());
  
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
}

/// Gets the address of a direct base class within a complete object.
/// This should only be used for (1) non-virtual bases or (2) virtual bases
/// when the type is known to be complete (e.g. in complete destructors).
///
/// The object pointed to by 'This' is assumed to be non-null.
llvm::Value *
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This,
                                                   const CXXRecordDecl *Derived,
                                                   const CXXRecordDecl *Base,
                                                   bool BaseIsVirtual) {
  // 'this' must be a pointer (in some address space) to Derived.
  assert(This->getType()->isPointerTy() &&
         cast<llvm::PointerType>(This->getType())->getElementType()
           == ConvertType(Derived));

  // Compute the offset of the virtual base.
  CharUnits Offset;
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
  if (BaseIsVirtual)
    Offset = Layout.getVBaseClassOffset(Base);
  else
    Offset = Layout.getBaseClassOffset(Base);

  // Shift and cast down to the base type.
  // TODO: for complete types, this should be possible with a GEP.
  llvm::Value *V = This;
  if (Offset.isPositive()) {
    llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
    V = Builder.CreateBitCast(V, Int8PtrTy);
    V = Builder.CreateConstInBoundsGEP1_64(V, Offset.getQuantity());
  }
  V = Builder.CreateBitCast(V, ConvertType(Base)->getPointerTo());

  return V;
}

static llvm::Value *
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ThisPtr,
                                CharUnits NonVirtual, llvm::Value *Virtual) {
  llvm::Type *PtrDiffTy = 
    CGF.ConvertType(CGF.getContext().getPointerDiffType());
  
  llvm::Value *NonVirtualOffset = 0;
  if (!NonVirtual.isZero())
    NonVirtualOffset = llvm::ConstantInt::get(PtrDiffTy, 
                                              NonVirtual.getQuantity());
  
  llvm::Value *BaseOffset;
  if (Virtual) {
    if (NonVirtualOffset)
      BaseOffset = CGF.Builder.CreateAdd(Virtual, NonVirtualOffset);
    else
      BaseOffset = Virtual;
  } else
    BaseOffset = NonVirtualOffset;
  
  // Apply the base offset.
  llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
  ThisPtr = CGF.Builder.CreateBitCast(ThisPtr, Int8PtrTy);
  ThisPtr = CGF.Builder.CreateGEP(ThisPtr, BaseOffset, "add.ptr");

  return ThisPtr;
}

llvm::Value *
CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value, 
                                       const CXXRecordDecl *Derived,
                                       CastExpr::path_const_iterator PathBegin,
                                       CastExpr::path_const_iterator PathEnd,
                                       bool NullCheckValue) {
  assert(PathBegin != PathEnd && "Base path should not be empty!");

  CastExpr::path_const_iterator Start = PathBegin;
  const CXXRecordDecl *VBase = 0;
  
  // Get the virtual base.
  if ((*Start)->isVirtual()) {
    VBase = 
      cast<CXXRecordDecl>((*Start)->getType()->getAs<RecordType>()->getDecl());
    ++Start;
  }
  
  CharUnits NonVirtualOffset = 
    ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived,
                                     Start, PathEnd);

  // Get the base pointer type.
  llvm::Type *BasePtrTy = 
    ConvertType((PathEnd[-1])->getType())->getPointerTo();
  
  if (NonVirtualOffset.isZero() && !VBase) {
    // Just cast back.
    return Builder.CreateBitCast(Value, BasePtrTy);
  }    
  
  llvm::BasicBlock *CastNull = 0;
  llvm::BasicBlock *CastNotNull = 0;
  llvm::BasicBlock *CastEnd = 0;
  
  if (NullCheckValue) {
    CastNull = createBasicBlock("cast.null");
    CastNotNull = createBasicBlock("cast.notnull");
    CastEnd = createBasicBlock("cast.end");
    
    llvm::Value *IsNull = Builder.CreateIsNull(Value);
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
    EmitBlock(CastNotNull);
  }

  llvm::Value *VirtualOffset = 0;

  if (VBase) {
    if (Derived->hasAttr<FinalAttr>()) {
      VirtualOffset = 0;

      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);

      CharUnits VBaseOffset = Layout.getVBaseClassOffset(VBase);
      NonVirtualOffset += VBaseOffset;
    } else
      VirtualOffset = GetVirtualBaseClassOffset(Value, Derived, VBase);
  }

  // Apply the offsets.
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, 
                                          NonVirtualOffset,
                                          VirtualOffset);
  
  // Cast back.
  Value = Builder.CreateBitCast(Value, BasePtrTy);
 
  if (NullCheckValue) {
    Builder.CreateBr(CastEnd);
    EmitBlock(CastNull);
    Builder.CreateBr(CastEnd);
    EmitBlock(CastEnd);
    
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
    PHI->addIncoming(Value, CastNotNull);
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), 
                     CastNull);
    Value = PHI;
  }
  
  return Value;
}

llvm::Value *
CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
                                          const CXXRecordDecl *Derived,
                                        CastExpr::path_const_iterator PathBegin,
                                          CastExpr::path_const_iterator PathEnd,
                                          bool NullCheckValue) {
  assert(PathBegin != PathEnd && "Base path should not be empty!");

  QualType DerivedTy =
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo();
  
  llvm::Value *NonVirtualOffset =
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
  
  if (!NonVirtualOffset) {
    // No offset, we can just cast back.
    return Builder.CreateBitCast(Value, DerivedPtrTy);
  }
  
  llvm::BasicBlock *CastNull = 0;
  llvm::BasicBlock *CastNotNull = 0;
  llvm::BasicBlock *CastEnd = 0;
  
  if (NullCheckValue) {
    CastNull = createBasicBlock("cast.null");
    CastNotNull = createBasicBlock("cast.notnull");
    CastEnd = createBasicBlock("cast.end");
    
    llvm::Value *IsNull = Builder.CreateIsNull(Value);
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
    EmitBlock(CastNotNull);
  }
  
  // Apply the offset.
  Value = Builder.CreatePtrToInt(Value, NonVirtualOffset->getType());
  Value = Builder.CreateSub(Value, NonVirtualOffset);
  Value = Builder.CreateIntToPtr(Value, DerivedPtrTy);

  // Just cast.
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);

  if (NullCheckValue) {
    Builder.CreateBr(CastEnd);
    EmitBlock(CastNull);
    Builder.CreateBr(CastEnd);
    EmitBlock(CastEnd);
    
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
    PHI->addIncoming(Value, CastNotNull);
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), 
                     CastNull);
    Value = PHI;
  }
  
  return Value;
}
                             
/// GetVTTParameter - Return the VTT parameter that should be passed to a
/// base constructor/destructor with virtual bases.
static llvm::Value *GetVTTParameter(CodeGenFunction &CGF, GlobalDecl GD,
                                    bool ForVirtualBase) {
  if (!CodeGenVTables::needsVTTParameter(GD)) {
    // This constructor/destructor does not need a VTT parameter.
    return 0;
  }
  
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CGF.CurFuncDecl)->getParent();
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();

  llvm::Value *VTT;

  uint64_t SubVTTIndex;

  // If the record matches the base, this is the complete ctor/dtor
  // variant calling the base variant in a class with virtual bases.
  if (RD == Base) {
    assert(!CodeGenVTables::needsVTTParameter(CGF.CurGD) &&
           "doing no-op VTT offset in base dtor/ctor?");
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
    SubVTTIndex = 0;
  } else {
    const ASTRecordLayout &Layout = 
      CGF.getContext().getASTRecordLayout(RD);
    CharUnits BaseOffset = ForVirtualBase ? 
      Layout.getVBaseClassOffset(Base) : 
      Layout.getBaseClassOffset(Base);

    SubVTTIndex = 
      CGF.CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
  }
  
  if (CodeGenVTables::needsVTTParameter(CGF.CurGD)) {
    // A VTT parameter was passed to the constructor, use it.
    VTT = CGF.LoadCXXVTT();
    VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
  } else {
    // We're the complete constructor, so get the VTT by name.
    VTT = CGF.CGM.getVTables().GetAddrOfVTT(RD);
    VTT = CGF.Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
  }

  return VTT;
}

namespace {
  /// Call the destructor for a direct base class.
  struct CallBaseDtor : EHScopeStack::Cleanup {
    const CXXRecordDecl *BaseClass;
    bool BaseIsVirtual;
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}

    void Emit(CodeGenFunction &CGF, Flags flags) {
      const CXXRecordDecl *DerivedClass =
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();

      const CXXDestructorDecl *D = BaseClass->getDestructor();
      llvm::Value *Addr = 
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(),
                                                  DerivedClass, BaseClass,
                                                  BaseIsVirtual);
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual, Addr);
    }
  };

  /// A visitor which checks whether an initializer uses 'this' in a
  /// way which requires the vtable to be properly set.
  struct DynamicThisUseChecker : EvaluatedExprVisitor<DynamicThisUseChecker> {
    typedef EvaluatedExprVisitor<DynamicThisUseChecker> super;

    bool UsesThis;

    DynamicThisUseChecker(ASTContext &C) : super(C), UsesThis(false) {}

    // Black-list all explicit and implicit references to 'this'.
    //
    // Do we need to worry about external references to 'this' derived
    // from arbitrary code?  If so, then anything which runs arbitrary
    // external code might potentially access the vtable.
    void VisitCXXThisExpr(CXXThisExpr *E) { UsesThis = true; }
  };
}

static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
  DynamicThisUseChecker Checker(C);
  Checker.Visit(const_cast<Expr*>(Init));
  return Checker.UsesThis;
}

static void EmitBaseInitializer(CodeGenFunction &CGF, 
                                const CXXRecordDecl *ClassDecl,
                                CXXCtorInitializer *BaseInit,
                                CXXCtorType CtorType) {
  assert(BaseInit->isBaseInitializer() &&
         "Must have base initializer!");

  llvm::Value *ThisPtr = CGF.LoadCXXThis();
  
  const Type *BaseType = BaseInit->getBaseClass();
  CXXRecordDecl *BaseClassDecl =
    cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());

  bool isBaseVirtual = BaseInit->isBaseVirtual();

  // The base constructor doesn't construct virtual bases.
  if (CtorType == Ctor_Base && isBaseVirtual)
    return;

  // If the initializer for the base (other than the constructor
  // itself) accesses 'this' in any way, we need to initialize the
  // vtables.
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
    CGF.InitializeVTablePointers(ClassDecl);

  // We can pretend to be a complete class because it only matters for
  // virtual bases, and we only do virtual bases for complete ctors.
  llvm::Value *V = 
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
                                              BaseClassDecl,
                                              isBaseVirtual);
  CharUnits Alignment = CGF.getContext().getTypeAlignInChars(BaseType);
  AggValueSlot AggSlot =
    AggValueSlot::forAddr(V, Alignment, Qualifiers(),
                          AggValueSlot::IsDestructed,
                          AggValueSlot::DoesNotNeedGCBarriers,
                          AggValueSlot::IsNotAliased);

  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
  
  if (CGF.CGM.getLangOptions().Exceptions && 
      !BaseClassDecl->hasTrivialDestructor())
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
                                          isBaseVirtual);
}

static void EmitAggMemberInitializer(CodeGenFunction &CGF,
                                     LValue LHS,
                                     llvm::Value *ArrayIndexVar,
                                     CXXCtorInitializer *MemberInit,
                                     QualType T,
                                     unsigned Index) {
  if (Index == MemberInit->getNumArrayIndices()) {
    CodeGenFunction::RunCleanupsScope Cleanups(CGF);

    LValue LV = LHS;
    if (ArrayIndexVar) {
      // If we have an array index variable, load it and use it as an offset.
      // Then, increment the value.
      llvm::Value *Dest = LHS.getAddress();
      llvm::Value *ArrayIndex = CGF.Builder.CreateLoad(ArrayIndexVar);
      Dest = CGF.Builder.CreateInBoundsGEP(Dest, ArrayIndex, "destaddress");
      llvm::Value *Next = llvm::ConstantInt::get(ArrayIndex->getType(), 1);
      Next = CGF.Builder.CreateAdd(ArrayIndex, Next, "inc");
      CGF.Builder.CreateStore(Next, ArrayIndexVar);    

      // Update the LValue.
      LV.setAddress(Dest);
      CharUnits Align = CGF.getContext().getTypeAlignInChars(T);
      LV.setAlignment(std::min(Align, LV.getAlignment()));
    }

    if (!CGF.hasAggregateLLVMType(T)) {
      CGF.EmitScalarInit(MemberInit->getInit(), /*decl*/ 0, LV, false);
    } else if (T->isAnyComplexType()) {
      CGF.EmitComplexExprIntoAddr(MemberInit->getInit(), LV.getAddress(),
                                  LV.isVolatileQualified());
    } else {
      AggValueSlot Slot =
        AggValueSlot::forLValue(LV,
                                AggValueSlot::IsDestructed,
                                AggValueSlot::DoesNotNeedGCBarriers,
                                AggValueSlot::IsNotAliased);
      
      CGF.EmitAggExpr(MemberInit->getInit(), Slot);
    }
    
    return;
  }
  
  const ConstantArrayType *Array = CGF.getContext().getAsConstantArrayType(T);
  assert(Array && "Array initialization without the array type?");
  llvm::Value *IndexVar
    = CGF.GetAddrOfLocalVar(MemberInit->getArrayIndex(Index));
  assert(IndexVar && "Array index variable not loaded");
  
  // Initialize this index variable to zero.
  llvm::Value* Zero
    = llvm::Constant::getNullValue(
                              CGF.ConvertType(CGF.getContext().getSizeType()));
  CGF.Builder.CreateStore(Zero, IndexVar);
                                   
  // Start the loop with a block that tests the condition.
  llvm::BasicBlock *CondBlock = CGF.createBasicBlock("for.cond");
  llvm::BasicBlock *AfterFor = CGF.createBasicBlock("for.end");
  
  CGF.EmitBlock(CondBlock);

  llvm::BasicBlock *ForBody = CGF.createBasicBlock("for.body");
  // Generate: if (loop-index < number-of-elements) fall to the loop body,
  // otherwise, go to the block after the for-loop.
  uint64_t NumElements = Array->getSize().getZExtValue();
  llvm::Value *Counter = CGF.Builder.CreateLoad(IndexVar);
  llvm::Value *NumElementsPtr =
    llvm::ConstantInt::get(Counter->getType(), NumElements);
  llvm::Value *IsLess = CGF.Builder.CreateICmpULT(Counter, NumElementsPtr,
                                                  "isless");
                                   
  // If the condition is true, execute the body.
  CGF.Builder.CreateCondBr(IsLess, ForBody, AfterFor);

  CGF.EmitBlock(ForBody);
  llvm::BasicBlock *ContinueBlock = CGF.createBasicBlock("for.inc");
  
  {
    CodeGenFunction::RunCleanupsScope Cleanups(CGF);
    
    // Inside the loop body recurse to emit the inner loop or, eventually, the
    // constructor call.
    EmitAggMemberInitializer(CGF, LHS, ArrayIndexVar, MemberInit, 
                             Array->getElementType(), Index + 1);
  }
  
  CGF.EmitBlock(ContinueBlock);

  // Emit the increment of the loop counter.
  llvm::Value *NextVal = llvm::ConstantInt::get(Counter->getType(), 1);
  Counter = CGF.Builder.CreateLoad(IndexVar);
  NextVal = CGF.Builder.CreateAdd(Counter, NextVal, "inc");
  CGF.Builder.CreateStore(NextVal, IndexVar);

  // Finally, branch back up to the condition for the next iteration.
  CGF.EmitBranch(CondBlock);

  // Emit the fall-through block.
  CGF.EmitBlock(AfterFor, true);
}

namespace {
  struct CallMemberDtor : EHScopeStack::Cleanup {
    FieldDecl *Field;
    CXXDestructorDecl *Dtor;

    CallMemberDtor(FieldDecl *Field, CXXDestructorDecl *Dtor)
      : Field(Field), Dtor(Dtor) {}

    void Emit(CodeGenFunction &CGF, Flags flags) {
      // FIXME: Is this OK for C++0x delegating constructors?
      llvm::Value *ThisPtr = CGF.LoadCXXThis();
      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 0);

      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
                                LHS.getAddress());
    }
  };
}

static bool hasTrivialCopyOrMoveConstructor(const CXXRecordDecl *Record,
                                            bool Moving) {
  return Moving ? Record->hasTrivialMoveConstructor() :
                  Record->hasTrivialCopyConstructor();
}
  
static void EmitMemberInitializer(CodeGenFunction &CGF,
                                  const CXXRecordDecl *ClassDecl,
                                  CXXCtorInitializer *MemberInit,
                                  const CXXConstructorDecl *Constructor,
                                  FunctionArgList &Args) {
  assert(MemberInit->isAnyMemberInitializer() &&
         "Must have member initializer!");
  assert(MemberInit->getInit() && "Must have initializer!");
  
  // non-static data member initializers.
  FieldDecl *Field = MemberInit->getAnyMember();
  QualType FieldType = CGF.getContext().getCanonicalType(Field->getType());

  llvm::Value *ThisPtr = CGF.LoadCXXThis();
  LValue LHS;
  
  // If we are initializing an anonymous union field, drill down to the field.
  if (MemberInit->isIndirectMemberInitializer()) {
    LHS = CGF.EmitLValueForAnonRecordField(ThisPtr,
                                           MemberInit->getIndirectMember(), 0);
    FieldType = MemberInit->getIndirectMember()->getAnonField()->getType();
  } else {
    LHS = CGF.EmitLValueForFieldInitialization(ThisPtr, Field, 0);
  }

  if (!CGF.hasAggregateLLVMType(Field->getType())) {
    if (LHS.isSimple()) {
      CGF.EmitExprAsInit(MemberInit->getInit(), Field, LHS, false);
    } else {
      RValue RHS = RValue::get(CGF.EmitScalarExpr(MemberInit->getInit()));
      CGF.EmitStoreThroughLValue(RHS, LHS);
    }
  } else if (MemberInit->getInit()->getType()->isAnyComplexType()) {
    CGF.EmitComplexExprIntoAddr(MemberInit->getInit(), LHS.getAddress(),
                                LHS.isVolatileQualified());
  } else {
    llvm::Value *ArrayIndexVar = 0;
    const ConstantArrayType *Array
      = CGF.getContext().getAsConstantArrayType(FieldType);
    if (Array && Constructor->isImplicitlyDefined() &&
        Constructor->isCopyOrMoveConstructor()) {
      llvm::Type *SizeTy
        = CGF.ConvertType(CGF.getContext().getSizeType());
      
      // The LHS is a pointer to the first object we'll be constructing, as
      // a flat array.
      QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
      llvm::Type *BasePtr = CGF.ConvertType(BaseElementTy);
      BasePtr = llvm::PointerType::getUnqual(BasePtr);
      llvm::Value *BaseAddrPtr = CGF.Builder.CreateBitCast(LHS.getAddress(), 
                                                           BasePtr);
      LHS = CGF.MakeAddrLValue(BaseAddrPtr, BaseElementTy);
      
      // Create an array index that will be used to walk over all of the
      // objects we're constructing.
      ArrayIndexVar = CGF.CreateTempAlloca(SizeTy, "object.index");
      llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy);
      CGF.Builder.CreateStore(Zero, ArrayIndexVar);
      
      // If we are copying an array of PODs or classes with trivial copy 
      // constructors, perform a single aggregate copy.
      const CXXRecordDecl *Record = BaseElementTy->getAsCXXRecordDecl();
      if (BaseElementTy.isPODType(CGF.getContext()) ||
          (Record && hasTrivialCopyOrMoveConstructor(Record,
                         Constructor->isMoveConstructor()))) {
        // Find the source pointer. We knows it's the last argument because
        // we know we're in a copy constructor.
        unsigned SrcArgIndex = Args.size() - 1;
        llvm::Value *SrcPtr
          = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
        LValue Src = CGF.EmitLValueForFieldInitialization(SrcPtr, Field, 0);
        
        // Copy the aggregate.
        CGF.EmitAggregateCopy(LHS.getAddress(), Src.getAddress(), FieldType,
                              LHS.isVolatileQualified());
        return;
      }
      
      // Emit the block variables for the array indices, if any.
      for (unsigned I = 0, N = MemberInit->getNumArrayIndices(); I != N; ++I)
        CGF.EmitAutoVarDecl(*MemberInit->getArrayIndex(I));
    }
    
    EmitAggMemberInitializer(CGF, LHS, ArrayIndexVar, MemberInit, FieldType, 0);
    
    if (!CGF.CGM.getLangOptions().Exceptions)
      return;

    // FIXME: If we have an array of classes w/ non-trivial destructors, 
    // we need to destroy in reverse order of construction along the exception
    // path.
    const RecordType *RT = FieldType->getAs<RecordType>();
    if (!RT)
      return;
    
    CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
    if (!RD->hasTrivialDestructor())
      CGF.EHStack.pushCleanup<CallMemberDtor>(EHCleanup, Field,
                                              RD->getDestructor());
  }
}

/// Checks whether the given constructor is a valid subject for the
/// complete-to-base constructor delegation optimization, i.e.
/// emitting the complete constructor as a simple call to the base
/// constructor.
static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor) {

  // Currently we disable the optimization for classes with virtual
  // bases because (1) the addresses of parameter variables need to be
  // consistent across all initializers but (2) the delegate function
  // call necessarily creates a second copy of the parameter variable.
  //
  // The limiting example (purely theoretical AFAIK):
  //   struct A { A(int &c) { c++; } };
  //   struct B : virtual A {
  //     B(int count) : A(count) { printf("%d\n", count); }
  //   };
  // ...although even this example could in principle be emitted as a
  // delegation since the address of the parameter doesn't escape.
  if (Ctor->getParent()->getNumVBases()) {
    // TODO: white-list trivial vbase initializers.  This case wouldn't
    // be subject to the restrictions below.

    // TODO: white-list cases where:
    //  - there are no non-reference parameters to the constructor
    //  - the initializers don't access any non-reference parameters
    //  - the initializers don't take the address of non-reference
    //    parameters
    //  - etc.
    // If we ever add any of the above cases, remember that:
    //  - function-try-blocks will always blacklist this optimization
    //  - we need to perform the constructor prologue and cleanup in
    //    EmitConstructorBody.

    return false;
  }

  // We also disable the optimization for variadic functions because
  // it's impossible to "re-pass" varargs.
  if (Ctor->getType()->getAs<FunctionProtoType>()->isVariadic())
    return false;

  // FIXME: Decide if we can do a delegation of a delegating constructor.
  if (Ctor->isDelegatingConstructor())
    return false;

  return true;
}

/// EmitConstructorBody - Emits the body of the current constructor.
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
  CXXCtorType CtorType = CurGD.getCtorType();

  // Before we go any further, try the complete->base constructor
  // delegation optimization.
  if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor)) {
    if (CGDebugInfo *DI = getDebugInfo()) 
      DI->EmitLocation(Builder, Ctor->getLocEnd());
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args);
    return;
  }

  Stmt *Body = Ctor->getBody();

  // Enter the function-try-block before the constructor prologue if
  // applicable.
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
  if (IsTryBody)
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);

  EHScopeStack::stable_iterator CleanupDepth = EHStack.stable_begin();

  // Emit the constructor prologue, i.e. the base and member
  // initializers.
  EmitCtorPrologue(Ctor, CtorType, Args);

  // Emit the body of the statement.
  if (IsTryBody)
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
  else if (Body)
    EmitStmt(Body);

  // Emit any cleanup blocks associated with the member or base
  // initializers, which includes (along the exceptional path) the
  // destructors for those members and bases that were fully
  // constructed.
  PopCleanupBlocks(CleanupDepth);

  if (IsTryBody)
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
}

/// EmitCtorPrologue - This routine generates necessary code to initialize
/// base classes and non-static data members belonging to this constructor.
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
                                       CXXCtorType CtorType,
                                       FunctionArgList &Args) {
  if (CD->isDelegatingConstructor())
    return EmitDelegatingCXXConstructorCall(CD, Args);

  const CXXRecordDecl *ClassDecl = CD->getParent();

  SmallVector<CXXCtorInitializer *, 8> MemberInitializers;
  
  for (CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
       E = CD->init_end();
       B != E; ++B) {
    CXXCtorInitializer *Member = (*B);
    
    if (Member->isBaseInitializer()) {
      EmitBaseInitializer(*this, ClassDecl, Member, CtorType);
    } else {
      assert(Member->isAnyMemberInitializer() &&
            "Delegating initializer on non-delegating constructor");
      MemberInitializers.push_back(Member);
    }
  }

  InitializeVTablePointers(ClassDecl);

  for (unsigned I = 0, E = MemberInitializers.size(); I != E; ++I)
    EmitMemberInitializer(*this, ClassDecl, MemberInitializers[I], CD, Args);
}

static bool
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);

static bool
HasTrivialDestructorBody(ASTContext &Context, 
                         const CXXRecordDecl *BaseClassDecl,
                         const CXXRecordDecl *MostDerivedClassDecl)
{
  // If the destructor is trivial we don't have to check anything else.
  if (BaseClassDecl->hasTrivialDestructor())
    return true;

  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
    return false;

  // Check fields.
  for (CXXRecordDecl::field_iterator I = BaseClassDecl->field_begin(),
       E = BaseClassDecl->field_end(); I != E; ++I) {
    const FieldDecl *Field = *I;
    
    if (!FieldHasTrivialDestructorBody(Context, Field))
      return false;
  }

  // Check non-virtual bases.
  for (CXXRecordDecl::base_class_const_iterator I = 
       BaseClassDecl->bases_begin(), E = BaseClassDecl->bases_end();
       I != E; ++I) {
    if (I->isVirtual())
      continue;

    const CXXRecordDecl *NonVirtualBase =
      cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
                                  MostDerivedClassDecl))
      return false;
  }

  if (BaseClassDecl == MostDerivedClassDecl) {
    // Check virtual bases.
    for (CXXRecordDecl::base_class_const_iterator I = 
         BaseClassDecl->vbases_begin(), E = BaseClassDecl->vbases_end();
         I != E; ++I) {
      const CXXRecordDecl *VirtualBase =
        cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
      if (!HasTrivialDestructorBody(Context, VirtualBase,
                                    MostDerivedClassDecl))
        return false;      
    }
  }

  return true;
}

static bool
FieldHasTrivialDestructorBody(ASTContext &Context,
                              const FieldDecl *Field)
{
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());

  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
  if (!RT)
    return true;
  
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
}

/// CanSkipVTablePointerInitialization - Check whether we need to initialize
/// any vtable pointers before calling this destructor.
static bool CanSkipVTablePointerInitialization(ASTContext &Context,
                                               const CXXDestructorDecl *Dtor) {
  if (!Dtor->hasTrivialBody())
    return false;

  // Check the fields.
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
  for (CXXRecordDecl::field_iterator I = ClassDecl->field_begin(),
       E = ClassDecl->field_end(); I != E; ++I) {
    const FieldDecl *Field = *I;

    if (!FieldHasTrivialDestructorBody(Context, Field))
      return false;
  }

  return true;
}

/// EmitDestructorBody - Emits the body of the current destructor.
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
  CXXDtorType DtorType = CurGD.getDtorType();

  // The call to operator delete in a deleting destructor happens
  // outside of the function-try-block, which means it's always
  // possible to delegate the destructor body to the complete
  // destructor.  Do so.
  if (DtorType == Dtor_Deleting) {
    EnterDtorCleanups(Dtor, Dtor_Deleting);
    EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
                          LoadCXXThis());
    PopCleanupBlock();
    return;
  }

  Stmt *Body = Dtor->getBody();

  // If the body is a function-try-block, enter the try before
  // anything else.
  bool isTryBody = (Body && isa<CXXTryStmt>(Body));
  if (isTryBody)
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);

  // Enter the epilogue cleanups.
  RunCleanupsScope DtorEpilogue(*this);
  
  // If this is the complete variant, just invoke the base variant;
  // the epilogue will destruct the virtual bases.  But we can't do
  // this optimization if the body is a function-try-block, because
  // we'd introduce *two* handler blocks.
  switch (DtorType) {
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");

  case Dtor_Complete:
    // Enter the cleanup scopes for virtual bases.
    EnterDtorCleanups(Dtor, Dtor_Complete);

    if (!isTryBody) {
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
                            LoadCXXThis());
      break;
    }
    // Fallthrough: act like we're in the base variant.
      
  case Dtor_Base:
    // Enter the cleanup scopes for fields and non-virtual bases.
    EnterDtorCleanups(Dtor, Dtor_Base);

    // Initialize the vtable pointers before entering the body.
    if (!CanSkipVTablePointerInitialization(getContext(), Dtor))
        InitializeVTablePointers(Dtor->getParent());

    if (isTryBody)
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
    else if (Body)
      EmitStmt(Body);
    else {
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
      // nothing to do besides what's in the epilogue
    }
    // -fapple-kext must inline any call to this dtor into
    // the caller's body.
    if (getContext().getLangOptions().AppleKext)
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
    break;
  }

  // Jump out through the epilogue cleanups.
  DtorEpilogue.ForceCleanup();

  // Exit the try if applicable.
  if (isTryBody)
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
}

namespace {
  /// Call the operator delete associated with the current destructor.
  struct CallDtorDelete : EHScopeStack::Cleanup {
    CallDtorDelete() {}

    void Emit(CodeGenFunction &CGF, Flags flags) {
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(), CGF.LoadCXXThis(),
                         CGF.getContext().getTagDeclType(ClassDecl));
    }
  };

  class DestroyField  : public EHScopeStack::Cleanup {
    const FieldDecl *field;
    CodeGenFunction::Destroyer &destroyer;
    bool useEHCleanupForArray;

  public:
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
                 bool useEHCleanupForArray)
      : field(field), destroyer(*destroyer),
        useEHCleanupForArray(useEHCleanupForArray) {}

    void Emit(CodeGenFunction &CGF, Flags flags) {
      // Find the address of the field.
      llvm::Value *thisValue = CGF.LoadCXXThis();
      LValue LV = CGF.EmitLValueForField(thisValue, field, /*CVRQualifiers=*/0);
      assert(LV.isSimple());
      
      CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer,
                      flags.isForNormalCleanup() && useEHCleanupForArray);
    }
  };
}

/// EmitDtorEpilogue - Emit all code that comes at the end of class's
/// destructor. This is to call destructors on members and base classes
/// in reverse order of their construction.
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
                                        CXXDtorType DtorType) {
  assert(!DD->isTrivial() &&
         "Should not emit dtor epilogue for trivial dtor!");

  // The deleting-destructor phase just needs to call the appropriate
  // operator delete that Sema picked up.
  if (DtorType == Dtor_Deleting) {
    assert(DD->getOperatorDelete() && 
           "operator delete missing - EmitDtorEpilogue");
    EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
    return;
  }

  const CXXRecordDecl *ClassDecl = DD->getParent();

  // Unions have no bases and do not call field destructors.
  if (ClassDecl->isUnion())
    return;

  // The complete-destructor phase just destructs all the virtual bases.
  if (DtorType == Dtor_Complete) {

    // We push them in the forward order so that they'll be popped in
    // the reverse order.
    for (CXXRecordDecl::base_class_const_iterator I = 
           ClassDecl->vbases_begin(), E = ClassDecl->vbases_end();
              I != E; ++I) {
      const CXXBaseSpecifier &Base = *I;