[CIR][Upstream] Local initialization for ArrayType (#132974)

This change adds local initialization for ArrayType

Issue #130197

13 files changed, 460 insertions, 3 deletions

diff --git a/clang/lib/CIR/CodeGen/Address.h b/clang/lib/CIR/CodeGen/Address.h
index fba1ffd..2cc8ada 100644
--- a/clang/lib/CIR/CodeGen/Address.h
+++ b/clang/lib/CIR/CodeGen/Address.h
@@ -70,6 +70,14 @@ public:
     return pointerAndKnownNonNull.getPointer();
   }
 
+  mlir::Type getType() const {
+    assert(mlir::cast<cir::PointerType>(
+               pointerAndKnownNonNull.getPointer().getType())
+               .getPointee() == elementType);
+
+    return mlir::cast<cir::PointerType>(getPointer().getType());
+  }
+
   mlir::Type getElementType() const {
     assert(isValid());
     assert(mlir::cast<cir::PointerType>(
@@ -77,6 +85,8 @@ public:
                .getPointee() == elementType);
     return elementType;
   }
+
+  clang::CharUnits getAlignment() const { return alignment; }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index f2153c2..5b832b4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -251,7 +251,7 @@ void CIRGenFunction::emitExprAsInit(const Expr *init, const ValueDecl *d,
     return;
   }
   case cir::TEK_Aggregate:
-    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: aggregate type");
+    emitAggExpr(init, AggValueSlot::forLValue(lvalue));
     return;
   }
   llvm_unreachable("bad evaluation kind");
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
new file mode 100644
index 0000000..36da63d
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
@@ -0,0 +1,277 @@
+//===--- CIRGenExprAgg.cpp - Emit CIR Code from Aggregate Expressions -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code to emit Aggregate Expr nodes as CIR code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenBuilder.h"
+#include "CIRGenFunction.h"
+#include "CIRGenValue.h"
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+
+#include "clang/AST/Expr.h"
+#include "clang/AST/StmtVisitor.h"
+#include <cstdint>
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
+
+  CIRGenFunction &cgf;
+  AggValueSlot dest;
+
+  AggValueSlot ensureSlot(mlir::Location loc, QualType t) {
+    if (!dest.isIgnored())
+      return dest;
+
+    cgf.cgm.errorNYI(loc, "Slot for ignored address");
+    return dest;
+  }
+
+public:
+  AggExprEmitter(CIRGenFunction &cgf, AggValueSlot dest)
+      : cgf(cgf), dest(dest) {}
+
+  void emitArrayInit(Address destPtr, cir::ArrayType arrayTy, QualType arrayQTy,
+                     Expr *exprToVisit, ArrayRef<Expr *> args,
+                     Expr *arrayFiller);
+
+  void emitInitializationToLValue(Expr *e, LValue lv);
+
+  void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
+
+  void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
+
+  void VisitInitListExpr(InitListExpr *e);
+
+  void visitCXXParenListOrInitListExpr(Expr *e, ArrayRef<Expr *> args,
+                                       FieldDecl *initializedFieldInUnion,
+                                       Expr *arrayFiller);
+};
+
+} // namespace
+
+static bool isTrivialFiller(Expr *e) {
+  if (!e)
+    return true;
+
+  if (isa<ImplicitValueInitExpr>(e))
+    return true;
+
+  if (auto *ile = dyn_cast<InitListExpr>(e)) {
+    if (ile->getNumInits())
+      return false;
+    return isTrivialFiller(ile->getArrayFiller());
+  }
+
+  if (const auto *cons = dyn_cast_or_null<CXXConstructExpr>(e))
+    return cons->getConstructor()->isDefaultConstructor() &&
+           cons->getConstructor()->isTrivial();
+
+  return false;
+}
+
+void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
+                                   QualType arrayQTy, Expr *e,
+                                   ArrayRef<Expr *> args, Expr *arrayFiller) {
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
+
+  const uint64_t numInitElements = args.size();
+
+  const QualType elementType =
+      cgf.getContext().getAsArrayType(arrayQTy)->getElementType();
+
+  if (elementType.isDestructedType()) {
+    cgf.cgm.errorNYI(loc, "dtorKind NYI");
+    return;
+  }
+
+  const QualType elementPtrType = cgf.getContext().getPointerType(elementType);
+
+  const mlir::Type cirElementType = cgf.convertType(elementType);
+  const cir::PointerType cirElementPtrType =
+      builder.getPointerTo(cirElementType);
+
+  auto begin = builder.create<cir::CastOp>(loc, cirElementPtrType,
+                                           cir::CastKind::array_to_ptrdecay,
+                                           destPtr.getPointer());
+
+  const CharUnits elementSize =
+      cgf.getContext().getTypeSizeInChars(elementType);
+  const CharUnits elementAlign =
+      destPtr.getAlignment().alignmentOfArrayElement(elementSize);
+
+  // The 'current element to initialize'.  The invariants on this
+  // variable are complicated.  Essentially, after each iteration of
+  // the loop, it points to the last initialized element, except
+  // that it points to the beginning of the array before any
+  // elements have been initialized.
+  mlir::Value element = begin;
+
+  // Don't build the 'one' before the cycle to avoid
+  // emmiting the redundant `cir.const 1` instrs.
+  mlir::Value one;
+
+  // Emit the explicit initializers.
+  for (uint64_t i = 0; i != numInitElements; ++i) {
+    // Advance to the next element.
+    if (i > 0) {
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, i);
+      element = builder.createPtrStride(loc, begin, one);
+    }
+
+    const Address address = Address(element, cirElementType, elementAlign);
+    const LValue elementLV = LValue::makeAddr(address, elementType);
+    emitInitializationToLValue(args[i], elementLV);
+  }
+
+  const uint64_t numArrayElements = arrayTy.getSize();
+
+  // Check whether there's a non-trivial array-fill expression.
+  const bool hasTrivialFiller = isTrivialFiller(arrayFiller);
+
+  // Any remaining elements need to be zero-initialized, possibly
+  // using the filler expression.  We can skip this if the we're
+  // emitting to zeroed memory.
+  if (numInitElements != numArrayElements &&
+      !(dest.isZeroed() && hasTrivialFiller &&
+        cgf.getTypes().isZeroInitializable(elementType))) {
+    // Advance to the start of the rest of the array.
+    if (numInitElements) {
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
+      element = builder.create<cir::PtrStrideOp>(loc, cirElementPtrType,
+                                                 element, one);
+    }
+
+    // Allocate the temporary variable
+    // to store the pointer to first unitialized element
+    const Address tmpAddr = cgf.createTempAlloca(
+        cirElementPtrType, cgf.getPointerAlign(), loc, "arrayinit.temp",
+        /*insertIntoFnEntryBlock=*/false);
+    LValue tmpLV = LValue::makeAddr(tmpAddr, elementPtrType);
+    cgf.emitStoreThroughLValue(RValue::get(element), tmpLV);
+
+    // TODO(CIR): Replace this part later with cir::DoWhileOp
+    for (unsigned i = numInitElements; i != numArrayElements; ++i) {
+      cir::LoadOp currentElement =
+          builder.createLoad(loc, tmpAddr.getPointer());
+
+      // Emit the actual filler expression.
+      const LValue elementLV = LValue::makeAddr(
+          Address(currentElement, cirElementType, elementAlign), elementType);
+
+      if (arrayFiller)
+        emitInitializationToLValue(arrayFiller, elementLV);
+      else
+        emitNullInitializationToLValue(loc, elementLV);
+
+      // Advance pointer and store them to temporary variable
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
+      cir::PtrStrideOp nextElement =
+          builder.createPtrStride(loc, currentElement, one);
+      cgf.emitStoreThroughLValue(RValue::get(nextElement), tmpLV);
+    }
+  }
+}
+
+void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
+  const QualType type = lv.getType();
+
+  if (isa<ImplicitValueInitExpr, CXXScalarValueInitExpr>(e)) {
+    const mlir::Location loc = e->getSourceRange().isValid()
+                                   ? cgf.getLoc(e->getSourceRange())
+                                   : *cgf.currSrcLoc;
+    return emitNullInitializationToLValue(loc, lv);
+  }
+
+  if (isa<NoInitExpr>(e))
+    return;
+
+  if (type->isReferenceType())
+    cgf.cgm.errorNYI("emitInitializationToLValue ReferenceType");
+
+  switch (cgf.getEvaluationKind(type)) {
+  case cir::TEK_Complex:
+    cgf.cgm.errorNYI("emitInitializationToLValue TEK_Complex");
+    break;
+  case cir::TEK_Aggregate:
+    cgf.emitAggExpr(e, AggValueSlot::forLValue(lv));
+    return;
+  case cir::TEK_Scalar:
+    if (lv.isSimple())
+      cgf.emitScalarInit(e, cgf.getLoc(e->getSourceRange()), lv);
+    else
+      cgf.emitStoreThroughLValue(RValue::get(cgf.emitScalarExpr(e)), lv);
+    return;
+  }
+}
+
+void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
+                                                    LValue lv) {
+  const QualType type = lv.getType();
+
+  // If the destination slot is already zeroed out before the aggregate is
+  // copied into it, we don't have to emit any zeros here.
+  if (dest.isZeroed() && cgf.getTypes().isZeroInitializable(type))
+    return;
+
+  if (cgf.hasScalarEvaluationKind(type)) {
+    // For non-aggregates, we can store the appropriate null constant.
+    mlir::Value null = cgf.cgm.emitNullConstant(type, loc);
+    if (lv.isSimple()) {
+      cgf.emitStoreOfScalar(null, lv, /* isInitialization */ true);
+      return;
+    }
+
+    cgf.cgm.errorNYI("emitStoreThroughBitfieldLValue");
+    return;
+  }
+
+  // There's a potential optimization opportunity in combining
+  // memsets; that would be easy for arrays, but relatively
+  // difficult for structures with the current code.
+  cgf.emitNullInitialization(loc, lv.getAddress(), lv.getType());
+}
+
+void AggExprEmitter::VisitInitListExpr(InitListExpr *e) {
+  if (e->hadArrayRangeDesignator())
+    llvm_unreachable("GNU array range designator extension");
+
+  if (e->isTransparent())
+    return Visit(e->getInit(0));
+
+  visitCXXParenListOrInitListExpr(
+      e, e->inits(), e->getInitializedFieldInUnion(), e->getArrayFiller());
+}
+
+void AggExprEmitter::visitCXXParenListOrInitListExpr(
+    Expr *e, ArrayRef<Expr *> args, FieldDecl *initializedFieldInUnion,
+    Expr *arrayFiller) {
+
+  const AggValueSlot dest =
+      ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
+
+  if (e->getType()->isConstantArrayType()) {
+    cir::ArrayType arrayTy =
+        cast<cir::ArrayType>(dest.getAddress().getElementType());
+    emitArrayInit(dest.getAddress(), arrayTy, e->getType(), e, args,
+                  arrayFiller);
+    return;
+  }
+
+  cgf.cgm.errorNYI(
+      "visitCXXParenListOrInitListExpr Record or VariableSizeArray type");
+}
+
+void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
+  AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index fc49d6d..50fa029 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -412,3 +412,25 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
   }
   llvm_unreachable("Unknown APValue kind");
 }
+
+mlir::Value CIRGenModule::emitNullConstant(QualType t, mlir::Location loc) {
+  if (t->getAs<PointerType>()) {
+    return builder.getNullPtr(getTypes().convertTypeForMem(t), loc);
+  }
+
+  if (getTypes().isZeroInitializable(t))
+    return builder.getNullValue(getTypes().convertTypeForMem(t), loc);
+
+  if (getASTContext().getAsConstantArrayType(t)) {
+    errorNYI("CIRGenModule::emitNullConstant ConstantArrayType");
+  }
+
+  if (t->getAs<RecordType>())
+    errorNYI("CIRGenModule::emitNullConstant RecordType");
+
+  assert(t->isMemberDataPointerType() &&
+         "Should only see pointers to data members here!");
+
+  errorNYI("CIRGenModule::emitNullConstant unsupported type");
+  return {};
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index 47fc908..2465ccf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -453,4 +453,47 @@ LValue CIRGenFunction::emitLValue(const Expr *e) {
   }
 }
 
+void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
+                                            QualType ty) {
+  // Ignore empty classes in C++.
+  if (getLangOpts().CPlusPlus) {
+    if (const RecordType *rt = ty->getAs<RecordType>()) {
+      if (cast<CXXRecordDecl>(rt->getDecl())->isEmpty())
+        return;
+    }
+  }
+
+  // Cast the dest ptr to the appropriate i8 pointer type.
+  if (builder.isInt8Ty(destPtr.getElementType())) {
+    cgm.errorNYI(loc, "Cast the dest ptr to the appropriate i8 pointer type");
+  }
+
+  // Get size and alignment info for this aggregate.
+  const CharUnits size = getContext().getTypeSizeInChars(ty);
+  if (size.isZero()) {
+    // But note that getTypeInfo returns 0 for a VLA.
+    if (isa<VariableArrayType>(getContext().getAsArrayType(ty))) {
+      cgm.errorNYI(loc,
+                   "emitNullInitialization for zero size VariableArrayType");
+    } else {
+      return;
+    }
+  }
+
+  // If the type contains a pointer to data member we can't memset it to zero.
+  // Instead, create a null constant and copy it to the destination.
+  // TODO: there are other patterns besides zero that we can usefully memset,
+  // like -1, which happens to be the pattern used by member-pointers.
+  if (!cgm.getTypes().isZeroInitializable(ty)) {
+    cgm.errorNYI(loc, "type is not zero initializable");
+  }
+
+  // In LLVM Codegen: otherwise, just memset the whole thing to zero using
+  // Builder.CreateMemSet. In CIR just emit a store of #cir.zero to the
+  // respective address.
+  // Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
+  const mlir::Value zeroValue = builder.getNullValue(convertType(ty), loc);
+  builder.createStore(loc, zeroValue, destPtr.getPointer());
+}
+
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 5cae4d5..4889c3c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -110,6 +110,8 @@ private:
 public:
   mlir::Value createDummyValue(mlir::Location loc, clang::QualType qt);
 
+  void emitNullInitialization(mlir::Location loc, Address destPtr, QualType ty);
+
 private:
   // Track current variable initialization (if there's one)
   const clang::VarDecl *currVarDecl = nullptr;
@@ -377,6 +379,8 @@ public:
                          mlir::OpBuilder::InsertPoint ip,
                          mlir::Value arraySize = nullptr);
 
+  void emitAggExpr(const clang::Expr *e, AggValueSlot slot);
+
   /// Emit code to compute the specified expression which can have any type. The
   /// result is returned as an RValue struct. If this is an aggregate
   /// expression, the aggloc/agglocvolatile arguments indicate where the result
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 2a37d6c..d3b3b06 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -57,6 +57,18 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
   FP80Ty = cir::FP80Type::get(&getMLIRContext());
   FP128Ty = cir::FP128Type::get(&getMLIRContext());
 
+  PointerAlignInBytes =
+      astContext
+          .toCharUnitsFromBits(
+              astContext.getTargetInfo().getPointerAlign(LangAS::Default))
+          .getQuantity();
+
+  // TODO(CIR): Should be updated once TypeSizeInfoAttr is upstreamed
+  const unsigned sizeTypeSize =
+      astContext.getTypeSize(astContext.getSignedSizeType());
+  PtrDiffTy =
+      cir::IntType::get(&getMLIRContext(), sizeTypeSize, /*isSigned=*/true);
+
   theModule->setAttr(cir::CIRDialect::getTripleAttrName(),
                      builder.getStringAttr(getTriple().str()));
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 734cafa..6ba1ccc 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -113,6 +113,10 @@ public:
   void emitGlobalVarDefinition(const clang::VarDecl *vd,
                                bool isTentative = false);
 
+  /// Return the result of value-initializing the given type, i.e. a null
+  /// expression of the given type.
+  mlir::Value emitNullConstant(QualType t, mlir::Location loc);
+
   cir::FuncOp
   getOrCreateCIRFunction(llvm::StringRef mangledName, mlir::Type funcType,
                          clang::GlobalDecl gd, bool forVTable,
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
index 99c0123..a5b7f0c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 #define LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 
+#include "clang/AST/CharUnits.h"
 #include "clang/CIR/Dialect/IR/CIRTypes.h"
 
 namespace clang::CIRGen {
@@ -47,6 +48,18 @@ struct CIRGenTypeCache {
   cir::DoubleType DoubleTy;
   cir::FP80Type FP80Ty;
   cir::FP128Type FP128Ty;
+
+  mlir::Type PtrDiffTy;
+
+  /// The size and alignment of a pointer into the generic address space.
+  union {
+    unsigned char PointerAlignInBytes;
+    unsigned char PointerSizeInBytes;
+  };
+
+  clang::CharUnits getPointerAlign() const {
+    return clang::CharUnits::fromQuantity(PointerAlignInBytes);
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index aaf3fe2..1e47ccc 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -254,3 +254,30 @@ mlir::Type CIRGenTypes::convertTypeForMem(clang::QualType qualType,
 
   return convertedType;
 }
+
+bool CIRGenTypes::isZeroInitializable(clang::QualType t) {
+  if (t->getAs<PointerType>())
+    return astContext.getTargetNullPointerValue(t) == 0;
+
+  if (const auto *at = astContext.getAsArrayType(t)) {
+    if (isa<IncompleteArrayType>(at))
+      return true;
+
+    if (const auto *cat = dyn_cast<ConstantArrayType>(at))
+      if (astContext.getConstantArrayElementCount(cat) == 0)
+        return true;
+  }
+
+  if (t->getAs<RecordType>()) {
+    cgm.errorNYI(SourceLocation(), "isZeroInitializable for RecordType", t);
+    return false;
+  }
+
+  if (t->getAs<MemberPointerType>()) {
+    cgm.errorNYI(SourceLocation(), "isZeroInitializable for MemberPointerType",
+                 t);
+    return false;
+  }
+
+  return true;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index f280e17e..73948f5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -71,6 +71,10 @@ public:
   /// representation is usually i8 or i32, depending on the target.
   // TODO: convert this comment to account for MLIR's equivalence
   mlir::Type convertTypeForMem(clang::QualType, bool forBitField = false);
+
+  /// Return whether a type can be zero-initialized (in the C++ sense) with an
+  /// LLVM zeroinitializer.
+  bool isZeroInitializable(clang::QualType t);
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index c559e85..d22d518 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -85,11 +85,15 @@ class LValue {
     MatrixElt     // This is a matrix element, use getVector*
   } lvType;
   clang::QualType type;
+  clang::Qualifiers quals;
 
   mlir::Value v;
   mlir::Type elementType;
 
-  void initialize(clang::QualType type) { this->type = type; }
+  void initialize(clang::QualType type, clang::Qualifiers quals) {
+    this->type = type;
+    this->quals = quals;
+  }
 
 public:
   bool isSimple() const { return lvType == Simple; }
@@ -111,16 +115,52 @@ public:
     return Address(getPointer(), elementType, getAlignment());
   }
 
+  const clang::Qualifiers &getQuals() const { return quals; }
+
   static LValue makeAddr(Address address, clang::QualType t) {
     LValue r;
     r.lvType = Simple;
     r.v = address.getPointer();
     r.elementType = address.getElementType();
-    r.initialize(t);
+    r.initialize(t, t.getQualifiers());
     return r;
   }
 };
 
+/// An aggregate value slot.
+class AggValueSlot {
+
+  Address addr;
+  clang::Qualifiers quals;
+
+  /// This is set to true if the memory in the slot is known to be zero before
+  /// the assignment into it.  This means that zero fields don't need to be set.
+  bool zeroedFlag : 1;
+
+public:
+  enum IsZeroed_t { IsNotZeroed, IsZeroed };
+
+  AggValueSlot(Address addr, clang::Qualifiers quals, bool zeroedFlag)
+      : addr(addr), quals(quals), zeroedFlag(zeroedFlag) {}
+
+  static AggValueSlot forAddr(Address addr, clang::Qualifiers quals,
+                              IsZeroed_t isZeroed = IsNotZeroed) {
+    return AggValueSlot(addr, quals, isZeroed);
+  }
+
+  static AggValueSlot forLValue(const LValue &lv) {
+    return forAddr(lv.getAddress(), lv.getQuals());
+  }
+
+  clang::Qualifiers getQualifiers() const { return quals; }
+
+  Address getAddress() const { return addr; }
+
+  bool isIgnored() const { return !addr.isValid(); }
+
+  IsZeroed_t isZeroed() const { return IsZeroed_t(zeroedFlag); }
+};
+
 } // namespace clang::CIRGen
 
 #endif // CLANG_LIB_CIR_CIRGENVALUE_H
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 8ee65c2..da8d63c 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -10,6 +10,7 @@ add_clang_library(clangCIR
   CIRGenerator.cpp
   CIRGenDecl.cpp
   CIRGenExpr.cpp
+  CIRGenExprAggregate.cpp
   CIRGenExprConstant.cpp
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp