[CIR][Upstream] Local initialization for ArrayType (#132974)

This change adds local initialization for ArrayType

Issue #130197

18 files changed, 751 insertions, 10 deletions

diff --git a/clang/lib/CIR/CodeGen/Address.h b/clang/lib/CIR/CodeGen/Address.h
index fba1ffd..2cc8ada 100644
--- a/clang/lib/CIR/CodeGen/Address.h
+++ b/clang/lib/CIR/CodeGen/Address.h
@@ -70,6 +70,14 @@ public:
     return pointerAndKnownNonNull.getPointer();
   }
 
+  mlir::Type getType() const {
+    assert(mlir::cast<cir::PointerType>(
+               pointerAndKnownNonNull.getPointer().getType())
+               .getPointee() == elementType);
+
+    return mlir::cast<cir::PointerType>(getPointer().getType());
+  }
+
   mlir::Type getElementType() const {
     assert(isValid());
     assert(mlir::cast<cir::PointerType>(
@@ -77,6 +85,8 @@ public:
                .getPointee() == elementType);
     return elementType;
   }
+
+  clang::CharUnits getAlignment() const { return alignment; }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index f2153c2..5b832b4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -251,7 +251,7 @@ void CIRGenFunction::emitExprAsInit(const Expr *init, const ValueDecl *d,
     return;
   }
   case cir::TEK_Aggregate:
-    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: aggregate type");
+    emitAggExpr(init, AggValueSlot::forLValue(lvalue));
     return;
   }
   llvm_unreachable("bad evaluation kind");
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
new file mode 100644
index 0000000..36da63d
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
@@ -0,0 +1,277 @@
+//===--- CIRGenExprAgg.cpp - Emit CIR Code from Aggregate Expressions -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code to emit Aggregate Expr nodes as CIR code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenBuilder.h"
+#include "CIRGenFunction.h"
+#include "CIRGenValue.h"
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+
+#include "clang/AST/Expr.h"
+#include "clang/AST/StmtVisitor.h"
+#include <cstdint>
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
+
+  CIRGenFunction &cgf;
+  AggValueSlot dest;
+
+  AggValueSlot ensureSlot(mlir::Location loc, QualType t) {
+    if (!dest.isIgnored())
+      return dest;
+
+    cgf.cgm.errorNYI(loc, "Slot for ignored address");
+    return dest;
+  }
+
+public:
+  AggExprEmitter(CIRGenFunction &cgf, AggValueSlot dest)
+      : cgf(cgf), dest(dest) {}
+
+  void emitArrayInit(Address destPtr, cir::ArrayType arrayTy, QualType arrayQTy,
+                     Expr *exprToVisit, ArrayRef<Expr *> args,
+                     Expr *arrayFiller);
+
+  void emitInitializationToLValue(Expr *e, LValue lv);
+
+  void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
+
+  void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
+
+  void VisitInitListExpr(InitListExpr *e);
+
+  void visitCXXParenListOrInitListExpr(Expr *e, ArrayRef<Expr *> args,
+                                       FieldDecl *initializedFieldInUnion,
+                                       Expr *arrayFiller);
+};
+
+} // namespace
+
+static bool isTrivialFiller(Expr *e) {
+  if (!e)
+    return true;
+
+  if (isa<ImplicitValueInitExpr>(e))
+    return true;
+
+  if (auto *ile = dyn_cast<InitListExpr>(e)) {
+    if (ile->getNumInits())
+      return false;
+    return isTrivialFiller(ile->getArrayFiller());
+  }
+
+  if (const auto *cons = dyn_cast_or_null<CXXConstructExpr>(e))
+    return cons->getConstructor()->isDefaultConstructor() &&
+           cons->getConstructor()->isTrivial();
+
+  return false;
+}
+
+void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
+                                   QualType arrayQTy, Expr *e,
+                                   ArrayRef<Expr *> args, Expr *arrayFiller) {
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
+
+  const uint64_t numInitElements = args.size();
+
+  const QualType elementType =
+      cgf.getContext().getAsArrayType(arrayQTy)->getElementType();
+
+  if (elementType.isDestructedType()) {
+    cgf.cgm.errorNYI(loc, "dtorKind NYI");
+    return;
+  }
+
+  const QualType elementPtrType = cgf.getContext().getPointerType(elementType);
+
+  const mlir::Type cirElementType = cgf.convertType(elementType);
+  const cir::PointerType cirElementPtrType =
+      builder.getPointerTo(cirElementType);
+
+  auto begin = builder.create<cir::CastOp>(loc, cirElementPtrType,
+                                           cir::CastKind::array_to_ptrdecay,
+                                           destPtr.getPointer());
+
+  const CharUnits elementSize =
+      cgf.getContext().getTypeSizeInChars(elementType);
+  const CharUnits elementAlign =
+      destPtr.getAlignment().alignmentOfArrayElement(elementSize);
+
+  // The 'current element to initialize'.  The invariants on this
+  // variable are complicated.  Essentially, after each iteration of
+  // the loop, it points to the last initialized element, except
+  // that it points to the beginning of the array before any
+  // elements have been initialized.
+  mlir::Value element = begin;
+
+  // Don't build the 'one' before the cycle to avoid
+  // emmiting the redundant `cir.const 1` instrs.
+  mlir::Value one;
+
+  // Emit the explicit initializers.
+  for (uint64_t i = 0; i != numInitElements; ++i) {
+    // Advance to the next element.
+    if (i > 0) {
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, i);
+      element = builder.createPtrStride(loc, begin, one);
+    }
+
+    const Address address = Address(element, cirElementType, elementAlign);
+    const LValue elementLV = LValue::makeAddr(address, elementType);
+    emitInitializationToLValue(args[i], elementLV);
+  }
+
+  const uint64_t numArrayElements = arrayTy.getSize();
+
+  // Check whether there's a non-trivial array-fill expression.
+  const bool hasTrivialFiller = isTrivialFiller(arrayFiller);
+
+  // Any remaining elements need to be zero-initialized, possibly
+  // using the filler expression.  We can skip this if the we're
+  // emitting to zeroed memory.
+  if (numInitElements != numArrayElements &&
+      !(dest.isZeroed() && hasTrivialFiller &&
+        cgf.getTypes().isZeroInitializable(elementType))) {
+    // Advance to the start of the rest of the array.
+    if (numInitElements) {
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
+      element = builder.create<cir::PtrStrideOp>(loc, cirElementPtrType,
+                                                 element, one);
+    }
+
+    // Allocate the temporary variable
+    // to store the pointer to first unitialized element
+    const Address tmpAddr = cgf.createTempAlloca(
+        cirElementPtrType, cgf.getPointerAlign(), loc, "arrayinit.temp",
+        /*insertIntoFnEntryBlock=*/false);
+    LValue tmpLV = LValue::makeAddr(tmpAddr, elementPtrType);
+    cgf.emitStoreThroughLValue(RValue::get(element), tmpLV);
+
+    // TODO(CIR): Replace this part later with cir::DoWhileOp
+    for (unsigned i = numInitElements; i != numArrayElements; ++i) {
+      cir::LoadOp currentElement =
+          builder.createLoad(loc, tmpAddr.getPointer());
+
+      // Emit the actual filler expression.
+      const LValue elementLV = LValue::makeAddr(
+          Address(currentElement, cirElementType, elementAlign), elementType);
+
+      if (arrayFiller)
+        emitInitializationToLValue(arrayFiller, elementLV);
+      else
+        emitNullInitializationToLValue(loc, elementLV);
+
+      // Advance pointer and store them to temporary variable
+      one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
+      cir::PtrStrideOp nextElement =
+          builder.createPtrStride(loc, currentElement, one);
+      cgf.emitStoreThroughLValue(RValue::get(nextElement), tmpLV);
+    }
+  }
+}
+
+void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
+  const QualType type = lv.getType();
+
+  if (isa<ImplicitValueInitExpr, CXXScalarValueInitExpr>(e)) {
+    const mlir::Location loc = e->getSourceRange().isValid()
+                                   ? cgf.getLoc(e->getSourceRange())
+                                   : *cgf.currSrcLoc;
+    return emitNullInitializationToLValue(loc, lv);
+  }
+
+  if (isa<NoInitExpr>(e))
+    return;
+
+  if (type->isReferenceType())
+    cgf.cgm.errorNYI("emitInitializationToLValue ReferenceType");
+
+  switch (cgf.getEvaluationKind(type)) {
+  case cir::TEK_Complex:
+    cgf.cgm.errorNYI("emitInitializationToLValue TEK_Complex");
+    break;
+  case cir::TEK_Aggregate:
+    cgf.emitAggExpr(e, AggValueSlot::forLValue(lv));
+    return;
+  case cir::TEK_Scalar:
+    if (lv.isSimple())
+      cgf.emitScalarInit(e, cgf.getLoc(e->getSourceRange()), lv);
+    else
+      cgf.emitStoreThroughLValue(RValue::get(cgf.emitScalarExpr(e)), lv);
+    return;
+  }
+}
+
+void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
+                                                    LValue lv) {
+  const QualType type = lv.getType();
+
+  // If the destination slot is already zeroed out before the aggregate is
+  // copied into it, we don't have to emit any zeros here.
+  if (dest.isZeroed() && cgf.getTypes().isZeroInitializable(type))
+    return;
+
+  if (cgf.hasScalarEvaluationKind(type)) {
+    // For non-aggregates, we can store the appropriate null constant.
+    mlir::Value null = cgf.cgm.emitNullConstant(type, loc);
+    if (lv.isSimple()) {
+      cgf.emitStoreOfScalar(null, lv, /* isInitialization */ true);
+      return;
+    }
+
+    cgf.cgm.errorNYI("emitStoreThroughBitfieldLValue");
+    return;
+  }
+
+  // There's a potential optimization opportunity in combining
+  // memsets; that would be easy for arrays, but relatively
+  // difficult for structures with the current code.
+  cgf.emitNullInitialization(loc, lv.getAddress(), lv.getType());
+}
+
+void AggExprEmitter::VisitInitListExpr(InitListExpr *e) {
+  if (e->hadArrayRangeDesignator())
+    llvm_unreachable("GNU array range designator extension");
+
+  if (e->isTransparent())
+    return Visit(e->getInit(0));
+
+  visitCXXParenListOrInitListExpr(
+      e, e->inits(), e->getInitializedFieldInUnion(), e->getArrayFiller());
+}
+
+void AggExprEmitter::visitCXXParenListOrInitListExpr(
+    Expr *e, ArrayRef<Expr *> args, FieldDecl *initializedFieldInUnion,
+    Expr *arrayFiller) {
+
+  const AggValueSlot dest =
+      ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
+
+  if (e->getType()->isConstantArrayType()) {
+    cir::ArrayType arrayTy =
+        cast<cir::ArrayType>(dest.getAddress().getElementType());
+    emitArrayInit(dest.getAddress(), arrayTy, e->getType(), e, args,
+                  arrayFiller);
+    return;
+  }
+
+  cgf.cgm.errorNYI(
+      "visitCXXParenListOrInitListExpr Record or VariableSizeArray type");
+}
+
+void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
+  AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index fc49d6d..50fa029 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -412,3 +412,25 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
   }
   llvm_unreachable("Unknown APValue kind");
 }
+
+mlir::Value CIRGenModule::emitNullConstant(QualType t, mlir::Location loc) {
+  if (t->getAs<PointerType>()) {
+    return builder.getNullPtr(getTypes().convertTypeForMem(t), loc);
+  }
+
+  if (getTypes().isZeroInitializable(t))
+    return builder.getNullValue(getTypes().convertTypeForMem(t), loc);
+
+  if (getASTContext().getAsConstantArrayType(t)) {
+    errorNYI("CIRGenModule::emitNullConstant ConstantArrayType");
+  }
+
+  if (t->getAs<RecordType>())
+    errorNYI("CIRGenModule::emitNullConstant RecordType");
+
+  assert(t->isMemberDataPointerType() &&
+         "Should only see pointers to data members here!");
+
+  errorNYI("CIRGenModule::emitNullConstant unsupported type");
+  return {};
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index 47fc908..2465ccf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -453,4 +453,47 @@ LValue CIRGenFunction::emitLValue(const Expr *e) {
   }
 }
 
+void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
+                                            QualType ty) {
+  // Ignore empty classes in C++.
+  if (getLangOpts().CPlusPlus) {
+    if (const RecordType *rt = ty->getAs<RecordType>()) {
+      if (cast<CXXRecordDecl>(rt->getDecl())->isEmpty())
+        return;
+    }
+  }
+
+  // Cast the dest ptr to the appropriate i8 pointer type.
+  if (builder.isInt8Ty(destPtr.getElementType())) {
+    cgm.errorNYI(loc, "Cast the dest ptr to the appropriate i8 pointer type");
+  }
+
+  // Get size and alignment info for this aggregate.
+  const CharUnits size = getContext().getTypeSizeInChars(ty);
+  if (size.isZero()) {
+    // But note that getTypeInfo returns 0 for a VLA.
+    if (isa<VariableArrayType>(getContext().getAsArrayType(ty))) {
+      cgm.errorNYI(loc,
+                   "emitNullInitialization for zero size VariableArrayType");
+    } else {
+      return;
+    }
+  }
+
+  // If the type contains a pointer to data member we can't memset it to zero.
+  // Instead, create a null constant and copy it to the destination.
+  // TODO: there are other patterns besides zero that we can usefully memset,
+  // like -1, which happens to be the pattern used by member-pointers.
+  if (!cgm.getTypes().isZeroInitializable(ty)) {
+    cgm.errorNYI(loc, "type is not zero initializable");
+  }
+
+  // In LLVM Codegen: otherwise, just memset the whole thing to zero using
+  // Builder.CreateMemSet. In CIR just emit a store of #cir.zero to the
+  // respective address.
+  // Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
+  const mlir::Value zeroValue = builder.getNullValue(convertType(ty), loc);
+  builder.createStore(loc, zeroValue, destPtr.getPointer());
+}
+
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 5cae4d5..4889c3c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -110,6 +110,8 @@ private:
 public:
   mlir::Value createDummyValue(mlir::Location loc, clang::QualType qt);
 
+  void emitNullInitialization(mlir::Location loc, Address destPtr, QualType ty);
+
 private:
   // Track current variable initialization (if there's one)
   const clang::VarDecl *currVarDecl = nullptr;
@@ -377,6 +379,8 @@ public:
                          mlir::OpBuilder::InsertPoint ip,
                          mlir::Value arraySize = nullptr);
 
+  void emitAggExpr(const clang::Expr *e, AggValueSlot slot);
+
   /// Emit code to compute the specified expression which can have any type. The
   /// result is returned as an RValue struct. If this is an aggregate
   /// expression, the aggloc/agglocvolatile arguments indicate where the result
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 2a37d6c..d3b3b06 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -57,6 +57,18 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
   FP80Ty = cir::FP80Type::get(&getMLIRContext());
   FP128Ty = cir::FP128Type::get(&getMLIRContext());
 
+  PointerAlignInBytes =
+      astContext
+          .toCharUnitsFromBits(
+              astContext.getTargetInfo().getPointerAlign(LangAS::Default))
+          .getQuantity();
+
+  // TODO(CIR): Should be updated once TypeSizeInfoAttr is upstreamed
+  const unsigned sizeTypeSize =
+      astContext.getTypeSize(astContext.getSignedSizeType());
+  PtrDiffTy =
+      cir::IntType::get(&getMLIRContext(), sizeTypeSize, /*isSigned=*/true);
+
   theModule->setAttr(cir::CIRDialect::getTripleAttrName(),
                      builder.getStringAttr(getTriple().str()));
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 734cafa..6ba1ccc 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -113,6 +113,10 @@ public:
   void emitGlobalVarDefinition(const clang::VarDecl *vd,
                                bool isTentative = false);
 
+  /// Return the result of value-initializing the given type, i.e. a null
+  /// expression of the given type.
+  mlir::Value emitNullConstant(QualType t, mlir::Location loc);
+
   cir::FuncOp
   getOrCreateCIRFunction(llvm::StringRef mangledName, mlir::Type funcType,
                          clang::GlobalDecl gd, bool forVTable,
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
index 99c0123..a5b7f0c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 #define LLVM_CLANG_LIB_CIR_CIRGENTYPECACHE_H
 
+#include "clang/AST/CharUnits.h"
 #include "clang/CIR/Dialect/IR/CIRTypes.h"
 
 namespace clang::CIRGen {
@@ -47,6 +48,18 @@ struct CIRGenTypeCache {
   cir::DoubleType DoubleTy;
   cir::FP80Type FP80Ty;
   cir::FP128Type FP128Ty;
+
+  mlir::Type PtrDiffTy;
+
+  /// The size and alignment of a pointer into the generic address space.
+  union {
+    unsigned char PointerAlignInBytes;
+    unsigned char PointerSizeInBytes;
+  };
+
+  clang::CharUnits getPointerAlign() const {
+    return clang::CharUnits::fromQuantity(PointerAlignInBytes);
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index aaf3fe2..1e47ccc 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -254,3 +254,30 @@ mlir::Type CIRGenTypes::convertTypeForMem(clang::QualType qualType,
 
   return convertedType;
 }
+
+bool CIRGenTypes::isZeroInitializable(clang::QualType t) {
+  if (t->getAs<PointerType>())
+    return astContext.getTargetNullPointerValue(t) == 0;
+
+  if (const auto *at = astContext.getAsArrayType(t)) {
+    if (isa<IncompleteArrayType>(at))
+      return true;
+
+    if (const auto *cat = dyn_cast<ConstantArrayType>(at))
+      if (astContext.getConstantArrayElementCount(cat) == 0)
+        return true;
+  }
+
+  if (t->getAs<RecordType>()) {
+    cgm.errorNYI(SourceLocation(), "isZeroInitializable for RecordType", t);
+    return false;
+  }
+
+  if (t->getAs<MemberPointerType>()) {
+    cgm.errorNYI(SourceLocation(), "isZeroInitializable for MemberPointerType",
+                 t);
+    return false;
+  }
+
+  return true;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index f280e17e..73948f5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -71,6 +71,10 @@ public:
   /// representation is usually i8 or i32, depending on the target.
   // TODO: convert this comment to account for MLIR's equivalence
   mlir::Type convertTypeForMem(clang::QualType, bool forBitField = false);
+
+  /// Return whether a type can be zero-initialized (in the C++ sense) with an
+  /// LLVM zeroinitializer.
+  bool isZeroInitializable(clang::QualType t);
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index c559e85..d22d518 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -85,11 +85,15 @@ class LValue {
     MatrixElt     // This is a matrix element, use getVector*
   } lvType;
   clang::QualType type;
+  clang::Qualifiers quals;
 
   mlir::Value v;
   mlir::Type elementType;
 
-  void initialize(clang::QualType type) { this->type = type; }
+  void initialize(clang::QualType type, clang::Qualifiers quals) {
+    this->type = type;
+    this->quals = quals;
+  }
 
 public:
   bool isSimple() const { return lvType == Simple; }
@@ -111,16 +115,52 @@ public:
     return Address(getPointer(), elementType, getAlignment());
   }
 
+  const clang::Qualifiers &getQuals() const { return quals; }
+
   static LValue makeAddr(Address address, clang::QualType t) {
     LValue r;
     r.lvType = Simple;
     r.v = address.getPointer();
     r.elementType = address.getElementType();
-    r.initialize(t);
+    r.initialize(t, t.getQualifiers());
     return r;
   }
 };
 
+/// An aggregate value slot.
+class AggValueSlot {
+
+  Address addr;
+  clang::Qualifiers quals;
+
+  /// This is set to true if the memory in the slot is known to be zero before
+  /// the assignment into it.  This means that zero fields don't need to be set.
+  bool zeroedFlag : 1;
+
+public:
+  enum IsZeroed_t { IsNotZeroed, IsZeroed };
+
+  AggValueSlot(Address addr, clang::Qualifiers quals, bool zeroedFlag)
+      : addr(addr), quals(quals), zeroedFlag(zeroedFlag) {}
+
+  static AggValueSlot forAddr(Address addr, clang::Qualifiers quals,
+                              IsZeroed_t isZeroed = IsNotZeroed) {
+    return AggValueSlot(addr, quals, isZeroed);
+  }
+
+  static AggValueSlot forLValue(const LValue &lv) {
+    return forAddr(lv.getAddress(), lv.getQuals());
+  }
+
+  clang::Qualifiers getQualifiers() const { return quals; }
+
+  Address getAddress() const { return addr; }
+
+  bool isIgnored() const { return !addr.isValid(); }
+
+  IsZeroed_t isZeroed() const { return IsZeroed_t(zeroedFlag); }
+};
+
 } // namespace clang::CIRGen
 
 #endif // CLANG_LIB_CIR_CIRGENVALUE_H
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 8ee65c2..da8d63c 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -10,6 +10,7 @@ add_clang_library(clangCIR
   CIRGenerator.cpp
   CIRGenDecl.cpp
   CIRGenExpr.cpp
+  CIRGenExprAggregate.cpp
   CIRGenExprConstant.cpp
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 4ace083..143ed55 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -246,8 +246,8 @@ OpFoldResult cir::ConstantOp::fold(FoldAdaptor /*adaptor*/) {
 //===----------------------------------------------------------------------===//
 
 LogicalResult cir::CastOp::verify() {
-  auto resType = getResult().getType();
-  auto srcType = getSrc().getType();
+  const mlir::Type resType = getResult().getType();
+  const mlir::Type srcType = getSrc().getType();
 
   switch (getKind()) {
   case cir::CastKind::int_to_bool: {
@@ -271,6 +271,15 @@ LogicalResult cir::CastOp::verify() {
       return emitOpError() << "requires !cir.int type for source";
     return success();
   }
+  case cir::CastKind::array_to_ptrdecay: {
+    const auto arrayPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
+    const auto flatPtrTy = mlir::dyn_cast<cir::PointerType>(resType);
+    if (!arrayPtrTy || !flatPtrTy)
+      return emitOpError() << "requires !cir.ptr type for source and result";
+
+    // TODO(CIR): Make sure the AddrSpace of both types are equals
+    return success();
+  }
   case cir::CastKind::bitcast: {
     // Handle the pointer types first.
     auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
@@ -453,9 +462,9 @@ mlir::LogicalResult cir::ReturnOp::verify() {
 
 /// Given the region at `index`, or the parent operation if `index` is None,
 /// return the successor regions. These are the regions that may be selected
-/// during the flow of control. `operands` is a set of optional attributes that
-/// correspond to a constant value for each operand, or null if that operand is
-/// not a constant.
+/// during the flow of control. `operands` is a set of optional attributes
+/// that correspond to a constant value for each operand, or null if that
+/// operand is not a constant.
 void cir::ScopeOp::getSuccessorRegions(
     mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
   // The only region always branch back to the parent operation.
@@ -683,8 +692,8 @@ ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
 }
 
 bool cir::FuncOp::isDeclaration() {
-  // TODO(CIR): This function will actually do something once external function
-  // declarations and aliases are upstreamed.
+  // TODO(CIR): This function will actually do something once external
+  // function declarations and aliases are upstreamed.
   return false;
 }
 
@@ -710,6 +719,25 @@ void cir::FuncOp::print(OpAsmPrinter &p) {
   }
 }
 
+//===----------------------------------------------------------------------===//
+// CIR defined traits
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+mlir::OpTrait::impl::verifySameFirstOperandAndResultType(Operation *op) {
+  if (failed(verifyAtLeastNOperands(op, 1)) || failed(verifyOneResult(op)))
+    return failure();
+
+  const Type type = op->getResult(0).getType();
+  const Type opType = op->getOperand(0).getType();
+
+  if (type != opType)
+    return op->emitOpError()
+           << "requires the same type for first operand and result";
+
+  return success();
+}
+
 // TODO(CIR): The properties of functions that require verification haven't
 // been implemented yet.
 mlir::LogicalResult cir::FuncOp::verify() { return success(); }
diff --git a/clang/lib/CIR/Lowering/CMakeLists.txt b/clang/lib/CIR/Lowering/CMakeLists.txt
index 09e4886..28ec3c5 100644
--- a/clang/lib/CIR/Lowering/CMakeLists.txt
+++ b/clang/lib/CIR/Lowering/CMakeLists.txt
@@ -7,6 +7,7 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_clang_library(clangCIRLoweringCommon
   CIRPasses.cpp
+  LoweringHelpers.cpp
 
   LINK_LIBS
   clangCIR
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 48dc09d..1c45503 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -13,6 +13,7 @@
 #include "LowerToLLVM.h"
 
 #include <deque>
+#include <optional>
 
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
@@ -28,6 +29,7 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/Dialect/Passes.h"
+#include "clang/CIR/LoweringHelpers.h"
 #include "clang/CIR/MissingFeatures.h"
 #include "clang/CIR/Passes.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -523,6 +525,66 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
+    cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+
+  const mlir::TypeConverter *tc = getTypeConverter();
+  const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());
+
+  mlir::Type elementTy =
+      convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementTy());
+  mlir::MLIRContext *ctx = elementTy.getContext();
+
+  // void and function types doesn't really have a layout to use in GEPs,
+  // make it i8 instead.
+  if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
+      mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
+    elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,
+                                       mlir::IntegerType::Signless);
+  // Zero-extend, sign-extend or trunc the pointer value.
+  mlir::Value index = adaptor.getStride();
+  const unsigned width =
+      mlir::cast<mlir::IntegerType>(index.getType()).getWidth();
+  const std::optional<std::uint64_t> layoutWidth =
+      dataLayout.getTypeIndexBitwidth(adaptor.getBase().getType());
+
+  mlir::Operation *indexOp = index.getDefiningOp();
+  if (indexOp && layoutWidth && width != *layoutWidth) {
+    // If the index comes from a subtraction, make sure the extension happens
+    // before it. To achieve that, look at unary minus, which already got
+    // lowered to "sub 0, x".
+    const auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);
+    auto unary = dyn_cast_if_present<cir::UnaryOp>(
+        ptrStrideOp.getStride().getDefiningOp());
+    bool rewriteSub =
+        unary && unary.getKind() == cir::UnaryOpKind::Minus && sub;
+    if (rewriteSub)
+      index = indexOp->getOperand(1);
+
+    // Handle the cast
+    const auto llvmDstType = mlir::IntegerType::get(ctx, *layoutWidth);
+    index = getLLVMIntCast(rewriter, index, llvmDstType,
+                           ptrStrideOp.getStride().getType().isUnsigned(),
+                           width, *layoutWidth);
+
+    // Rewrite the sub in front of extensions/trunc
+    if (rewriteSub) {
+      index = rewriter.create<mlir::LLVM::SubOp>(
+          index.getLoc(), index.getType(),
+          rewriter.create<mlir::LLVM::ConstantOp>(
+              index.getLoc(), index.getType(),
+              mlir::IntegerAttr::get(index.getType(), 0)),
+          index);
+      rewriter.eraseOp(sub);
+    }
+  }
+
+  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
+      ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
     cir::AllocaOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -603,6 +665,15 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
   return mlir::LogicalResult::success();
 }
 
+bool hasTrailingZeros(cir::ConstArrayAttr attr) {
+  auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());
+  return attr.hasTrailingZeros() ||
+         (array && std::count_if(array.begin(), array.end(), [](auto elt) {
+            auto ar = dyn_cast<cir::ConstArrayAttr>(elt);
+            return ar && hasTrailingZeros(ar);
+          }));
+}
+
 mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     cir::ConstantOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -641,6 +712,27 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     }
     assert(!cir::MissingFeatures::opGlobalViewAttr());
     attr = op.getValue();
+  } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
+    const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
+    if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))
+      return op.emitError() << "array does not have a constant initializer";
+
+    std::optional<mlir::Attribute> denseAttr;
+    if (constArr && hasTrailingZeros(constArr)) {
+      const mlir::Value newOp =
+          lowerCirAttrAsValue(op, constArr, rewriter, getTypeConverter());
+      rewriter.replaceOp(op, newOp);
+      return mlir::success();
+    } else if (constArr &&
+               (denseAttr = lowerConstArrayAttr(constArr, typeConverter))) {
+      attr = denseAttr.value();
+    } else {
+      const mlir::Value initVal =
+          lowerCirAttrAsValue(op, op.getValue(), rewriter, typeConverter);
+      rewriter.replaceAllUsesWith(op, initVal);
+      rewriter.eraseOp(op);
+      return mlir::success();
+    }
   } else {
     return op.emitError() << "unsupported constant type " << op.getType();
   }
@@ -1230,6 +1322,8 @@ void ConvertCIRToLLVMPass::runOnOperation() {
   patterns.add<CIRToLLVMStoreOpLowering>(converter, patterns.getContext(), dl);
   patterns.add<CIRToLLVMGlobalOpLowering>(converter, patterns.getContext(), dl);
   patterns.add<CIRToLLVMCastOpLowering>(converter, patterns.getContext(), dl);
+  patterns.add<CIRToLLVMPtrStrideOpLowering>(converter, patterns.getContext(),
+                                             dl);
   patterns.add<
       // clang-format off
                CIRToLLVMBinOpLowering,
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index b2926e7..6f489fb 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -204,6 +204,21 @@ public:
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMPtrStrideOpLowering
+    : public mlir::OpConversionPattern<cir::PtrStrideOp> {
+  mlir::DataLayout const &dataLayout;
+
+public:
+  CIRToLLVMPtrStrideOpLowering(const mlir::TypeConverter &typeConverter,
+                               mlir::MLIRContext *context,
+                               mlir::DataLayout const &dataLayout)
+      : OpConversionPattern(typeConverter, context), dataLayout(dataLayout) {}
+  using mlir::OpConversionPattern<cir::PtrStrideOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::PtrStrideOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
 } // namespace direct
 } // namespace cir
 
diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
new file mode 100644
index 0000000..0320bc4
--- /dev/null
+++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
@@ -0,0 +1,146 @@
+//====- LoweringHelpers.cpp - Lowering helper functions -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains helper functions for lowering from CIR to LLVM or MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/CIR/LoweringHelpers.h"
+#include "clang/CIR/MissingFeatures.h"
+
+mlir::DenseElementsAttr
+convertStringAttrToDenseElementsAttr(cir::ConstArrayAttr attr,
+                                     mlir::Type type) {
+  auto values = llvm::SmallVector<mlir::APInt, 8>{};
+  const auto stringAttr = mlir::cast<mlir::StringAttr>(attr.getElts());
+
+  for (const char element : stringAttr)
+    values.push_back({8, (uint64_t)element});
+
+  const auto arrayTy = mlir::cast<cir::ArrayType>(attr.getType());
+  if (arrayTy.getSize() != stringAttr.size())
+    assert(!cir::MissingFeatures::stringTypeWithDifferentArraySize());
+
+  return mlir::DenseElementsAttr::get(
+      mlir::RankedTensorType::get({(int64_t)values.size()}, type),
+      llvm::ArrayRef(values));
+}
+
+template <> mlir::APInt getZeroInitFromType(mlir::Type ty) {
+  assert(mlir::isa<cir::IntType>(ty) && "expected int type");
+  const auto intTy = mlir::cast<cir::IntType>(ty);
+  return mlir::APInt::getZero(intTy.getWidth());
+}
+
+template <> mlir::APFloat getZeroInitFromType(mlir::Type ty) {
+  assert((mlir::isa<cir::SingleType, cir::DoubleType>(ty)) &&
+         "only float and double supported");
+
+  if (ty.isF32() || mlir::isa<cir::SingleType>(ty))
+    return mlir::APFloat(0.f);
+
+  if (ty.isF64() || mlir::isa<cir::DoubleType>(ty))
+    return mlir::APFloat(0.0);
+
+  llvm_unreachable("NYI");
+}
+
+/// \param attr the ConstArrayAttr to convert
+/// \param values the output parameter, the values array to fill
+/// \param currentDims the shpae of tensor we're going to convert to
+/// \param dimIndex the current dimension we're processing
+/// \param currentIndex the current index in the values array
+template <typename AttrTy, typename StorageTy>
+void convertToDenseElementsAttrImpl(
+    cir::ConstArrayAttr attr, llvm::SmallVectorImpl<StorageTy> &values,
+    const llvm::SmallVectorImpl<int64_t> &currentDims, int64_t dimIndex,
+    int64_t currentIndex) {
+  if (auto stringAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {
+    if (auto arrayType = mlir::dyn_cast<cir::ArrayType>(attr.getType())) {
+      for (auto element : stringAttr) {
+        auto intAttr = cir::IntAttr::get(arrayType.getEltType(), element);
+        values[currentIndex++] = mlir::dyn_cast<AttrTy>(intAttr).getValue();
+      }
+      return;
+    }
+  }
+
+  dimIndex++;
+  std::size_t elementsSizeInCurrentDim = 1;
+  for (std::size_t i = dimIndex; i < currentDims.size(); i++)
+    elementsSizeInCurrentDim *= currentDims[i];
+
+  auto arrayAttr = mlir::cast<mlir::ArrayAttr>(attr.getElts());
+  for (auto eltAttr : arrayAttr) {
+    if (auto valueAttr = mlir::dyn_cast<AttrTy>(eltAttr)) {
+      values[currentIndex++] = valueAttr.getValue();
+      continue;
+    }
+
+    if (auto subArrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(eltAttr)) {
+      convertToDenseElementsAttrImpl<AttrTy>(subArrayAttr, values, currentDims,
+                                             dimIndex, currentIndex);
+      currentIndex += elementsSizeInCurrentDim;
+      continue;
+    }
+
+    if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(eltAttr)) {
+      currentIndex += elementsSizeInCurrentDim;
+      continue;
+    }
+
+    llvm_unreachable("unknown element in ConstArrayAttr");
+  }
+}
+
+template <typename AttrTy, typename StorageTy>
+mlir::DenseElementsAttr convertToDenseElementsAttr(
+    cir::ConstArrayAttr attr, const llvm::SmallVectorImpl<int64_t> &dims,
+    mlir::Type elementType, mlir::Type convertedElementType) {
+  unsigned vectorSize = 1;
+  for (auto dim : dims)
+    vectorSize *= dim;
+  auto values = llvm::SmallVector<StorageTy, 8>(
+      vectorSize, getZeroInitFromType<StorageTy>(elementType));
+  convertToDenseElementsAttrImpl<AttrTy>(attr, values, dims, /*currentDim=*/0,
+                                         /*initialIndex=*/0);
+  return mlir::DenseElementsAttr::get(
+      mlir::RankedTensorType::get(dims, convertedElementType),
+      llvm::ArrayRef(values));
+}
+
+std::optional<mlir::Attribute>
+lowerConstArrayAttr(cir::ConstArrayAttr constArr,
+                    const mlir::TypeConverter *converter) {
+  // Ensure ConstArrayAttr has a type.
+  const auto typedConstArr = mlir::cast<mlir::TypedAttr>(constArr);
+
+  // Ensure ConstArrayAttr type is a ArrayType.
+  const auto cirArrayType = mlir::cast<cir::ArrayType>(typedConstArr.getType());
+
+  // Is a ConstArrayAttr with an cir::ArrayType: fetch element type.
+  mlir::Type type = cirArrayType;
+  auto dims = llvm::SmallVector<int64_t, 2>{};
+  while (auto arrayType = mlir::dyn_cast<cir::ArrayType>(type)) {
+    dims.push_back(arrayType.getSize());
+    type = arrayType.getEltType();
+  }
+
+  if (mlir::isa<mlir::StringAttr>(constArr.getElts()))
+    return convertStringAttrToDenseElementsAttr(constArr,
+                                                converter->convertType(type));
+  if (mlir::isa<cir::IntType>(type))
+    return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>(
+        constArr, dims, type, converter->convertType(type));
+
+  if (mlir::isa<cir::CIRFPTypeInterface>(type))
+    return convertToDenseElementsAttr<cir::FPAttr, mlir::APFloat>(
+        constArr, dims, type, converter->convertType(type));
+
+  return std::nullopt;
+}