path: root/hw/xive2.c

// SPDX-License-Identifier: Apache-2.0
/*
 * XIVE2: eXternal Interrupt Virtualization Engine. POWER10 interrupt
 * controller
 *
 * Copyright (c) 2016-2019, IBM Corporation.
 */

#define pr_fmt(fmt) "XIVE: " fmt

#include <skiboot.h>
#include <slw.h>
#include <xscom.h>
#include <chip.h>
#include <io.h>
#include <xive.h>
#include <xive2-regs.h>
#include <xscom-p10-regs.h>
#include <interrupts.h>
#include <timebase.h>
#include <bitmap.h>
#include <buddy.h>
#include <phys-map.h>
#include <p10_stop_api.H>


/* Verbose debug */
#undef XIVE_VERBOSE_DEBUG

/* Extra debug options used in debug builds */
#ifdef DEBUG
#define XIVE_CHECK_LOCKS
#define XIVE_DEBUG_INIT_CACHE_UPDATES
#define XIVE_EXTRA_CHECK_INIT_CACHE
#else
#undef  XIVE_CHECK_LOCKS
#undef  XIVE_DEBUG_INIT_CACHE_UPDATES
#undef  XIVE_EXTRA_CHECK_INIT_CACHE
#endif

/*
 * VSDs, blocks, set translation etc...
 *
 * For the following data structures, the XIVE use a mechanism called
 * Virtualization Structure Tables (VST) to manage the memory layout
 * and access: ESBs (Event State Buffers), EAS (Event assignment
 * structures), ENDs (Event Notification Descriptors) and NVT/NVP
 * (Notification Virtual Targets/Processors).
 *
 * These structures divide those tables into 16 "blocks". Each XIVE
 * instance has a definition for all 16 blocks that can either represent
 * an actual table in memory or a remote XIVE MMIO port to access a
 * block that is owned by that remote XIVE.
 *
 * Our SW design will consist of allocating one block per chip (and thus
 * per XIVE instance) for now, thus giving us up to 16 supported chips in
 * the system. We may have to revisit that if we ever support systems with
 * more than 16 chips but that isn't on our radar at the moment or if we
 * want to do like pHyp on some machines and dedicate 2 blocks per chip
 * for some structures.
 *
 * Thus we need to be careful that we never expose to Linux the concept
 * of block and block boundaries, but instead we provide full number ranges
 * so that consecutive blocks can be supported.
 *
 * Similarily, for MMIO access, the BARs support what is called "set
 * translation" which allows the BAR to be devided into a certain
 * number of sets. Each "set" can be routed to a specific block and
 * offset within a block.
 */

#define XIVE_MAX_BLOCKS		16
#define XIVE_VSD_SIZE		8

/*
 * Max number of ESBs. (direct table)
 *
 * The max number of ESBs supported in the P10 MMIO space is 1TB/128K: 8M.
 *
 * 1M is our current top limit of ESB entries and EAS entries
 * pre-allocated per chip. That allocates 256KB per chip for the state
 * bits and 8M per chip for the EAS.
 */

#define XIVE_INT_ORDER		20 /* 1M interrupts */
#define XIVE_INT_COUNT		(1ul << XIVE_INT_ORDER)

/*
 * First interrupt number, also the first logical interrupt number
 * allocated by Linux (maximum ISA interrupt number + 1)
 */
#define XIVE_INT_FIRST		0x10

/* Corresponding direct table sizes */
#define XIVE_ESB_SIZE		(XIVE_INT_COUNT / 4)
#define XIVE_EAT_SIZE		(XIVE_INT_COUNT * 8)

/* Use 64K for everything by default */
#define XIVE_ESB_SHIFT		(16 + 1) /* trigger + mgmt pages */
#define XIVE_ESB_PAGE_SIZE     (1ul << XIVE_ESB_SHIFT) /* 2 pages */

/*
 * Max number of ENDs. (indirect table)
 *
 * The max number of ENDs supported in the P10 MMIO space is 2TB/128K: 16M.
 * Since one END is 32 bytes, a 64K indirect subpage can hold 2K ENDs.
 * We need 8192 subpages, ie, 64K of memory for the indirect table.
 */
#define END_PER_PAGE		(PAGE_SIZE / sizeof(struct xive_end))

#define XIVE_END_ORDER		23 /* 8M ENDs */
#define XIVE_END_COUNT		(1ul << XIVE_END_ORDER)
#define XIVE_END_TABLE_SIZE	((XIVE_END_COUNT / END_PER_PAGE) * XIVE_VSD_SIZE)

#define XIVE_END_SHIFT		(16 + 1) /* ESn + ESe pages */

/* One bit per number of priorities configured */
#define xive_end_bitmap_size(x)	(XIVE_END_COUNT >> xive_cfg_vp_prio_shift(x))

/* Number of priorities (and thus ENDs) we allocate for each VP */
#define xive_cfg_vp_prio_shift(x) GETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, (x)->config)
#define xive_cfg_vp_prio(x)	(1 << xive_cfg_vp_prio_shift(x))

/* Max priority number */
#define xive_max_prio(x)	(xive_cfg_vp_prio(x) - 1)

/* Priority used for gather/silent escalation (KVM) */
#define xive_escalation_prio(x)	xive_max_prio(x)

/*
 * Max number of VPs. (indirect table)
 *
 * The max number of NVPs we support in our MMIO space is 1TB/128K: 8M.
 * Since one NVP is 32 bytes, a 64K indirect subpage can hold 2K NVPs.
 * We need 4096 pointers, ie, 32K of memory for the indirect table.
 *
 * However, we use 8 priorities (by default) per NVP and the number of
 * ENDs is configured to 8M. Therefore, our VP space is limited to 1M.
 */
#define VP_PER_PAGE		(PAGE_SIZE / sizeof(struct xive_nvp))

#define XIVE_VP_ORDER(x)	(XIVE_END_ORDER - xive_cfg_vp_prio_shift(x))
#define XIVE_VP_COUNT(x)	(1ul << XIVE_VP_ORDER(x))
#define XIVE_VP_TABLE_SIZE(x)	((XIVE_VP_COUNT(x) / VP_PER_PAGE) * XIVE_VSD_SIZE)

#define XIVE_NVP_SHIFT		17 /* NVPG BAR: two pages, even NVP, odd NVG */

/* VP Space maximums in Gen1 and Gen2 modes */
#define VP_SHIFT_GEN1		19	/* in sync with END_W6_VP_OFFSET_GEN1 */
#define VP_SHIFT_GEN2		24	/* in sync with END_W6_VP_OFFSET */

/*
 * VP ids for HW threads.
 *
 * Depends on the thread id bits configuration of the IC. 8bit is the
 * default for P10 and 7bit for p9.
 *
 * These values are global because they should be common to all chips
 */
static uint32_t xive_threadid_shift;
static uint32_t	xive_hw_vp_base;
static uint32_t xive_hw_vp_count;

/*
 * The XIVE operation mode indicates the active "API" and corresponds
 * to the "version/mode" parameter of the opal_xive_reset() call
 */
static enum {
	/* No XICS emulation */
	XIVE_MODE_EXPL	= OPAL_XIVE_MODE_EXPL, /* default */
	XIVE_MODE_NONE,
} xive_mode = XIVE_MODE_NONE;

/*
 * The XIVE exploitation mode options indicates the active features and
 * is part of the mode parameter of the opal_xive_reset() call
 */
static uint64_t xive_expl_options;

#define XIVE_EXPL_ALL_OPTIONS 0

/*
 * Each source controller has one of these. There's one embedded in
 * the XIVE struct for IPIs
 */
struct xive_src {
	struct irq_source		is;
	const struct irq_source_ops	*orig_ops;
	struct xive			*xive;
	void				*esb_mmio;
	uint32_t			esb_base;
	uint32_t			esb_shift;
	uint32_t			flags;
};

struct xive_cpu_state {
	struct xive	*xive;
	void		*tm_ring1;

	/* Base HW VP and associated queues */
	uint32_t	vp_blk;
	uint32_t	vp_idx;
	uint32_t	end_blk;
	uint32_t	end_idx; /* Base end index of a block of 8 */

	struct lock	lock;
};

enum xive_generation {
	XIVE_GEN1 = 1, /* P9 compat mode */
	XIVE_GEN2 = 2, /* P10 default */
};

enum xive_quirks {
	/* HW527671 - 8bits Hardwired Thread Id range not implemented */
	XIVE_QUIRK_THREADID_7BITS	= 0x00000001,
	/* HW542974 - interrupt command priority checker not working properly */
	XIVE_QUIRK_BROKEN_PRIO_CHECK	= 0x00000002,
};

struct xive {
	uint32_t		 chip_id;
	uint32_t		 block_id;
	struct dt_node		*x_node;

	enum xive_generation	 generation;
	uint64_t		 capabilities;
	uint64_t		 config;

	uint64_t		 xscom_base;

	/* MMIO regions */
	void			*ic_base;
	uint64_t		 ic_size;
	uint32_t		 ic_shift;
	void			*ic_tm_direct_base;

	void			*tm_base;
	uint64_t		 tm_size;
	uint32_t		 tm_shift;
	void			*nvp_base;
	uint64_t		 nvp_size;
	void			*esb_base;
	uint64_t		 esb_size;
	void			*end_base;
	uint64_t		 end_size;

	/* Set on XSCOM register access error */
	bool			 last_reg_error;

	/* Per-XIVE mutex */
	struct lock		 lock;

	/* Pre-allocated tables.
	 *
	 * We setup all the VDS for actual tables (ie, by opposition to
	 * forwarding ports) as either direct pre-allocated or indirect
	 * and partially populated.
	 *
	 * Currently, the ESB and the EAS tables are direct and fully
	 * pre-allocated based on XIVE_INT_COUNT.
	 *
	 * The other tables are indirect, we thus pre-allocate the indirect
	 * table (ie, pages of pointers) and populate enough of the pages
	 * for our basic setup using 64K subpages.
	 *
	 * The size of the indirect tables are driven by XIVE_VP_COUNT
	 * and XIVE_END_COUNT. The number of pre-allocated ones are
	 * driven by xive_hw_vp_count for the HW threads. The number
	 * of END depends on number of VP.
	 */

	/* Direct SBE and EAT tables */
	void			*sbe_base;
	void			*eat_base;

	/* Indirect END table. NULL entries are unallocated, count is
	 * the numbre of pointers (ie, sub page placeholders).
	 */
	beint64_t		*end_ind_base;
	uint32_t		 end_ind_count;
	uint64_t 		 end_ind_size;

	/* END allocation bitmap. Each bit represent #priority ENDs */
	bitmap_t		*end_map;

	/* Indirect NVT/VP table. NULL entries are unallocated, count is
	 * the numbre of pointers (ie, sub page placeholders).
	 */
	beint64_t		*vp_ind_base;
	uint32_t		 vp_ind_count;
	uint64_t 		 vp_ind_size;

	/* VP space size. Depends on Gen1/2 mode */
	uint32_t		 vp_shift;

	/* Pool of donated pages for provisioning indirect END and VP pages */
	struct list_head	 donated_pages;

	/* To ease a possible change to supporting more than one block of
	 * interrupts per chip, we store here the "base" global number
	 * and max number of interrupts for this chip. The global number
	 * encompass the block number and index.
	 */
	uint32_t		 int_base;
	uint32_t		 int_count;

	/* Due to the overlap between IPIs and HW sources in the EAS table,
	 * we keep some kind of top-down allocator. It is used for HW sources
	 * to "allocate" interrupt entries and will limit what can be handed
	 * out as IPIs. Of course this assumes we "allocate" all HW sources
	 * before we start handing out IPIs.
	 *
	 * Note: The numbers here are global interrupt numbers so that we can
	 * potentially handle more than one block per chip in the future.
	 */
	uint32_t		 int_hw_bot;	/* Bottom of HW allocation */
	uint32_t		 int_ipi_top;	/* Highest IPI handed out so far + 1 */

	/* The IPI allocation bitmap */
	bitmap_t		*ipi_alloc_map;

	/* We keep track of which interrupts were ever enabled to
	 * speed up xive_reset
	 */
	bitmap_t		*int_enabled_map;

	/* Embedded source IPIs */
	struct xive_src		 ipis;

	/* Embedded escalation interrupts */
	struct xive_src		 esc_irqs;

	/* In memory queue overflow */
	void			*q_ovf;

	/* Cache/sync injection */
	uint64_t		 sync_inject_size;
	void			*sync_inject;

	/* INT HW Errata */
	uint64_t		quirks;
};

/* First XIVE unit configured on the system */
static struct xive *one_xive;

/* Global DT node */
static struct dt_node *xive_dt_node;

/* Block <-> Chip conversions.
 *
 * As chipIDs may not be within the range of 16 block IDs supported by XIVE,
 * we have a 2 way conversion scheme.
 *
 * From block to chip, use the global table below.
 *
 * From chip to block, a field in struct proc_chip contains the first block
 * of that chip. For now we only support one block per chip but that might
 * change in the future
 */
#define XIVE_INVALID_CHIP	0xffffffff
#define XIVE_MAX_CHIPS		16
static uint32_t xive_block_to_chip[XIVE_MAX_CHIPS];
static uint32_t xive_block_count;

static uint32_t xive_chip_to_block(uint32_t chip_id)
{
	struct proc_chip *c = get_chip(chip_id);

	assert(c);
	assert(c->xive);
	return c->xive->block_id;
}

/*
 * Conversion between GIRQ and block/index.
 *
 * ------------------------------------
 * |000E|BLOC|                   INDEX|
 * ------------------------------------
 *   4     4           24
 *
 * the E bit indicates that this is an escalation interrupt, in
 * that case, the BLOC/INDEX represents the END containing the
 * corresponding escalation descriptor.
 *
 * Global interrupt numbers for non-escalation interrupts are thus
 * limited to 28 bits.
 */

#define INT_SHIFT		24
#define INT_ESC_SHIFT		(INT_SHIFT + 4) /* 4bits block id */

#if XIVE_INT_ORDER > INT_SHIFT
#error "Too many ESBs for IRQ encoding"
#endif

#if XIVE_END_ORDER > INT_SHIFT
#error "Too many ENDs for escalation IRQ number encoding"
#endif

#define GIRQ_TO_BLK(__g)	(((__g) >> INT_SHIFT) & 0xf)
#define GIRQ_TO_IDX(__g)	((__g) & ((1 << INT_SHIFT) - 1))
#define BLKIDX_TO_GIRQ(__b,__i)	(((uint32_t)(__b)) << INT_SHIFT | (__i))

#define GIRQ_IS_ESCALATION(__g)	((__g) & (1 << INT_ESC_SHIFT))
#define MAKE_ESCALATION_GIRQ(__b,__i)(BLKIDX_TO_GIRQ(__b,__i) | (1 << INT_ESC_SHIFT))


/* Block/IRQ to chip# conversions */
#define PC_BLK_TO_CHIP(__b)	(xive_block_to_chip[__b])
#define VC_BLK_TO_CHIP(__b)	(xive_block_to_chip[__b])
#define GIRQ_TO_CHIP(__isn)	(VC_BLK_TO_CHIP(GIRQ_TO_BLK(__isn)))

/* Routing of physical processors to VPs */
#define PIR2VP_IDX( __pir)	(xive_hw_vp_base | P10_PIR2LOCALCPU(__pir))
#define PIR2VP_BLK(__pir)	(xive_chip_to_block(P10_PIR2GCID(__pir)))
#define VP2PIR(__blk, __idx)	(P10_PIRFROMLOCALCPU(VC_BLK_TO_CHIP(__blk), (__idx) & 0xff))

/* Decoding of OPAL API VP IDs. The VP IDs are encoded as follow
 *
 * Block group mode:
 *
 * -----------------------------------
 * |GVEOOOOO|                   INDEX|
 * -----------------------------------
 *  ||   |
 *  ||  Order
 *  |Virtual
 *  Group
 *
 * G (Group)   : Set to 1 for a group VP (not currently supported)
 * V (Virtual) : Set to 1 for an allocated VP (vs. a physical processor ID)
 * E (Error)   : Should never be 1, used internally for errors
 * O (Order)   : Allocation order of the VP block
 *
 * The conversion is thus done as follow (groups aren't implemented yet)
 *
 *  If V=0, O must be 0 and 24-bit INDEX value is the PIR
 *  If V=1, the order O group is allocated such that if N is the number of
 *          chip bits considered for allocation (*)
 *          then the INDEX is constructed as follow (bit numbers such as 0=LSB)
 *           - bottom O-N bits is the index within the "VP block"
 *           - next N bits is the XIVE blockID of the VP
 *           - the remaining bits is the per-chip "base"
 *          so the conversion consists of "extracting" the block ID and moving
 *          down the upper bits by N bits.
 *
 * In non-block-group mode, the difference is that the blockID is
 * on the left of the index (the entire VP block is in a single
 * block ID)
 */

#define VP_GROUP_SHIFT		31
#define VP_VIRTUAL_SHIFT	30
#define VP_ERROR_SHIFT		29
#define VP_ORDER_SHIFT		24

#define vp_group(vp)		(((vp) >> VP_GROUP_SHIFT) & 1)
#define vp_virtual(vp) 		(((vp) >> VP_VIRTUAL_SHIFT) & 1)
#define vp_order(vp)		(((vp) >> VP_ORDER_SHIFT) & 0x1f)
#define vp_index(vp)		((vp) & ((1 << VP_ORDER_SHIFT) - 1))

/* VP allocation */
static uint32_t xive_chips_alloc_bits = 0;
static struct buddy *xive_vp_buddy;
static struct lock xive_buddy_lock = LOCK_UNLOCKED;

/* VP# decoding/encoding */
static bool xive_decode_vp(uint32_t vp, uint32_t *blk, uint32_t *idx,
			   uint8_t *order, bool *group)
{
	uint32_t o = vp_order(vp);
	uint32_t n = xive_chips_alloc_bits;
	uint32_t index = vp_index(vp);
	uint32_t imask = (1 << (o - n)) - 1;

	/* Groups not supported yet */
	if (vp_group(vp))
		return false;
	if (group)
		*group = false;

	/* PIR case */
	if (!vp_virtual(vp)) {
		if (find_cpu_by_pir(index) == NULL)
			return false;
		if (blk)
			*blk = PIR2VP_BLK(index);
		if (idx)
			*idx = PIR2VP_IDX(index);
		return true;
	}

	/* Ensure o > n, we have *at least* 2 VPs per block */
	if (o <= n)
		return false;

	/* Combine the index base and index */
	if (idx)
		*idx = ((index >> n) & ~imask) | (index & imask);
	/* Extract block ID */
	if (blk)
		*blk = (index >> (o - n)) & ((1 << n) - 1);

	/* Return order as well if asked for */
	if (order)
		*order = o;

	return true;
}

static uint32_t xive_encode_vp(uint32_t blk, uint32_t idx, uint32_t order)
{
	uint32_t vp = (1 << VP_VIRTUAL_SHIFT) | (order << VP_ORDER_SHIFT);
	uint32_t n = xive_chips_alloc_bits;
	uint32_t imask = (1 << (order - n)) - 1;

	vp |= (idx & ~imask) << n;
	vp |= blk << (order - n);
	vp |= idx & imask;
	return  vp;
}

/*
 * XSCOM/MMIO helpers
 */
#define XIVE_NO_MMIO -1

#define xive_regw(__x, __r, __v) \
	__xive_regw(__x, __r, X_##__r, __v, #__r)
#define xive_regr(__x, __r) \
	__xive_regr(__x, __r, X_##__r, #__r)
#define xive_regwx(__x, __r, __v) \
	__xive_regw(__x, XIVE_NO_MMIO, X_##__r, __v, #__r)
#define xive_regrx(__x, __r) \
	__xive_regr(__x, XIVE_NO_MMIO, X_##__r, #__r)

#ifdef XIVE_VERBOSE_DEBUG
#define xive_vdbg(__x,__fmt,...)	prlog(PR_DEBUG,"[ IC %02x  ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_vdbg(__c,__fmt,...)	prlog(PR_DEBUG,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#else
#define xive_vdbg(x,fmt,...)		do { } while(0)
#define xive_cpu_vdbg(x,fmt,...)	do { } while(0)
#endif

#define xive_dbg(__x,__fmt,...)		prlog(PR_DEBUG,"[ IC %02x  ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_dbg(__c,__fmt,...)	prlog(PR_DEBUG,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_notice(__x,__fmt,...)	prlog(PR_NOTICE,"[ IC %02x  ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_notice(__c,__fmt,...)	prlog(PR_NOTICE,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_warn(__x,__fmt,...)	prlog(PR_WARNING,"[ IC %02x  ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_warn(__c,__fmt,...)	prlog(PR_WARNING,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_err(__x,__fmt,...)		prlog(PR_ERR,"[ IC %02x  ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_err(__c,__fmt,...)	prlog(PR_ERR,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)

/*
 * The XIVE subengine being accessed can be deduced from the XSCOM
 * reg, and from there, the page offset in the IC BAR.
 */
static void* xive_ic_page(struct xive *x, uint32_t x_reg)
{
	uint64_t pgoff = (x_reg >> 8) & 0x3;

	return x->ic_base + (pgoff << x->ic_shift);
}

static void __xive_regw(struct xive *x, uint32_t m_reg, uint32_t x_reg, uint64_t v,
			const char *rname)
{
	bool use_xscom = (m_reg == XIVE_NO_MMIO) || !x->ic_base;
	int64_t rc;

	x->last_reg_error = false;

	assert(x_reg != 0);

	if (use_xscom) {
		rc = xscom_write(x->chip_id, x->xscom_base + x_reg, v);
		if (rc) {
			if (!rname)
				rname = "???";
			xive_err(x, "Error writing register %s\n", rname);
			/* Anything else we can do here ? */
			x->last_reg_error = true;
		}
	} else {
		out_be64(xive_ic_page(x, x_reg) + m_reg, v);
	}
}

static uint64_t __xive_regr(struct xive *x, uint32_t m_reg, uint32_t x_reg,
			    const char *rname)
{
	bool use_xscom = (m_reg == XIVE_NO_MMIO) || !x->ic_base;
	int64_t rc;
	uint64_t val;

	x->last_reg_error = false;

	assert(x_reg != 0);

	if (use_xscom) {
		rc = xscom_read(x->chip_id, x->xscom_base + x_reg, &val);
		if (rc) {
			if (!rname)
				rname = "???";
			xive_err(x, "Error reading register %s\n", rname);
			/* Anything else we can do here ? */
			x->last_reg_error = true;
			return -1ull;
		}
	} else {
		val = in_be64(xive_ic_page(x, x_reg) + m_reg);
	}
	return val;
}

/* Locate a controller from an IRQ number */
static struct xive *xive_from_isn(uint32_t isn)
{
	uint32_t chip_id = GIRQ_TO_CHIP(isn);
	struct proc_chip *c = get_chip(chip_id);

	if (!c)
		return NULL;
	return c->xive;
}

static struct xive *xive_from_pc_blk(uint32_t blk)
{
	uint32_t chip_id = PC_BLK_TO_CHIP(blk);
	struct proc_chip *c = get_chip(chip_id);

	if (!c)
		return NULL;
	return c->xive;
}

static struct xive *xive_from_vc_blk(uint32_t blk)
{
	uint32_t chip_id = VC_BLK_TO_CHIP(blk);
	struct proc_chip *c = get_chip(chip_id);

	if (!c)
		return NULL;
	return c->xive;
}

static struct xive_end *xive_get_end(struct xive *x, unsigned int idx)
{
	struct xive_end *p;

	if (idx >= (x->end_ind_count * END_PER_PAGE))
		return NULL;
	p = (struct xive_end *)(be64_to_cpu(x->end_ind_base[idx / END_PER_PAGE]) &
			       VSD_ADDRESS_MASK);
	if (!p)
		return NULL;

	return &p[idx % END_PER_PAGE];
}

static struct xive_eas *xive_get_eas(struct xive *x, unsigned int isn)
{
	struct xive_eas *eat;
	uint32_t idx = GIRQ_TO_IDX(isn);

	if (GIRQ_IS_ESCALATION(isn)) {
		/* Allright, an escalation EAS is buried inside an END, let's
		 * try to find it
		 */
		struct xive_end *end;

		if (x->chip_id != VC_BLK_TO_CHIP(GIRQ_TO_BLK(isn))) {
			xive_err(x, "%s, ESC ISN 0x%x not on right chip\n",
				 __func__, isn);
			return NULL;
		}
		end = xive_get_end(x, idx);
		if (!end) {
			xive_err(x, "%s, ESC ISN 0x%x END not found\n",
				 __func__, isn);
			return NULL;
		}

		/* If using single-escalation, don't let anybody get
		 * to the individual escalation interrupts
		 */
		if (xive_get_field32(END_W0_UNCOND_ESCALATE, end->w0))
			return NULL;

		/* Grab the escalation END */
		return (struct xive_eas *)(char *)&end->w4;
	} else {
		/* Check the block matches */
		if (isn < x->int_base || isn >= x->int_count) {
			xive_err(x, "%s, ISN 0x%x not on right chip\n",
				 __func__, isn);
			return NULL;
		}
		assert (idx < XIVE_INT_COUNT);

		/* If we support >1 block per chip, this should still
		 * work as we are likely to make the table contiguous
		 * anyway
		 */
		eat = x->eat_base;
		assert(eat);

		return eat + idx;
	}
}

static struct xive_nvp *xive_get_vp(struct xive *x, unsigned int idx)
{
	struct xive_nvp *p;

	assert(idx < (x->vp_ind_count * VP_PER_PAGE));
	p = (struct xive_nvp *)(be64_to_cpu(x->vp_ind_base[idx / VP_PER_PAGE]) &
			       VSD_ADDRESS_MASK);
	if (!p)
		return NULL;

	return &p[idx % VP_PER_PAGE];
}

/*
 * Store the END base of the VP in W5, using the new architected field
 * in P10. Used to be the pressure relief interrupt field on P9.
 */
static void xive_vp_set_end_base(struct xive_nvp *vp,
				 uint32_t end_blk, uint32_t end_idx)
{
	vp->w5 = xive_set_field32(NVP_W5_VP_END_BLOCK, 0, end_blk) |
		xive_set_field32(NVP_W5_VP_END_INDEX, 0, end_idx);

	/* This is the criteria to know if a VP was allocated */
	assert(vp->w5 != 0);
}

static void xive_init_default_vp(struct xive_nvp *vp,
				 uint32_t end_blk, uint32_t end_idx)
{
	memset(vp, 0, sizeof(struct xive_nvp));

	xive_vp_set_end_base(vp, end_blk, end_idx);

	vp->w0 = xive_set_field32(NVP_W0_VALID, 0, 1);
}

/*
 * VPs of the HW threads have their own set of ENDs which is allocated
 * when XIVE is initialized. These are tagged with a FIRMWARE bit so
 * that they can be identified when the driver is reset (kexec).
 */
static void xive_init_hw_end(struct xive_end *end)
{
	memset(end, 0, sizeof(struct xive_end));
	end->w0 = xive_set_field32(END_W0_FIRMWARE1, 0, 1);
}

static void *xive_get_donated_page(struct xive *x)
{
	return (void *)list_pop_(&x->donated_pages, 0);
}

#define XIVE_ALLOC_IS_ERR(_idx)	((_idx) >= 0xfffffff0)

#define XIVE_ALLOC_NO_SPACE	0xffffffff /* No possible space */
#define XIVE_ALLOC_NO_IND	0xfffffffe /* Indirect need provisioning */
#define XIVE_ALLOC_NO_MEM	0xfffffffd /* Local allocation failed */

static uint32_t xive_alloc_end_set(struct xive *x, bool alloc_indirect)
{
	uint32_t ind_idx;
	int idx;
	int end_base_idx;

	xive_vdbg(x, "Allocating END set...\n");

	assert(x->end_map);

	/* Allocate from the END bitmap. Each bit is 8 ENDs */
	idx = bitmap_find_zero_bit(*x->end_map, 0, xive_end_bitmap_size(x));
	if (idx < 0) {
		xive_dbg(x, "Allocation from END bitmap failed !\n");
		return XIVE_ALLOC_NO_SPACE;
	}

	end_base_idx = idx << xive_cfg_vp_prio_shift(x);

	xive_vdbg(x, "Got ENDs 0x%x..0x%x\n", end_base_idx,
		  end_base_idx + xive_max_prio(x));

	/* Calculate the indirect page where the ENDs reside */
	ind_idx = end_base_idx / END_PER_PAGE;

	/* Is there an indirect page ? If not, check if we can provision it */
	if (!x->end_ind_base[ind_idx]) {
		/* Default flags */
		uint64_t vsd_flags = SETFIELD(VSD_TSIZE, 0ull, 4) |
			SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
		void *page;

		/* If alloc_indirect is set, allocate the memory from OPAL own,
		 * otherwise try to provision from the donated pool
		 */
		if (alloc_indirect) {
			/* Allocate/provision indirect page during boot only */
			xive_vdbg(x, "Indirect empty, provisioning from local pool\n");
			page = local_alloc(x->chip_id, PAGE_SIZE, PAGE_SIZE);
			if (!page) {
				xive_dbg(x, "provisioning failed !\n");
				return XIVE_ALLOC_NO_MEM;
			}
			vsd_flags |= VSD_FIRMWARE;
		} else {
			xive_vdbg(x, "Indirect empty, provisioning from donated pages\n");
			page = xive_get_donated_page(x);
			if (!page) {
				xive_vdbg(x, "no idirect pages available !\n");
				return XIVE_ALLOC_NO_IND;
			}
		}
		memset(page, 0, PAGE_SIZE);
		x->end_ind_base[ind_idx] = cpu_to_be64(vsd_flags |
			(((uint64_t)page) & VSD_ADDRESS_MASK));
		/* Any cache scrub needed ? */
	}

	bitmap_set_bit(*x->end_map, idx);
	return end_base_idx;
}

static void xive_free_end_set(struct xive *x, uint32_t ends)
{
	uint32_t idx;
	uint8_t  prio_mask = xive_max_prio(x);

	xive_vdbg(x, "Freeing END 0x%x..0x%x\n", ends, ends + xive_max_prio(x));

	assert((ends & prio_mask) == 0);
	assert(x->end_map);

	idx = ends >> xive_cfg_vp_prio_shift(x);
	bitmap_clr_bit(*x->end_map, idx);
}

static bool xive_provision_vp_ind(struct xive *x, uint32_t vp_idx, uint32_t order)
{
	uint32_t pbase, pend, i;

	pbase = vp_idx / VP_PER_PAGE;
	pend  = (vp_idx + (1 << order)) / VP_PER_PAGE;

	for (i = pbase; i <= pend; i++) {
		void *page;
		u64 vsd;

		/* Already provisioned ? */
		if (x->vp_ind_base[i])
			continue;

		/* Try to grab a donated page */
		page = xive_get_donated_page(x);
		if (!page)
			return false;

		/* Install the page */
		memset(page, 0, PAGE_SIZE);
		vsd = ((uint64_t)page) & VSD_ADDRESS_MASK;
		vsd |= SETFIELD(VSD_TSIZE, 0ull, 4);
		vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
		x->vp_ind_base[i] = cpu_to_be64(vsd);
	}
	return true;
}

static void xive_init_vp_allocator(void)
{
	/* Initialize chip alloc bits */
	xive_chips_alloc_bits = ilog2(xive_block_count);

	prlog(PR_INFO, "%d chips considered for VP allocations\n",
	      1 << xive_chips_alloc_bits);

	/* Allocate a buddy big enough for XIVE_VP_ORDER allocations.
	 *
	 * each bit in the buddy represents 1 << xive_chips_alloc_bits
	 * VPs.
	 */
	xive_vp_buddy = buddy_create(XIVE_VP_ORDER(one_xive));
	assert(xive_vp_buddy);

	/*
	 * We reserve the whole range of VP ids representing HW threads.
	 */
	assert(buddy_reserve(xive_vp_buddy, xive_hw_vp_base,
			     xive_threadid_shift));
}

static uint32_t xive_alloc_vps(uint32_t order)
{
	uint32_t local_order, i;
	int vp;

	/* The minimum order is 2 VPs per chip */
	if (order < (xive_chips_alloc_bits + 1))
		order = xive_chips_alloc_bits + 1;

	/* We split the allocation */
	local_order = order - xive_chips_alloc_bits;

	/* We grab that in the global buddy */
	assert(xive_vp_buddy);
	lock(&xive_buddy_lock);
	vp = buddy_alloc(xive_vp_buddy, local_order);
	unlock(&xive_buddy_lock);
	if (vp < 0)
		return XIVE_ALLOC_NO_SPACE;

	/* Provision on every chip considered for allocation */
	for (i = 0; i < (1 << xive_chips_alloc_bits); i++) {
		struct xive *x = xive_from_pc_blk(i);
		bool success;

		/* Return internal error & log rather than assert ? */
		assert(x);
		lock(&x->lock);
		success = xive_provision_vp_ind(x, vp, local_order);
		unlock(&x->lock);
		if (!success) {
			lock(&xive_buddy_lock);
			buddy_free(xive_vp_buddy, vp, local_order);
			unlock(&xive_buddy_lock);
			return XIVE_ALLOC_NO_IND;
		}
	}

	/* Encode the VP number. "blk" is 0 as this represents
	 * all blocks and the allocation always starts at 0
	 */
	return xive_encode_vp(0, vp, order);
}

static void xive_free_vps(uint32_t vp)
{
	uint32_t idx;
	uint8_t order, local_order;

	assert(xive_decode_vp(vp, NULL, &idx, &order, NULL));

	/* We split the allocation */
	local_order = order - xive_chips_alloc_bits;

	/* Free that in the buddy */
	lock(&xive_buddy_lock);
	buddy_free(xive_vp_buddy, idx, local_order);
	unlock(&xive_buddy_lock);
}

enum xive_cache_type {
	xive_cache_easc,
	xive_cache_esbc,
	xive_cache_endc,
	xive_cache_nxc,
};

/*
 * Cache update
 */

#define FLUSH_CTRL_POLL_VALID PPC_BIT(0)  /* POLL bit is the same for all */

static int64_t __xive_cache_scrub(struct xive *x,
				  enum xive_cache_type ctype,
				  uint64_t block, uint64_t idx,
				  bool want_inval __unused, bool want_disable __unused)
{
	uint64_t ctrl_reg, x_ctrl_reg;
	uint64_t poll_val, ctrl_val;

#ifdef XIVE_CHECK_LOCKS
	assert(lock_held_by_me(&x->lock));
#endif
	switch (ctype) {
	case xive_cache_easc:
		poll_val =
			SETFIELD(VC_EASC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
			SETFIELD(VC_EASC_FLUSH_POLL_OFFSET, 0ll, idx) |
			VC_EASC_FLUSH_POLL_BLOCK_ID_MASK |
			VC_EASC_FLUSH_POLL_OFFSET_MASK;
		xive_regw(x, VC_EASC_FLUSH_POLL, poll_val);
		ctrl_reg = VC_EASC_FLUSH_CTRL;
		x_ctrl_reg = X_VC_EASC_FLUSH_CTRL;
		break;
	case xive_cache_esbc:
		poll_val =
			SETFIELD(VC_ESBC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
			SETFIELD(VC_ESBC_FLUSH_POLL_OFFSET, 0ll, idx) |
			VC_ESBC_FLUSH_POLL_BLOCK_ID_MASK |
			VC_ESBC_FLUSH_POLL_OFFSET_MASK;
		xive_regw(x, VC_ESBC_FLUSH_POLL, poll_val);
		ctrl_reg = VC_ESBC_FLUSH_CTRL;
		x_ctrl_reg = X_VC_ESBC_FLUSH_CTRL;
		break;
	case xive_cache_endc:
		poll_val =
			SETFIELD(VC_ENDC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
			SETFIELD(VC_ENDC_FLUSH_POLL_OFFSET, 0ll, idx) |
			VC_ENDC_FLUSH_POLL_BLOCK_ID_MASK |
			VC_ENDC_FLUSH_POLL_OFFSET_MASK;
		xive_regw(x, VC_ENDC_FLUSH_POLL, poll_val);
		ctrl_reg = VC_ENDC_FLUSH_CTRL;
		x_ctrl_reg = X_VC_ENDC_FLUSH_CTRL;
		break;
	case xive_cache_nxc:
		poll_val =
			SETFIELD(PC_NXC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
			SETFIELD(PC_NXC_FLUSH_POLL_OFFSET, 0ll, idx) |
			PC_NXC_FLUSH_POLL_BLOCK_ID_MASK |
			PC_NXC_FLUSH_POLL_OFFSET_MASK;
		xive_regw(x, PC_NXC_FLUSH_POLL, poll_val);
		ctrl_reg = PC_NXC_FLUSH_CTRL;
		x_ctrl_reg = X_PC_NXC_FLUSH_CTRL;
		break;
	default:
		return OPAL_INTERNAL_ERROR;
	}

	/* XXX Add timeout !!! */
	for (;;) {
		ctrl_val = __xive_regr(x, ctrl_reg, x_ctrl_reg, NULL);
		if (!(ctrl_val & FLUSH_CTRL_POLL_VALID))
			break;
		/* Small delay */
		time_wait(100);
	}
	sync();
	return 0;
}

static int64_t xive_easc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
	return __xive_cache_scrub(x, xive_cache_easc, block, idx, false, false);
}

static int64_t xive_nxc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
	return __xive_cache_scrub(x, xive_cache_nxc, block, idx, false, false);
}

static int64_t xive_nxc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
{
	return __xive_cache_scrub(x, xive_cache_nxc, block, idx, true, false);
}

static int64_t xive_endc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
	return __xive_cache_scrub(x, xive_cache_endc, block, idx, false, false);
}

#define XIVE_CACHE_WATCH_MAX_RETRIES 10

static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
				  uint64_t block, uint64_t idx,
				  uint32_t start_dword, uint32_t dword_count,
				  beint64_t *new_data, bool light_watch,
				  bool synchronous)
{
	uint64_t sreg, sregx, dreg0, dreg0x;
	uint64_t dval0, sval, status;
	int64_t i;
	int retries = 0;

#ifdef XIVE_CHECK_LOCKS
	assert(lock_held_by_me(&x->lock));
#endif
	switch (ctype) {
	case xive_cache_endc:
		sreg = VC_ENDC_WATCH0_SPEC;
		sregx = X_VC_ENDC_WATCH0_SPEC;
		dreg0 = VC_ENDC_WATCH0_DATA0;
		dreg0x = X_VC_ENDC_WATCH0_DATA0;
		sval = SETFIELD(VC_ENDC_WATCH_BLOCK_ID, idx, block);
		break;
	case xive_cache_nxc:
		sreg = PC_NXC_WATCH0_SPEC;
		sregx = X_PC_NXC_WATCH0_SPEC;
		dreg0 = PC_NXC_WATCH0_DATA0;
		dreg0x = X_PC_NXC_WATCH0_DATA0;
		sval = SETFIELD(PC_NXC_WATCH_BLOCK_ID, idx, block);
		break;
	default:
		return OPAL_INTERNAL_ERROR;
	}

	/* The full bit is in the same position for ENDC and NXC */
	if (!light_watch)
		sval |= VC_ENDC_WATCH_FULL;

	for (;;) {
		/* Write the cache watch spec */
		__xive_regw(x, sreg, sregx, sval, NULL);

		/* Load data0 register to populate the watch */
		dval0 = __xive_regr(x, dreg0, dreg0x, NULL);

		/* If new_data is NULL, this is a dummy watch used as a
		 * workaround for a HW bug
		 */
		if (!new_data) {
			__xive_regw(x, dreg0, dreg0x, dval0, NULL);
			return 0;
		}

		/* Write the words into the watch facility. We write in reverse
		 * order in case word 0 is part of it as it must be the last
		 * one written.
		 */
		for (i = start_dword + dword_count - 1; i >= start_dword ;i--) {
			uint64_t dw = be64_to_cpu(new_data[i - start_dword]);
			__xive_regw(x, dreg0 + i * 8, dreg0x + i, dw, NULL);
		}

		/* Write data0 register to trigger the update if word 0 wasn't
		 * written above
		 */
		if (start_dword > 0)
			__xive_regw(x, dreg0, dreg0x, dval0, NULL);

		/* This may not be necessary for light updates (it's possible
		 * that a sync in sufficient, TBD). Ensure the above is
		 * complete and check the status of the watch.
		 */
		status = __xive_regr(x, sreg, sregx, NULL);

		/* Bits FULL and CONFLICT are in the same position in
		 * ENDC and NXC
		 */
		if (!(status & VC_ENDC_WATCH_FULL) ||
		    !(status & VC_ENDC_WATCH_CONFLICT))
			break;
		if (!synchronous)
			return OPAL_BUSY;

		if (++retries == XIVE_CACHE_WATCH_MAX_RETRIES) {
			xive_err(x, "Reached maximum retries %d when doing "
				 "a %s cache update\n", retries,
				 ctype == xive_cache_endc ? "ENDC" : "NXC");
			return OPAL_BUSY;
		}
	}

	/* Perform a scrub with "want_invalidate" set to false to push the
	 * cache updates to memory as well
	 */
	return __xive_cache_scrub(x, ctype, block, idx, false, false);
}

#ifdef XIVE_DEBUG_INIT_CACHE_UPDATES
static bool xive_check_endc_update(struct xive *x, uint32_t idx, struct xive_end *end)
{
	struct xive_end *end_p = xive_get_end(x, idx);
	struct xive_end end2;

	assert(end_p);
	end2 = *end_p;
	end2.w0 &= ~END_W0_RESERVED;
	end2.w1 &= ~END_W1_RESERVED;
	end2.w7 &= ~END_W7_F0_RESERVED;
	if (memcmp(end, &end2, sizeof(struct xive_end)) != 0) {
		xive_err(x, "END update mismatch idx %d\n", idx);
		xive_err(x, "want: %08x %08x %08x %08x\n",
			 end->w0, end->w1, end->w2, end->w3);
		xive_err(x, "      %08x %08x %08x %08x\n",
			 end->w4, end->w5, end->w6, end->w7);
		xive_err(x, "got : %08x %08x %08x %08x\n",
			 end2.w0, end2.w1, end2.w2, end2.w3);
		xive_err(x, "      %08x %08x %08x %08x\n",
			 end2.w4, end2.w5, end2.w6, end2.w7);
		return false;
	}
	return true;
}

static bool xive_check_nxc_update(struct xive *x, uint32_t idx, struct xive_nvp *vp)
{
	struct xive_nvp *vp_p = xive_get_vp(x, idx);
	struct xive_nvp vp2;

	assert(vp_p);
	vp2 = *vp_p;
	if (memcmp(vp, &vp2, sizeof(struct xive_nvp)) != 0) {
		xive_err(x, "VP update mismatch idx %d\n", idx);
		xive_err(x, "want: %08x %08x %08x %08x\n",
			 vp->w0, vp->w1, vp->w2, vp->w3);
		xive_err(x, "      %08x %08x %08x %08x\n",
			 vp->w4, vp->w5, vp->w6, vp->w7);
		xive_err(x, "got : %08x %08x %08x %08x\n",
			 vp2.w0, vp2.w1, vp2.w2, vp2.w3);
		xive_err(x, "      %08x %08x %08x %08x\n",
			 vp2.w4, vp2.w5, vp2.w6, vp2.w7);
		return false;
	}
	return true;
}
#else
static inline bool xive_check_endc_update(struct xive *x __unused,
					uint32_t idx __unused,
					struct xive_end *end __unused)
{
	return true;
}

static inline bool xive_check_nxc_update(struct xive *x __unused,
					 uint32_t idx __unused,
					 struct xive_nvp *vp __unused)
{
	return true;
}
#endif

static int64_t xive_escalation_ive_cache_update(struct xive *x, uint64_t block,
				     uint64_t idx, struct xive_eas *eas,
				     bool synchronous)
{
	return __xive_cache_watch(x, xive_cache_endc, block, idx,
				  2, 1, &eas->w, true, synchronous);
}

static int64_t xive_endc_cache_update(struct xive *x, uint64_t block,
				     uint64_t idx, struct xive_end *end,
				     bool synchronous)
{
	int64_t ret;

	ret = __xive_cache_watch(x, xive_cache_endc, block, idx,
				 0, 4, (beint64_t *)end, false, synchronous);
	xive_check_endc_update(x, idx, end);
	return ret;
}

static int64_t xive_nxc_cache_update(struct xive *x, uint64_t block,
				     uint64_t idx, struct xive_nvp *vp,
				     bool synchronous)
{
	int64_t ret;

	ret = __xive_cache_watch(x, xive_cache_nxc, block, idx,
				 0, 4, (beint64_t *)vp, false, synchronous);
	xive_check_nxc_update(x, idx, vp);
	return ret;
}

/*
 * VSD
 */
static bool xive_set_vsd(struct xive *x, uint32_t tbl, uint32_t idx, uint64_t v)
{
	/* Set VC subengine */
	xive_regw(x, VC_VSD_TABLE_ADDR,
		  SETFIELD(VC_VSD_TABLE_SELECT, 0ull, tbl) |
		  SETFIELD(VC_VSD_TABLE_ADDRESS, 0ull, idx));
	if (x->last_reg_error)
		return false;
	xive_regw(x, VC_VSD_TABLE_DATA, v);
	if (x->last_reg_error)
		return false;

	/* also set PC subengine if table is used */
	if (tbl == VST_EAS || tbl == VST_ERQ || tbl == VST_IC)
		return true;

	xive_regw(x, PC_VSD_TABLE_ADDR,
		  SETFIELD(PC_VSD_TABLE_SELECT, 0ull, tbl) |
		  SETFIELD(PC_VSD_TABLE_ADDRESS, 0ull, idx));
	if (x->last_reg_error)
		return false;
	xive_regw(x, PC_VSD_TABLE_DATA, v);
	if (x->last_reg_error)
		return false;
	return true;
}

static bool xive_set_local_tables(struct xive *x)
{
	uint64_t base, i;

	/* These have to be power of 2 sized */
	assert(is_pow2(XIVE_ESB_SIZE));
	assert(is_pow2(XIVE_EAT_SIZE));

	/* All tables set as exclusive */
	base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);

	/* ESB: direct mode */
	if (!xive_set_vsd(x, VST_ESB, x->block_id, base |
			  (((uint64_t)x->sbe_base) & VSD_ADDRESS_MASK) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(XIVE_ESB_SIZE) - 12)))
		return false;

	/* EAS: direct mode */
	if (!xive_set_vsd(x, VST_EAS, x->block_id, base |
			  (((uint64_t)x->eat_base) & VSD_ADDRESS_MASK) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(XIVE_EAT_SIZE) - 12)))
		return false;

	/* END: indirect mode with 64K subpages */
	if (!xive_set_vsd(x, VST_END, x->block_id, base |
			  (((uint64_t)x->end_ind_base) & VSD_ADDRESS_MASK) |
			  VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull,
						  ilog2(x->end_ind_size) - 12)))
		return false;

	/* NVP: indirect mode with 64K subpages */
	if (!xive_set_vsd(x, VST_NVP, x->block_id, base |
			  (((uint64_t)x->vp_ind_base) & VSD_ADDRESS_MASK) |
			  VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull,
						  ilog2(x->vp_ind_size) - 12)))
		return false;

	/* NVG: not used  */
	/* NVC: not used */

	/* INT and SYNC: indexed with the Topology# */
	if (!xive_set_vsd(x, VST_IC, x->chip_id, base |
			  (((uint64_t)x->ic_base) & VSD_ADDRESS_MASK) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->ic_size) - 12)))
		return false;

	if (!xive_set_vsd(x, VST_SYNC, x->chip_id, base |
			  (((uint64_t)x->sync_inject) & VSD_ADDRESS_MASK) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->sync_inject_size) - 12)))
		return false;

	/*
	 * ERQ: one 64K page for each queue overflow. Indexed with :
	 *
	 * 0:IPI, 1:HWD, 2:NxC, 3:INT, 4:OS-Queue, 5:Pool-Queue, 6:Hard-Queue
	 */
	for (i = 0; i < VC_QUEUE_COUNT; i++) {
		u64 addr = ((uint64_t)x->q_ovf) + i * PAGE_SIZE;
		u64 cfg, sreg, sregx;

		if (!xive_set_vsd(x, VST_ERQ, i, base |
				  (addr & VSD_ADDRESS_MASK) |
			  SETFIELD(VSD_TSIZE, 0ull, 4)))
			return false;

		sreg = VC_QUEUES_CFG_REM0 + i * 8;
		sregx = X_VC_QUEUES_CFG_REM0 + i;
		cfg = __xive_regr(x, sreg, sregx, NULL);
		cfg |= VC_QUEUES_CFG_MEMB_EN;
		cfg = SETFIELD(VC_QUEUES_CFG_MEMB_SZ, cfg, 4);
		__xive_regw(x, sreg, sregx, cfg, NULL);
	}

	return true;
}


/*
 * IC BAR layout
 *
 * Page 0:		Internal CQ register accesses (reads & writes)
 * Page 1:		Internal PC register accesses (reads & writes)
 * Page 2:		Internal VC register accesses (reads & writes)
 * Page 3:		Internal TCTXT (TIMA) reg accesses (read & writes)
 * Page 4:		Notify Port page (writes only, w/data),
 * Page 5:		Reserved
 * Page 6:		Sync Poll page (writes only, dataless)
 * Page 7:		Sync Inject page (writes only, dataless)
 * Page 8:		LSI Trigger page (writes only, dataless)
 * Page 9:		LSI SB Management page (reads & writes dataless)
 * Pages 10-255:	Reserved
 * Pages 256-383: 	Direct mapped Thread Context Area (reads & writes)
 *                	covering the 128 threads in P10.
 * Pages 384-511: 	Reserved
 */

#define XIVE_IC_CQ_PGOFF	0
#define XIVE_IC_PC_PGOFF	1
#define XIVE_IC_VC_PGOFF	2
#define XIVE_IC_TCTXT_PGOFF	3
#define XIVE_NOTIFY_PGOFF	4
#define XIVE_SYNC_POLL_PGOFF	6
#define XIVE_SYNC_INJECT_PGOFF	7
#define XIVE_LSI_TRIGGER_PGOFF	8
#define XIVE_LSI_MGMT_PGOFF	9
#define XIVE_IC_TM_DIRECT_PGOFF 256

static bool xive_configure_ic_bars(struct xive *x)
{
	uint64_t chip_id = x->chip_id;
	uint64_t val;

	/* Reset all bars to zero */
	xive_regwx(x, CQ_RST_CTL, CQ_RST_PB_BAR_RESET);

	/* IC BAR */
	phys_map_get(chip_id, XIVE_IC, 0, (uint64_t *)&x->ic_base, &x->ic_size);
	val = (uint64_t)x->ic_base | CQ_IC_BAR_VALID | CQ_IC_BAR_64K;
	x->ic_shift = 16;

	xive_regwx(x, CQ_IC_BAR, val);
	if (x->last_reg_error)
		return false;

	/*
	 * TM BAR, same address for each chip. Hence we create a fake
	 * chip 0 and use that for all phys_map_get(XIVE_TM) calls.
	 */
	phys_map_get(0, XIVE_TM, 0, (uint64_t *)&x->tm_base, &x->tm_size);
	val = (uint64_t)x->tm_base | CQ_TM_BAR_VALID | CQ_TM_BAR_64K;
	x->tm_shift = 16;

	xive_regwx(x, CQ_TM_BAR, val);
	if (x->last_reg_error)
		return false;

	/* IC BAR sub-pages shortcuts */
	x->ic_tm_direct_base = x->ic_base +
		(XIVE_IC_TM_DIRECT_PGOFF << x->ic_shift);

	return true;
}

/*
 * NVPG, NVC, ESB, END BARs have common attributes: 64k page and only
 * one set covering the whole BAR.
 */
static bool xive_configure_bars(struct xive *x)
{
	uint64_t chip_id = x->chip_id;
	uint64_t val;
	uint64_t esb_size;
	uint64_t end_size;
	uint64_t nvp_size;

	x->nvp_size = XIVE_VP_COUNT(x) << XIVE_NVP_SHIFT;
	x->esb_size = XIVE_INT_COUNT << XIVE_ESB_SHIFT;
	x->end_size = XIVE_END_COUNT << XIVE_END_SHIFT;

	/*
	 * NVC BAR is not configured because we do not use the XIVE2
	 * Crowd capability.
	 */

	/* NVPG BAR: two pages, even NVP, odd NVG */
	phys_map_get(chip_id, XIVE_NVPG, 0, (uint64_t *)&x->nvp_base, &nvp_size);
	if (x->nvp_size > nvp_size) {
		xive_err(x, "NVP table is larger than default: "
			 "0x%012llx > 0x%012llx\n", x->nvp_size, nvp_size);
		return false;
	}

	val = (uint64_t)x->nvp_base | CQ_BAR_VALID | CQ_BAR_64K |
		SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->nvp_size) - 24);
	xive_regwx(x, CQ_NVPG_BAR, val);
	if (x->last_reg_error)
		return false;

	/* ESB BAR */
	phys_map_get(chip_id, XIVE_ESB, 0, (uint64_t *)&x->esb_base, &esb_size);
	if (x->esb_size > esb_size) {
		xive_err(x, "ESB table is larger than default: "
			 "0x%012llx > 0x%012llx\n", x->esb_size, esb_size);
		return false;
	}

	val = (uint64_t)x->esb_base | CQ_BAR_VALID | CQ_BAR_64K |
		SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->esb_size) - 24);
	xive_regwx(x, CQ_ESB_BAR, val);
	if (x->last_reg_error)
		return false;

	/* END BAR */
	phys_map_get(chip_id, XIVE_END, 0, (uint64_t *)&x->end_base, &end_size);
	if (x->end_size > end_size) {
		xive_err(x, "END table is larger than default: "
			 "0x%012llx > 0x%012llx\n", x->end_size, end_size);
		return false;
	}

	val = (uint64_t)x->end_base | CQ_BAR_VALID | CQ_BAR_64K |
		SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->end_size) - 24);
	xive_regwx(x, CQ_END_BAR, val);
	if (x->last_reg_error)
		return false;

	xive_dbg(x, "IC:  %14p [0x%012llx]\n", x->ic_base, x->ic_size);
	xive_dbg(x, "TM:  %14p [0x%012llx]\n", x->tm_base, x->tm_size);
	xive_dbg(x, "NVP: %14p [0x%012llx]\n", x->nvp_base, x->nvp_size);
	xive_dbg(x, "ESB: %14p [0x%012llx]\n", x->esb_base, x->esb_size);
	xive_dbg(x, "END: %14p [0x%012llx]\n", x->end_base, x->end_size);
	xive_dbg(x, "OVF: %14p [0x%012x]\n", x->q_ovf,
		 VC_QUEUE_COUNT * PAGE_SIZE);

	return true;
}

static void xive_dump_mmio(struct xive *x)
{
	prlog(PR_DEBUG, " CQ_CFG_PB_GEN = %016llx\n",
	      in_be64(x->ic_base + CQ_CFG_PB_GEN));
	prlog(PR_DEBUG, " CQ_MSGSND     = %016llx\n",
	      in_be64(x->ic_base + CQ_MSGSND));
}

static const struct {
	uint64_t bitmask;
	const char *name;
} xive_capabilities[] = {
	{ CQ_XIVE_CAP_PHB_PQ_DISABLE, "PHB PQ disable mode support" },
	{ CQ_XIVE_CAP_PHB_ABT, "PHB address based trigger mode support" },
	{ CQ_XIVE_CAP_EXPLOITATION_MODE, "Exploitation mode" },
	{ CQ_XIVE_CAP_STORE_EOI, "StoreEOI mode support" },
	{ CQ_XIVE_CAP_VP_SAVE_RESTORE, "VP Context Save and Restore" },
};

static void xive_dump_capabilities(struct xive *x, uint64_t cap_val)
{
	int i;

	xive_dbg(x, "capabilities: %016llx\n", cap_val);
	xive_dbg(x, "\tVersion: %lld\n",
		 GETFIELD(CQ_XIVE_CAP_VERSION, cap_val));
	xive_dbg(x, "\tUser interrupt priorities: [ 1 - %d ]\n",
		 1 << GETFIELD(CQ_XIVE_CAP_USER_INT_PRIO, cap_val));
	xive_dbg(x, "\tVP interrupt priorities: [ %d - 8 ]\n",
		 1 << GETFIELD(CQ_XIVE_CAP_VP_INT_PRIO, cap_val));
	xive_dbg(x, "\tExtended Blockid bits: %lld\n",
		 4 + GETFIELD(CQ_XIVE_CAP_BLOCK_ID_WIDTH, cap_val));

	for (i = 0; i < ARRAY_SIZE(xive_capabilities); i++) {
		if (xive_capabilities[i].bitmask & cap_val)
			xive_dbg(x, "\t%s\n", xive_capabilities[i].name);
	}
}

static const struct {
	uint64_t bitmask;
	const char *name;
} xive_configs[] = {
	{ CQ_XIVE_CFG_GEN1_TIMA_OS, "Gen1 mode TIMA OS" },
	{ CQ_XIVE_CFG_GEN1_TIMA_HYP, "Gen1 mode TIMA Hyp" },
	{ CQ_XIVE_CFG_GEN1_TIMA_HYP_BLK0, "Gen1 mode TIMA General Hypervisor Block0" },
	{ CQ_XIVE_CFG_GEN1_TIMA_CROWD_DIS, "Gen1 mode TIMA Crowd disable" },
	{ CQ_XIVE_CFG_GEN1_END_ESX, "Gen1 mode END ESx" },
	{ CQ_XIVE_CFG_EN_VP_SAVE_RESTORE, "VP Context Save and Restore" },
	{ CQ_XIVE_CFG_EN_VP_SAVE_REST_STRICT, "VP Context Save and Restore strict" },
};

static void xive_dump_configuration(struct xive *x, const char *prefix,
				    uint64_t cfg_val)
{
	int i ;

	xive_dbg(x, "%s configuration: %016llx\n", prefix, cfg_val);
	xive_dbg(x, "\tHardwired Thread Id range: %lld bits\n",
		 7 + GETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, cfg_val));
	xive_dbg(x, "\tUser Interrupt priorities: [ 1 - %d ]\n",
		 1 << GETFIELD(CQ_XIVE_CFG_USER_INT_PRIO, cfg_val));
	xive_dbg(x, "\tVP Interrupt priorities: [ 0 - %d ]\n", xive_max_prio(x));
	xive_dbg(x, "\tBlockId bits: %lld bits\n",
		 4 + GETFIELD(CQ_XIVE_CFG_BLOCK_ID_WIDTH, cfg_val));
	if (CQ_XIVE_CFG_HYP_HARD_BLKID_OVERRIDE & cfg_val)
		xive_dbg(x, "\tHardwired BlockId: %lld\n",
			 GETFIELD(CQ_XIVE_CFG_HYP_HARD_BLOCK_ID, cfg_val));

	for (i = 0; i < ARRAY_SIZE(xive_configs); i++) {
		if (xive_configs[i].bitmask & cfg_val)
			xive_dbg(x, "\t%s\n", xive_configs[i].name);
	}
}

/*
 * Default XIVE configuration
 */
#define XIVE_CONFIGURATION                                        \
	(SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, 0ull, CQ_XIVE_CFG_THREADID_8BITS) | \
	 SETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, 0ull, CQ_XIVE_CFG_INT_PRIO_8))

/*
 * Gen1 configuration for tests (QEMU)
 */
#define XIVE_CONFIGURATION_GEN1						\
	(SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, 0ull, CQ_XIVE_CFG_THREADID_7BITS) | \
	 SETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, 0ull, CQ_XIVE_CFG_INT_PRIO_8) | \
	 CQ_XIVE_CFG_GEN1_TIMA_OS |					\
	 CQ_XIVE_CFG_GEN1_TIMA_HYP |					\
	 CQ_XIVE_CFG_GEN1_TIMA_HYP_BLK0 |				\
	 CQ_XIVE_CFG_GEN1_TIMA_CROWD_DIS |				\
	 CQ_XIVE_CFG_GEN1_END_ESX)

static bool xive_has_cap(struct xive *x, uint64_t cap)
{
	return !!x && !!(x->capabilities & cap);
}

#define XIVE_CAN_STORE_EOI(x) xive_has_cap(x, CQ_XIVE_CAP_STORE_EOI)

static bool xive_cfg_save_restore(struct xive *x)
{
	return !!(x->config & CQ_XIVE_CFG_EN_VP_SAVE_RESTORE);
}

/*
 * When PQ_disable is available, configure the ESB cache to improve
 * performance for PHB ESBs.
 *
 * split_mode :
 *   1/3rd of the cache is reserved for PHB ESBs and the rest to
 *   IPIs. This is sufficient to keep all the PHB ESBs in cache and
 *   avoid ESB cache misses during IO interrupt processing.
 *
 * hash_array_enable :
 *   Internal cache hashing optimization. The hash_array tracks for
 *   ESBs where the original trigger came from so that we avoid
 *   getting the EAS into the cache twice.
 */
static void xive_config_esb_cache(struct xive *x)
{
	uint64_t val = xive_regr(x, VC_ESBC_CFG);

	if (xive_has_cap(x, CQ_XIVE_CAP_PHB_PQ_DISABLE)) {
		val |= VC_ESBC_CFG_SPLIT_MODE | VC_ESBC_CFG_HASH_ARRAY_ENABLE;
		val = SETFIELD(VC_ESBC_CFG_MAX_ENTRIES_IN_MODIFIED, val, 0xE);
		xive_dbg(x, "ESB cache configured with split mode "
			 "and hash array. VC_ESBC_CFG=%016llx\n", val);
	} else
		val &= ~VC_ESBC_CFG_SPLIT_MODE;

	xive_regw(x, VC_ESBC_CFG, val);
}

static void xive_config_fused_core(struct xive *x)
{
	uint64_t val = xive_regr(x, TCTXT_CFG);

	if (this_cpu()->is_fused_core) {
		val |= TCTXT_CFG_FUSE_CORE_EN;
		xive_dbg(x, "configured for fused cores. "
			 "PC_TCTXT_CFG=%016llx\n", val);
	} else
		val &= ~TCTXT_CFG_FUSE_CORE_EN;
	xive_regw(x, TCTXT_CFG, val);
}

static void xive_config_reduced_priorities_fixup(struct xive *x)
{
	if (xive_cfg_vp_prio_shift(x) < CQ_XIVE_CFG_INT_PRIO_8 &&
	    x->quirks & XIVE_QUIRK_BROKEN_PRIO_CHECK) {
		uint64_t val = xive_regr(x, PC_ERR1_CFG1);

		val &= ~PC_ERR1_CFG1_INTERRUPT_INVALID_PRIO;
		xive_dbg(x, "workaround for reduced priorities. "
			 "PC_ERR1_CFG1=%016llx\n", val);
		xive_regw(x, PC_ERR1_CFG1, val);
	}
}

static bool xive_config_init(struct xive *x)
{
	x->capabilities = xive_regr(x, CQ_XIVE_CAP);
	xive_dump_capabilities(x, x->capabilities);

	x->generation = GETFIELD(CQ_XIVE_CAP_VERSION, x->capabilities);

	/*
	 * Allow QEMU to override version for tests
	 */
	if (x->generation != XIVE_GEN2 && !chip_quirk(QUIRK_QEMU)) {
		xive_err(x, "Invalid XIVE controller version %d\n",
			 x->generation);
		return false;
	}

	x->config = xive_regr(x, CQ_XIVE_CFG);
	xive_dump_configuration(x, "default", x->config);

	/* Start with default settings */
	x->config = x->generation == XIVE_GEN1 ? XIVE_CONFIGURATION_GEN1 :
		XIVE_CONFIGURATION;

	if (x->quirks & XIVE_QUIRK_THREADID_7BITS)
		x->config = SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, x->config,
				     CQ_XIVE_CFG_THREADID_7BITS);

	/*
	 * Hardwire the block ID. The default value is the topology ID
	 * of the chip which is different from the block.
	 */
	x->config |= CQ_XIVE_CFG_HYP_HARD_BLKID_OVERRIDE |
		SETFIELD(CQ_XIVE_CFG_HYP_HARD_BLOCK_ID, 0ull, x->block_id);

	/*
	 * Enable "VP Context Save and Restore" by default. it is
	 * compatible with KVM which currently does the context
	 * save&restore in the entry/exit path of the vCPU
	 */
	if (x->capabilities & CQ_XIVE_CAP_VP_SAVE_RESTORE)
		x->config |= CQ_XIVE_CFG_EN_VP_SAVE_RESTORE;

	xive_dump_configuration(x, "new", x->config);
	xive_regw(x, CQ_XIVE_CFG, x->config);
	if (xive_regr(x, CQ_XIVE_CFG) != x->config) {
		xive_err(x, "configuration setting failed\n");
	}

	/*
	 * Disable error reporting in the FIR for info errors from the VC.
	 */
	xive_regw(x, CQ_FIRMASK_OR, CQ_FIR_VC_INFO_ERROR_0_2);

	/*
	 * Mask CI Load and Store to bad location, as IPI trigger
	 * pages may be mapped to user space, and a read on the
	 * trigger page causes a checkstop
	 */
	xive_regw(x, CQ_FIRMASK_OR, CQ_FIR_PB_RCMDX_CI_ERR1);

	/*
	 * VP space settings. P9 mode is 19bits.
	 */
	x->vp_shift = x->generation == XIVE_GEN1 ?
		VP_SHIFT_GEN1 : VP_SHIFT_GEN2;

	/*
	 * VP ids for HW threads. These values are hardcoded in the
	 * CAM line of the HW context
	 *
	 *     POWER10     |chip|0000000000000001|threadid|
	 *     28bits        4           16          8
	 *
	 *     POWER9           |chip|000000000001|thrdid |
	 *     23bits              4      12          7
	 */

	/* TODO (cosmetic): set VP ids for HW threads only once */
	xive_threadid_shift = 7 + GETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE,
					   x->config);

	xive_hw_vp_base  = 1 << xive_threadid_shift;
	xive_hw_vp_count = 1 << xive_threadid_shift;

	xive_dbg(x, "store EOI is %savailable\n",
		 XIVE_CAN_STORE_EOI(x) ? "" : "not ");

	xive_config_fused_core(x);

	xive_config_esb_cache(x);

	xive_config_reduced_priorities_fixup(x);

	return true;
}

/* Set Translation tables : 1 block per chip */
static bool xive_setup_set_xlate(struct xive *x)
{
	unsigned int i;

	/* Configure ESBs */
	xive_regw(x, CQ_TAR,
		  CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_ESB));
	if (x->last_reg_error)
		return false;
	for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
		xive_regw(x, CQ_TDR, CQ_TDR_VALID |
			  SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
		if (x->last_reg_error)
			return false;
	}

	/* Configure ENDs */
	xive_regw(x, CQ_TAR,
		  CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_END));
	if (x->last_reg_error)
		return false;
	for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
		xive_regw(x, CQ_TDR, CQ_TDR_VALID |
			  SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
		if (x->last_reg_error)
			return false;
	}

	/* Configure NVPs */
	xive_regw(x, CQ_TAR,
		  CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_NVPG));
	if (x->last_reg_error)
		return false;
	for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
		xive_regw(x, CQ_TDR, CQ_TDR_VALID |
			  SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
		if (x->last_reg_error)
			return false;
	}
	return true;
}

static bool xive_prealloc_tables(struct xive *x)
{
	uint32_t i;
	uint32_t pbase, pend;

	/* ESB has 4 entries per byte */
	x->sbe_base = local_alloc(x->chip_id, XIVE_ESB_SIZE, XIVE_ESB_SIZE);
	if (!x->sbe_base) {
		xive_err(x, "Failed to allocate SBE\n");
		return false;
	}

	/* PQs are initialized to 0b01 which corresponds to "ints off" */
	memset(x->sbe_base, 0x55, XIVE_ESB_SIZE);
	xive_dbg(x, "SBE  at %p size 0x%lx\n", x->sbe_base, XIVE_ESB_SIZE);

	/* EAS entries are 8 bytes */
	x->eat_base = local_alloc(x->chip_id, XIVE_EAT_SIZE, XIVE_EAT_SIZE);
	if (!x->eat_base) {
		xive_err(x, "Failed to allocate EAS\n");
		return false;
	}

	/*
	 * We clear the entries (non-valid). They will be initialized
	 * when actually used
	 */
	memset(x->eat_base, 0, XIVE_EAT_SIZE);
	xive_dbg(x, "EAT  at %p size 0x%lx\n", x->eat_base, XIVE_EAT_SIZE);

	/* Indirect END table. Limited to one top page. */
	x->end_ind_size = ALIGN_UP(XIVE_END_TABLE_SIZE, PAGE_SIZE);
	if (x->end_ind_size > PAGE_SIZE) {
		xive_err(x, "END indirect table is too big !\n");
		return false;
	}
	x->end_ind_base = local_alloc(x->chip_id, x->end_ind_size,
				      x->end_ind_size);
	if (!x->end_ind_base) {
		xive_err(x, "Failed to allocate END indirect table\n");
		return false;
	}
	memset(x->end_ind_base, 0, x->end_ind_size);
	xive_dbg(x, "ENDi at %p size 0x%llx #%ld entries\n", x->end_ind_base,
		 x->end_ind_size, XIVE_END_COUNT);
	x->end_ind_count = XIVE_END_TABLE_SIZE / XIVE_VSD_SIZE;

	/* Indirect VP table. Limited to one top page. */
	x->vp_ind_size = ALIGN_UP(XIVE_VP_TABLE_SIZE(x), PAGE_SIZE);
	if (x->vp_ind_size > PAGE_SIZE) {
		xive_err(x, "VP indirect table is too big !\n");
		return false;
	}
	x->vp_ind_base = local_alloc(x->chip_id, x->vp_ind_size,
				     x->vp_ind_size);
	if (!x->vp_ind_base) {
		xive_err(x, "Failed to allocate VP indirect table\n");
		return false;
	}
	xive_dbg(x, "VPi  at %p size 0x%llx #%ld entries\n", x->vp_ind_base,
		 x->vp_ind_size, XIVE_VP_COUNT(x));
	x->vp_ind_count = XIVE_VP_TABLE_SIZE(x) / XIVE_VSD_SIZE;
	memset(x->vp_ind_base, 0, x->vp_ind_size);

	/* Allocate pages for the VP ids representing HW threads */
	pbase = xive_hw_vp_base / VP_PER_PAGE;
	pend  = (xive_hw_vp_base + xive_hw_vp_count) / VP_PER_PAGE;

	xive_dbg(x, "Allocating pages %d to %d of VPs (for %d VPs)\n",
		 pbase, pend, xive_hw_vp_count);
	for (i = pbase; i <= pend; i++) {
		void *page;
		u64 vsd;

		/* Indirect entries have a VSD format */
		page = local_alloc(x->chip_id, PAGE_SIZE, PAGE_SIZE);
		if (!page) {
			xive_err(x, "Failed to allocate VP page\n");
			return false;
		}
		xive_dbg(x, "VP%d at %p size 0x%x\n", i, page, PAGE_SIZE);
		memset(page, 0, PAGE_SIZE);
		vsd = ((uint64_t)page) & VSD_ADDRESS_MASK;

		vsd |= SETFIELD(VSD_TSIZE, 0ull, 4);
		vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
		vsd |= VSD_FIRMWARE;
		x->vp_ind_base[i] = cpu_to_be64(vsd);
	}

	/*
	 * Allocate page for cache and sync injection (512 * 128 hw
	 * threads) + one extra page for future use
	 */
	x->sync_inject_size = PAGE_SIZE + PAGE_SIZE;
	x->sync_inject = local_alloc(x->chip_id, x->sync_inject_size,
				     x->sync_inject_size);
	if (!x->sync_inject) {
		xive_err(x, "Failed to allocate sync pages\n");
		return false;
	}

	/*
	 * The Memory Coherence Directory uses 16M "granule" to track
	 * shared copies of a cache line. If any cache line within the
	 * 16M range gets touched by someone outside of the group, the
	 * MCD forces accesses to any cache line within the range to
	 * include everyone that might have a shared copy.
	 */
#define QUEUE_OVF_ALIGN (16 << 20) /* MCD granule size */

	/*
	 * Allocate the queue overflow pages and use a 16M alignment
	 * to avoid sharing with other structures and reduce traffic
	 * on the PowerBus.
	 */
	x->q_ovf = local_alloc(x->chip_id, VC_QUEUE_COUNT * PAGE_SIZE,
			       QUEUE_OVF_ALIGN);
	if (!x->q_ovf) {
		xive_err(x, "Failed to allocate queue overflow\n");
		return false;
	}
	return true;
}

static void xive_add_provisioning_properties(void)
{
	beint32_t chips[XIVE_MAX_CHIPS];
	uint32_t i, count;

	dt_add_property_cells(xive_dt_node,
			      "ibm,xive-provision-page-size", PAGE_SIZE);

	count = 1 << xive_chips_alloc_bits;
	for (i = 0; i < count; i++)
		chips[i] = cpu_to_be32(xive_block_to_chip[i]);
	dt_add_property(xive_dt_node, "ibm,xive-provision-chips",
			chips, 4 * count);
}

static void xive_create_mmio_dt_node(struct xive *x)
{
	uint64_t tb = (uint64_t)x->tm_base;
	uint32_t stride = 1u << x->tm_shift;

	xive_dt_node = dt_new_addr(dt_root, "interrupt-controller", tb);
	assert(xive_dt_node);

	dt_add_property_u64s(xive_dt_node, "reg",
			     tb + 0 * stride, stride,
			     tb + 1 * stride, stride,
			     tb + 2 * stride, stride,
			     tb + 3 * stride, stride);

	dt_add_property_strings(xive_dt_node, "compatible",
				"ibm,opal-xive-pe", "ibm,opal-xive-vc",
				"ibm,opal-intc");

	dt_add_property(xive_dt_node, "interrupt-controller", NULL, 0);
	dt_add_property_cells(xive_dt_node, "#address-cells", 0);
	dt_add_property_cells(xive_dt_node, "#interrupt-cells", 2);

	dt_add_property_cells(xive_dt_node, "ibm,xive-eq-sizes",
			      12, 16, 21, 24);

	dt_add_property_cells(xive_dt_node, "ibm,xive-#priorities",
			      xive_cfg_vp_prio(x));

	dt_add_property(xive_dt_node, "single-escalation-support", NULL, 0);

	if (XIVE_CAN_STORE_EOI(x))
		dt_add_property(xive_dt_node, "store-eoi", NULL, 0);

	if (xive_cfg_save_restore(x))
		dt_add_property(xive_dt_node, "vp-save-restore", NULL, 0);

	xive_add_provisioning_properties();

}

uint32_t xive2_get_phandle(void)
{
	if (!xive_dt_node)
		return 0;

	return xive_dt_node->phandle;
}

static void xive_setup_forward_ports(struct xive *x, struct proc_chip *remote_chip)
{
	struct xive *remote_xive = remote_chip->xive;
	uint64_t base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_FORWARD);

	if (!xive_set_vsd(x, VST_ESB, remote_xive->block_id,
			  base | ((uint64_t)remote_xive->esb_base) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->esb_size) - 12)))
		goto error;

	/* EAS: No remote */

	if (!xive_set_vsd(x, VST_END, remote_xive->block_id,
			  base | ((uint64_t)remote_xive->end_base) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->end_size) - 12)))
		goto error;

	if (!xive_set_vsd(x, VST_NVP, remote_xive->block_id,
			  base | ((uint64_t)remote_xive->nvp_base) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->nvp_size) - 12)))
		goto error;

	/* NVG: not used */
	/* NVC: not used */

	if (!xive_set_vsd(x, VST_IC, remote_xive->chip_id,
			  base | ((uint64_t)remote_xive->ic_base) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->ic_size) - 12)))
		goto error;

	if (!xive_set_vsd(x, VST_SYNC, remote_xive->chip_id,
			  base | ((uint64_t)remote_xive->sync_inject) |
			  SETFIELD(VSD_TSIZE, 0ull, ilog2(x->sync_inject_size) - 12)))
		goto error;

	/* ERQ: No remote */

	return;

 error:
	xive_err(x, "Failure configuring forwarding ports\n");
}

static void late_init_one_xive(struct xive *x)
{
	struct proc_chip *chip;

	/* We need to setup the cross-chip forward ports. Let's
	 * iterate all chip and set them up accordingly
	 */
	for_each_chip(chip) {
		/* We skip ourselves or chips without a xive */
		if (chip->xive == x || !chip->xive)
			continue;

		/* Setup our forward ports to that chip */
		xive_setup_forward_ports(x, chip);
	}
}

static bool xive_check_ipi_free(struct xive *x, uint32_t irq, uint32_t count)
{
	uint32_t i, idx = GIRQ_TO_IDX(irq);

	for (i = 0; i < count; i++)
		if (bitmap_tst_bit(*x->ipi_alloc_map, idx + i))
			return false;
	return true;
}

uint32_t xive2_alloc_hw_irqs(uint32_t chip_id, uint32_t count,
				      uint32_t align)
{
	struct proc_chip *chip = get_chip(chip_id);
	struct xive *x;
	uint32_t base, i;

	assert(chip);
	assert(is_pow2(align));

	x = chip->xive;
	assert(x);

	lock(&x->lock);

	/* Allocate the HW interrupts */
	base = x->int_hw_bot - count;
	base &= ~(align - 1);
	if (base < x->int_ipi_top) {
		xive_err(x,
			 "HW alloc request for %d interrupts aligned to %d failed\n",
			 count, align);
		unlock(&x->lock);
		return XIVE_IRQ_ERROR;
	}
	if (!xive_check_ipi_free(x, base, count)) {
		xive_err(x, "HWIRQ boot allocator request overlaps dynamic allocator\n");
		unlock(&x->lock);
		return XIVE_IRQ_ERROR;
	}

	x->int_hw_bot = base;

	/* Initialize the corresponding EAS entries to sane defaults,
	 * IE entry is valid, not routed and masked, EQ data is set
	 * to the GIRQ number.
	 */
	for (i = 0; i < count; i++) {
		struct xive_eas *eas = xive_get_eas(x, base + i);

		eas->w = xive_set_field64(EAS_VALID, 0, 1) |
			 xive_set_field64(EAS_MASKED, 0, 1) |
			 xive_set_field64(EAS_END_DATA, 0, base + i);
	}

	unlock(&x->lock);
	return base;
}

uint32_t xive2_alloc_ipi_irqs(uint32_t chip_id, uint32_t count,
				       uint32_t align)
{
	struct proc_chip *chip = get_chip(chip_id);
	struct xive *x;
	uint32_t base, i;

	assert(chip);
	assert(is_pow2(align));

	x = chip->xive;
	assert(x);

	lock(&x->lock);

	/* Allocate the IPI interrupts */
	base = x->int_ipi_top + (align - 1);
	base &= ~(align - 1);
	if (base >= x->int_hw_bot) {
		xive_err(x,
			 "IPI alloc request for %d interrupts aligned to %d failed\n",
			 count, align);
		unlock(&x->lock);
		return XIVE_IRQ_ERROR;
	}
	if (!xive_check_ipi_free(x, base, count)) {
		xive_err(x, "IPI boot allocator request overlaps dynamic allocator\n");
		unlock(&x->lock);
		return XIVE_IRQ_ERROR;
	}

	x->int_ipi_top = base + count;

	/* Initialize the corresponding EAS entries to sane defaults,
	 * IE entry is valid, not routed and masked, END data is set
	 * to the GIRQ number.
	 */
	for (i = 0; i < count; i++) {
		struct xive_eas *eas = xive_get_eas(x, base + i);

		eas->w = xive_set_field64(EAS_VALID, 0, 1) |
			 xive_set_field64(EAS_MASKED, 0, 1) |
			 xive_set_field64(EAS_END_DATA, 0, base + i);
	}

	unlock(&x->lock);
	return base;
}

void *xive2_get_trigger_port(uint32_t girq)
{
	uint32_t idx = GIRQ_TO_IDX(girq);
	struct xive *x;

	/* Find XIVE on which the EAS resides */
	x = xive_from_isn(girq);
	if (!x)
		return NULL;

	if (GIRQ_IS_ESCALATION(girq)) {
		/* There is no trigger page for escalation interrupts */
		return NULL;
	} else {
		/* Make sure it's an IPI on that chip */
		if (girq < x->int_base ||
		    girq >= x->int_ipi_top)
			return NULL;

		return x->esb_base + idx * XIVE_ESB_PAGE_SIZE;
	}
}

/*
 *  Notify Port page (writes only, w/data), separated into two
 *  categories, both sent to VC:
 *   - IPI queue (Addr bit 52 = 0) (for NPU)
 *   - HW queue (Addr bit 52 = 1)
 */
uint64_t xive2_get_notify_port(uint32_t chip_id, uint32_t ent)
{
	struct proc_chip *chip = get_chip(chip_id);
	struct xive *x;
	uint32_t offset = 0;

	assert(chip);
	x = chip->xive;
	assert(x);

	/* This is where we can assign a different HW queue to a different
	 * source by offsetting into the cache lines of the notify port
	 *
	 * For now we keep it very basic, this will have to be looked at
	 * again on real HW with some proper performance analysis.
	 *
	 * Here's what Florian says on the matter:
	 *
	 * <<
	 * The first 2k of the notify port page can all be used for PCIe triggers
	 *
	 * However the idea would be that we try to use the first 4 cache lines to
	 * balance the PCIe Interrupt requests to use the least used snoop buses
	 * (we went from 2 to 4 snoop buses for P9). snoop 0 is heavily used
	 * (I think TLBIs are using that in addition to the normal addresses),
	 * snoop 3 is used for all Int commands, so I think snoop 2 (CL 2 in the
	 * page) is the least used overall. So we probably should that one for
	 * the Int commands from PCIe.
	 *
	 * In addition, our EAS cache supports hashing to provide "private" cache
	 * areas for the PHBs in the shared 1k EAS cache. This allows e.g. to avoid
	 * that one "thrashing" PHB thrashes the EAS cache for everyone, or provide
	 * a PHB with a private area that would allow high cache hits in case of a
	 * device using very few interrupts. The hashing is based on the offset within
	 * the cache line. So using that, you can e.g. set the EAS cache up so that
	 * IPIs use 512 entries, the x16 PHB uses 256 entries and the x8 PHBs 128
	 * entries each - or IPIs using all entries and sharing with PHBs, so PHBs
	 * would use 512 entries and 256 entries respectively.
	 *
	 * This is a tuning we would probably do later in the lab, but as a "prep"
	 * we should set up the different PHBs such that they are using different
	 * 8B-aligned offsets within the cache line, so e.g.
	 * PH4_0  addr        0x100        (CL 2 DW0
	 * PH4_1  addr        0x108        (CL 2 DW1)
	 * PH4_2  addr        0x110        (CL 2 DW2)
	 * etc.
	 * >>
	 *
	 * I'm using snoop1 for PHB0 and snoop2 for everybody else.
	 */

	/* Florian adds :
	 *
	 * we just set them up for a start to have different offsets
	 * within the cache line so that we could use the allocation
	 * restrictions that can be enforced in the interrupt
	 * controller
	 *
	 * P10 might now be randomizing the cache line bits in HW to
	 * balance snoop bus usage
	 */
	switch(ent) {
	case XIVE_HW_SRC_PHBn(0):
		offset = 0x800;
		break;
	case XIVE_HW_SRC_PHBn(1):
		offset = 0x908;
		break;
	case XIVE_HW_SRC_PHBn(2):
		offset = 0x910;
		break;
	case XIVE_HW_SRC_PHBn(3):
		offset = 0x918;
		break;
	case XIVE_HW_SRC_PHBn(4):
		offset = 0x920;
		break;
	case XIVE_HW_SRC_PHBn(5):
		offset = 0x928;
		break;
	case XIVE_HW_SRC_PSI:
		offset = 0x930;
		break;
	default:
		assert(false);
		return 0;
	}

	return ((uint64_t)x->ic_base) +
		(XIVE_NOTIFY_PGOFF << x->ic_shift) + offset;
}

/* Manufacture the powerbus packet bits 32:63 */
__attrconst uint32_t xive2_get_notify_base(uint32_t girq)
{
	return (GIRQ_TO_BLK(girq) << 28)  | GIRQ_TO_IDX(girq);
}

static bool xive_get_irq_targetting(uint32_t isn, uint32_t *out_target,
				    uint8_t *out_prio, uint32_t *out_lirq)
{
	struct xive_eas *eas;
	struct xive *x, *end_x;
	struct xive_end *end;
	uint32_t end_blk, end_idx;
	uint32_t vp_blk, vp_idx;
	uint32_t prio, server;
	bool is_escalation = GIRQ_IS_ESCALATION(isn);

	/* Find XIVE on which the EAS resides */
	x = xive_from_isn(isn);
	if (!x)
		return false;
	/* Grab the EAS */
	eas = xive_get_eas(x, isn);
	if (!eas)
		return false;
	if (!xive_get_field64(EAS_VALID, eas->w) && !is_escalation) {
		xive_err(x, "ISN %x lead to invalid EAS !\n", isn);
		return false;
	}

	if (out_lirq)
		*out_lirq = xive_get_field64(EAS_END_DATA, eas->w);

	/* Find the END and its xive instance */
	end_blk = xive_get_field64(EAS_END_BLOCK, eas->w);
	end_idx = xive_get_field64(EAS_END_INDEX, eas->w);
	end_x = xive_from_vc_blk(end_blk);

	/* This can fail if the interrupt hasn't been initialized yet
	 * but it should also be masked, so fail silently
	 */
	if (!end_x)
		goto pick_default;
	end = xive_get_end(end_x, end_idx);
	if (!end)
		goto pick_default;

	/* XXX Check valid and format 0 */

	/* No priority conversion, return the actual one ! */
	if (xive_get_field64(EAS_MASKED, eas->w))
		prio = 0xff;
	else
		prio = xive_get_field32(END_W7_F0_PRIORITY, end->w7);
	if (out_prio)
		*out_prio = prio;

	vp_blk = xive_get_field32(END_W6_VP_BLOCK, end->w6);
	vp_idx = xive_get_field32(END_W6_VP_OFFSET, end->w6);
	server = VP2PIR(vp_blk, vp_idx);

	if (out_target)
		*out_target = server;

	xive_vdbg(end_x, "END info for ISN %x: prio=%d, server=0x%x (VP %x/%x)\n",
		  isn, prio, server, vp_blk, vp_idx);
	return true;

pick_default:
	xive_vdbg(end_x, "END info for ISN %x: Using masked defaults\n", isn);

	if (out_prio)
		*out_prio = 0xff;
	/* Pick a random default, me will be fine ... */
	if (out_target)
		*out_target = mfspr(SPR_PIR);
	return true;
}

static inline bool xive_end_for_target(uint32_t target, uint8_t prio,
				      uint32_t *out_end_blk,
				      uint32_t *out_end_idx)
{
	struct xive *x;
	struct xive_nvp *vp;
	uint32_t vp_blk, vp_idx;
	uint32_t end_blk, end_idx;

	if (prio > xive_max_prio(one_xive))
		return false;

	/* Get the VP block/index from the target word */
	if (!xive_decode_vp(target, &vp_blk, &vp_idx, NULL, NULL))
		return false;

	/* Grab the target VP's XIVE */
	x = xive_from_pc_blk(vp_blk);
	if (!x)
		return false;

	/* Find the VP structrure where we stashed the END number */
	vp = xive_get_vp(x, vp_idx);
	if (!vp)
		return false;

	end_blk = xive_get_field32(NVP_W5_VP_END_BLOCK, vp->w5);
	end_idx = xive_get_field32(NVP_W5_VP_END_INDEX, vp->w5);

	/* Currently the END block and VP block should be the same */
	if (end_blk != vp_blk) {
		xive_err(x, "end_blk != vp_blk (%d vs. %d) for target 0x%08x/%d\n",
			 end_blk, vp_blk, target, prio);
		assert(false);
	}

	if (out_end_blk)
		*out_end_blk = end_blk;
	if (out_end_idx)
		*out_end_idx = end_idx + prio;

	return true;
}

static int64_t xive_set_irq_targetting(uint32_t isn, uint32_t target,
				       uint8_t prio, uint32_t lirq,
				       bool synchronous)
{
	struct xive *x;
	struct xive_eas *eas, new_eas;
	uint32_t end_blk, end_idx;
	bool is_escalation = GIRQ_IS_ESCALATION(isn);
	int64_t rc;

	/* Find XIVE on which the EAS resides */
	x = xive_from_isn(isn);
	if (!x)
		return OPAL_PARAMETER;
	/* Grab the EAS */
	eas = xive_get_eas(x, isn);
	if (!eas)
		return OPAL_PARAMETER;
	if (!xive_get_field64(EAS_VALID, eas->w) && !is_escalation) {
		xive_err(x, "ISN %x lead to invalid EAS !\n", isn);
		return OPAL_PARAMETER;
	}

	lock(&x->lock);

	/* Read existing EAS */
	new_eas = *eas;

	/* Are we masking ? */
	if (prio == 0xff && !is_escalation) {
		new_eas.w = xive_set_field64(EAS_MASKED, new_eas.w, 1);
		xive_vdbg(x, "ISN %x masked !\n", isn);

		/* Put prio 7 in the END */
		prio = xive_max_prio(x);
	} else {
		/* Unmasking */
		new_eas.w = xive_set_field64(EAS_MASKED, new_eas.w, 0);
		xive_vdbg(x, "ISN %x unmasked !\n", isn);

		/* For normal interrupt sources, keep track of which ones
		 * we ever enabled since the last reset
		 */
		if (!is_escalation)
			bitmap_set_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn));
	}

	/* If prio isn't 0xff, re-target the EAS. First find the END
	 * correponding to the target
	 */
	if (prio != 0xff) {
		if (!xive_end_for_target(target, prio, &end_blk, &end_idx)) {
			xive_err(x, "Can't find END for target/prio 0x%x/%d\n",
				 target, prio);
			unlock(&x->lock);
			return OPAL_PARAMETER;
		}

		/* Try to update it atomically to avoid an intermediary
		 * stale state
		 */
		new_eas.w = xive_set_field64(EAS_END_BLOCK, new_eas.w, end_blk);
		new_eas.w = xive_set_field64(EAS_END_INDEX, new_eas.w, end_idx);
	}
	new_eas.w = xive_set_field64(EAS_END_DATA, new_eas.w, lirq);

	xive_vdbg(x,"ISN %x routed to end %x/%x lirq=%08x EAS=%016llx !\n",
		  isn, end_blk, end_idx, lirq, new_eas.w);

	/* Updating the cache differs between real EAS and escalation
	 * EAS inside an END
	 */
	if (is_escalation) {
		rc = xive_escalation_ive_cache_update(x, x->block_id,
				GIRQ_TO_IDX(isn), &new_eas, synchronous);
	} else {
		sync();
		*eas = new_eas;
		rc = xive_easc_scrub(x, x->block_id, GIRQ_TO_IDX(isn));
	}

	unlock(&x->lock);
	return rc;
}

static void xive_update_irq_mask(struct xive_src *s, uint32_t idx, bool masked)
{
	void *mmio_base = s->esb_mmio + (1ul << s->esb_shift) * idx;
	uint32_t offset;

	/* XXX FIXME: A quick mask/umask can make us shoot an interrupt
	 * more than once to a queue. We need to keep track better
	 */
	if (s->flags & XIVE_SRC_EOI_PAGE1)
		mmio_base += 1ull << (s->esb_shift - 1);
	if (masked)
		offset = XIVE_ESB_SET_PQ_01;
	else
		offset = XIVE_ESB_SET_PQ_00;

	in_be64(mmio_base + offset);
}

#define XIVE_SYNC_IPI      0x000
#define XIVE_SYNC_HW       0x080
#define XIVE_SYNC_NxC      0x100
#define XIVE_SYNC_INT      0x180
#define XIVE_SYNC_OS_ESC   0x200
#define XIVE_SYNC_POOL_ESC 0x280
#define XIVE_SYNC_HARD_ESC 0x300

static int64_t xive_sync(struct xive *x __unused)
{
	uint64_t r;
	void *sync_base;

	lock(&x->lock);

	sync_base = x->ic_base + (XIVE_SYNC_POLL_PGOFF << x->ic_shift);

	out_be64(sync_base + XIVE_SYNC_IPI, 0);
	out_be64(sync_base + XIVE_SYNC_HW, 0);
	out_be64(sync_base + XIVE_SYNC_NxC, 0);
	out_be64(sync_base + XIVE_SYNC_INT, 0);
	out_be64(sync_base + XIVE_SYNC_OS_ESC, 0);
	out_be64(sync_base + XIVE_SYNC_POOL_ESC, 0);
	out_be64(sync_base + XIVE_SYNC_HARD_ESC, 0);

	/* XXX Add timeout */
	for (;;) {
		r = xive_regr(x, VC_ENDC_SYNC_DONE);
		if ((r & VC_ENDC_SYNC_POLL_DONE) == VC_ENDC_SYNC_POLL_DONE)
			break;
		cpu_relax();
	}
	xive_regw(x, VC_ENDC_SYNC_DONE, r & ~VC_ENDC_SYNC_POLL_DONE);

	/*
	 * Do a read after clearing the sync done bit to prevent any
	 * race between CI write and next sync command
	 */
	xive_regr(x, VC_ENDC_SYNC_DONE);

	unlock(&x->lock);
	return 0;
}

static int64_t __xive_set_irq_config(struct irq_source *is, uint32_t girq,
				     uint64_t vp, uint8_t prio, uint32_t lirq,
				     bool update_esb, bool sync)
{
	struct xive_src *s = container_of(is, struct xive_src, is);
	uint32_t old_target, vp_blk;
	u8 old_prio;
	int64_t rc;

	/* Grab existing target */
	if (!xive_get_irq_targetting(girq, &old_target, &old_prio, NULL))
		return OPAL_PARAMETER;

	/* Let XIVE configure the END. We do the update without the
	 * synchronous flag, thus a cache update failure will result
	 * in us returning OPAL_BUSY
	 */
	rc = xive_set_irq_targetting(girq, vp, prio, lirq, false);
	if (rc)
		return rc;

	/* Do we need to update the mask ? */
	if (old_prio != prio && (old_prio == 0xff || prio == 0xff)) {
		/* The source has special variants of masking/unmasking */
		if (update_esb) {
			/* Ensure it's enabled/disabled in the source
			 * controller
			 */
			xive_update_irq_mask(s, girq - s->esb_base,
					     prio == 0xff);
		}
	}

	/*
	 * Synchronize the source and old target XIVEs to ensure that
	 * all pending interrupts to the old target have reached their
	 * respective queue.
	 *
	 * WARNING: This assumes the VP and it's queues are on the same
	 *          XIVE instance !
	 */
	if (!sync)
		return OPAL_SUCCESS;
	xive_sync(s->xive);
	if (xive_decode_vp(old_target, &vp_blk, NULL, NULL, NULL)) {
		struct xive *x = xive_from_pc_blk(vp_blk);
		if (x)
			xive_sync(x);
	}

	return OPAL_SUCCESS;
}

static int64_t xive_set_irq_config(uint32_t girq, uint64_t vp, uint8_t prio,
				   uint32_t lirq, bool update_esb)
{
	struct irq_source *is = irq_find_source(girq);

	return __xive_set_irq_config(is, girq, vp, prio, lirq, update_esb,
				     true);
}

static void xive_source_interrupt(struct irq_source *is, uint32_t isn)
{
	struct xive_src *s = container_of(is, struct xive_src, is);

	if (!s->orig_ops || !s->orig_ops->interrupt)
		return;
	s->orig_ops->interrupt(is, isn);
}

static uint64_t xive_source_attributes(struct irq_source *is, uint32_t isn)
{
	struct xive_src *s = container_of(is, struct xive_src, is);

	if (!s->orig_ops || !s->orig_ops->attributes)
		return IRQ_ATTR_TARGET_LINUX;
	return s->orig_ops->attributes(is, isn);
}

static char *xive_source_name(struct irq_source *is, uint32_t isn)
{
	struct xive_src *s = container_of(is, struct xive_src, is);

	if (!s->orig_ops || !s->orig_ops->name)
		return NULL;
	return s->orig_ops->name(is, isn);
}

void xive2_source_mask(struct irq_source *is, uint32_t isn)
{
	struct xive_src *s = container_of(is, struct xive_src, is);

	xive_update_irq_mask(s, isn - s->esb_base, true);
}

static bool xive_has_opal_interrupts(struct irq_source *is)
{
	struct xive_src *s = container_of(is, struct xive_src, is);

	if (!s->orig_ops || !s->orig_ops->attributes || !s->orig_ops->interrupt)
		return false;
	return true;
}

static const struct irq_source_ops xive_irq_source_ops = {
	.interrupt = xive_source_interrupt,
	.attributes = xive_source_attributes,
	.name = xive_source_name,
	.has_opal_interrupts = xive_has_opal_interrupts,
};

static void __xive_register_source(struct xive *x, struct xive_src *s,
				   uint32_t base, uint32_t count,
				   uint32_t shift, void *mmio, uint32_t flags,
				   bool secondary, void *data,
				   const struct irq_source_ops *orig_ops)
{
	s->esb_base = base;
	s->esb_shift = shift;
	s->esb_mmio = mmio;
	s->flags = flags;
	s->orig_ops = orig_ops;
	s->xive = x;
	s->is.start = base;
	s->is.end = base + count;
	s->is.ops = &xive_irq_source_ops;
	s->is.data = data;

	__register_irq_source(&s->is, secondary);
}

void xive2_register_hw_source(uint32_t base, uint32_t count, uint32_t shift,
			     void *mmio, uint32_t flags, void *data,
			     const struct irq_source_ops *ops)
{
	struct xive_src *s;
	struct xive *x = xive_from_isn(base);

	assert(x);

	s = malloc(sizeof(struct xive_src));
	assert(s);
	__xive_register_source(x, s, base, count, shift, mmio, flags,
			       false, data, ops);
}

static void __xive2_register_esb_source(uint32_t base, uint32_t count,
				void *data, const struct irq_source_ops *ops)
{
	struct xive_src *s;
	struct xive *x = xive_from_isn(base);
	uint32_t base_idx = GIRQ_TO_IDX(base);
	void *mmio_base;
	uint32_t flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;

	assert(x);

	s = malloc(sizeof(struct xive_src));
	assert(s);

	if (XIVE_CAN_STORE_EOI(x))
		flags |= XIVE_SRC_STORE_EOI;

	/* Callbacks assume the MMIO base corresponds to the first
	 * interrupt of that source structure so adjust it
	 */
	mmio_base = x->esb_base + (1ul << XIVE_ESB_SHIFT) * base_idx;
	__xive_register_source(x, s, base, count, XIVE_ESB_SHIFT, mmio_base,
			       flags, false, data, ops);
}

/*
 * Check that IPI sources have interrupt numbers in the IPI interrupt
 * number range
 */
void xive2_register_ipi_source(uint32_t base, uint32_t count, void *data,
			       const struct irq_source_ops *ops)
{
	struct xive *x = xive_from_isn(base);

	assert(x);
	assert(base >= x->int_base && (base + count) <= x->int_ipi_top);

	__xive2_register_esb_source(base, count, data, ops);
}

/*
 * Some HW sources (PHB) can disable the use of their own ESB pages
 * and offload all the checks on ESB pages of the IC. The interrupt
 * numbers are not necessarily in the IPI range.
 */
void xive2_register_esb_source(uint32_t base, uint32_t count)
{
	__xive2_register_esb_source(base, count, NULL, NULL);
}

uint64_t xive2_get_esb_base(uint32_t base)
{
	struct xive *x = xive_from_isn(base);
	uint32_t base_idx = GIRQ_TO_IDX(base);

	assert(x);

	return (uint64_t) x->esb_base + (1ul << XIVE_ESB_SHIFT) * base_idx;
}

static void xive_set_quirks(struct xive *x, struct proc_chip *chip __unused)
{
	uint64_t quirks = 0;

	/* This extension is dropped for P10 */
	if (proc_gen == proc_gen_p10)
		quirks |= XIVE_QUIRK_THREADID_7BITS;

	/* Broken check on invalid priority when reduced priorities is in use */
	if (proc_gen == proc_gen_p10)
		quirks |= XIVE_QUIRK_BROKEN_PRIO_CHECK;

	xive_dbg(x, "setting XIVE quirks to %016llx\n", quirks);
	x->quirks = quirks;
}

static struct xive *init_one_xive(struct dt_node *np)
{
	struct xive *x;
	struct proc_chip *chip;
	uint32_t flags;

	x = zalloc(sizeof(struct xive));
	assert(x);
	x->x_node = np;
	x->xscom_base = dt_get_address(np, 0, NULL);
	x->chip_id = dt_get_chip_id(np);

	/* "Allocate" a new block ID for the chip */
	x->block_id = xive_block_count++;
	assert (x->block_id < XIVE_MAX_CHIPS);
	xive_block_to_chip[x->block_id] = x->chip_id;
	init_lock(&x->lock);

	chip = get_chip(x->chip_id);
	assert(chip);

	xive_notice(x, "Initializing XIVE block ID %d...\n", x->block_id);
	chip->xive = x;

	xive_set_quirks(x, chip);

	list_head_init(&x->donated_pages);

	/* Base interrupt numbers and allocator init */

	x->int_base	= BLKIDX_TO_GIRQ(x->block_id, 0);
	x->int_count	= x->int_base + XIVE_INT_COUNT;
	x->int_hw_bot	= x->int_count;
	x->int_ipi_top	= x->int_base;

	if (x->int_ipi_top < XIVE_INT_FIRST)
		x->int_ipi_top = XIVE_INT_FIRST;

	/* Allocate a few bitmaps */
	x->end_map = local_alloc(x->chip_id, BITMAP_BYTES(xive_end_bitmap_size(x)), PAGE_SIZE);
	assert(x->end_map);
	memset(x->end_map, 0, BITMAP_BYTES(xive_end_bitmap_size(x)));

	/*
	 * Allocate END index 0 to make sure it can not be used as an
	 * END base for a VP. This is the criteria to know if a VP was
	 * allocated.
	 */
	bitmap_set_bit(*x->end_map, 0);

	x->int_enabled_map = local_alloc(x->chip_id, BITMAP_BYTES(XIVE_INT_COUNT), PAGE_SIZE);
	assert(x->int_enabled_map);
	memset(x->int_enabled_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));
	x->ipi_alloc_map = local_alloc(x->chip_id, BITMAP_BYTES(XIVE_INT_COUNT), PAGE_SIZE);
	assert(x->ipi_alloc_map);
	memset(x->ipi_alloc_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));

	xive_dbg(x, "Handling interrupts [%08x..%08x]\n",
		 x->int_base, x->int_count - 1);

	/* Setup the IC BARs */
	if (!xive_configure_ic_bars(x))
		goto fail;

	/* Some basic global inits such as page sizes etc... */
	if (!xive_config_init(x))
		goto fail;

	/* Configure the set translations for MMIO */
	if (!xive_setup_set_xlate(x))
		goto fail;

	/* Dump some MMIO registers for diagnostics */
	xive_dump_mmio(x);

	/* Pre-allocate a number of tables */
	if (!xive_prealloc_tables(x))
		goto fail;

	/* Setup the XIVE structures BARs */
	if (!xive_configure_bars(x))
		goto fail;

	/*
	 * Configure local tables in VSDs (forward ports will be
	 * handled later)
	 */
	if (!xive_set_local_tables(x))
		goto fail;

	/* Register built-in source controllers (aka IPIs) */
	flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;
	if (XIVE_CAN_STORE_EOI(x))
		flags |= XIVE_SRC_STORE_EOI;
	__xive_register_source(x, &x->ipis, x->int_base,
			       x->int_hw_bot - x->int_base, XIVE_ESB_SHIFT,
			       x->esb_base, flags, true, NULL, NULL);

	/* Register escalation sources (ENDs)
	 *
	 * The ESe PQ bits are used for coalescing and the END ESB for
	 * interrupt management. The word 4&5 of the END is the EAS
	 * for the escalation source and the indexing is the same as
	 * the END.
	 *
	 * This is an OPAL primary source, IPIs are secondary.
	 */
	__xive_register_source(x, &x->esc_irqs,
			       MAKE_ESCALATION_GIRQ(x->block_id, 0),
			       XIVE_END_COUNT, XIVE_END_SHIFT,
			       x->end_base, XIVE_SRC_EOI_PAGE1,
			       false, NULL, NULL);


	return x;
 fail:
	xive_err(x, "Initialization failed...\n");

	/* Should this be fatal ? */
	//assert(false);
	return NULL;
}

static void xive_reset_enable_thread(struct cpu_thread *c)
{
	struct proc_chip *chip = get_chip(c->chip_id);
	struct xive *x = chip->xive;
	uint32_t fc, bit;
	uint64_t enable;

	/* Get fused core number */
	fc = (c->pir >> 3) & 0xf;

	/* Get bit in register */
	bit = c->pir & 0x3f;

	/* Get which register to access */
	if (fc < 8) {
		xive_regw(x, TCTXT_EN0_RESET, PPC_BIT(bit));
		xive_regw(x, TCTXT_EN0_SET, PPC_BIT(bit));

		enable = xive_regr(x, TCTXT_EN0);
		if (!(enable & PPC_BIT(bit)))
			xive_cpu_err(c, "Failed to enable thread\n");
	} else {
		xive_regw(x, TCTXT_EN1_RESET, PPC_BIT(bit));
		xive_regw(x, TCTXT_EN1_SET, PPC_BIT(bit));

		enable = xive_regr(x, TCTXT_EN1);
		if (!(enable & PPC_BIT(bit)))
			xive_cpu_err(c, "Failed to enable thread\n");
	}
}

void xive2_cpu_callin(struct cpu_thread *cpu)
{
	struct xive_cpu_state *xs = cpu->xstate;
	uint8_t old_w2 __unused, w2 __unused;

	if (!xs)
		return;

	/* Reset the HW thread context and enable it */
	xive_reset_enable_thread(cpu);

	/* Set VT to 1 */
	old_w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2);
	out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2, 0x80);
	w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2);

	xive_cpu_vdbg(cpu, "Initialized TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n",
		      xs->vp_blk, xs->vp_idx,
		      in_be64(xs->tm_ring1 + TM_QW3_HV_PHYS),
		      old_w2, w2);
}

#ifdef XIVE_EXTRA_CHECK_INIT_CACHE
#define CHECK_INIT_CACHE_LOOP 0x100
static void xive_special_cache_check(struct xive *x, uint32_t blk, uint32_t idx)
{
	struct xive_nvp vp = {0};
	uint32_t i;

	/*
	 * SIMICS checks the value of reserved fields
	 */
	if (chip_quirk(QUIRK_SIMICS))
		return;

	for (i = 0; i < CHECK_INIT_CACHE_LOOP; i++) {
		struct xive_nvp *vp_m = xive_get_vp(x, idx);

		memset(vp_m, (~i) & 0xff, sizeof(*vp_m));
		vp_m->w0 = xive_set_field32(NVP_W0_VALID, vp_m->w0, 0);
		sync();
		vp.w1 = (i << 16) | i;
		assert(!xive_nxc_cache_update(x, blk, idx, &vp, true));
		if (!xive_check_nxc_update(x, idx, &vp)) {
			xive_dbg(x, "NXC update test failed at %d iterations\n", i);
			return;
		}
	}
	xive_dbg(x, "NXC update test passed for %d/0x%x\n", blk, idx);
}
#else
static inline void xive_special_cache_check(struct xive *x __unused,
					    uint32_t blk __unused,
					    uint32_t idx __unused)
{
}
#endif

static void xive_init_cpu_exploitation(struct xive_cpu_state *xs)
{
	struct xive_end end;
	struct xive_nvp vp;
	struct xive *x_vp, *x_end;
	int i;

	/* Grab the XIVE where the VP resides. It could be different from
	 * the local chip XIVE if not using block group mode
	 */
	x_vp = xive_from_pc_blk(xs->vp_blk);
	assert(x_vp);

	/* Grab the XIVE where the END resides. It should be the same
	 * as the VP.
	 */
	x_end = xive_from_vc_blk(xs->end_blk);
	assert(x_end);

	xive_init_hw_end(&end);

	/* Use the cache watch to update all ENDs reserved for HW VPs */
	lock(&x_end->lock);
	for (i = 0; i < xive_cfg_vp_prio(x_end); i++)
		xive_endc_cache_update(x_end, xs->end_blk, xs->end_idx + i,
				       &end, true);
	unlock(&x_end->lock);

	/* Initialize/enable the VP */
	xive_init_default_vp(&vp, xs->end_blk, xs->end_idx);

	/* Use the cache watch to write it out */
	lock(&x_vp->lock);
	xive_special_cache_check(x_vp, xs->vp_blk, xs->vp_idx);
	xive_nxc_cache_update(x_vp, xs->vp_blk, xs->vp_idx, &vp, true);
	unlock(&x_vp->lock);
}

static void xive_configure_ex_special_bar(struct xive *x, struct cpu_thread *c)
{
	uint64_t xa, val;
	int64_t rc;

	xive_cpu_vdbg(c, "Setting up special BAR\n");
	xa = XSCOM_ADDR_P10_NCU(pir_to_core_id(c->pir), P10_NCU_SPEC_BAR);
	val = (uint64_t)x->tm_base | P10_NCU_SPEC_BAR_ENABLE;
	if (x->tm_shift == 16)
		val |= P10_NCU_SPEC_BAR_256K;
	xive_cpu_vdbg(c, "NCU_SPEC_BAR_XA[%08llx]=%016llx\n", xa, val);
	rc = xscom_write(c->chip_id, xa, val);
	if (rc) {
		xive_cpu_err(c, "Failed to setup NCU_SPEC_BAR\n");
		/* XXXX  what do do now ? */
	}
}

void xive2_late_init(void)
{
	struct cpu_thread *c;

	prlog(PR_INFO, "SLW: Configuring self-restore for NCU_SPEC_BAR\n");
	for_each_present_cpu(c) {
		if(cpu_is_thread0(c)) {
			struct proc_chip *chip = get_chip(c->chip_id);
			struct xive *x = chip->xive;
			uint64_t xa, val, rc;
			xa = XSCOM_ADDR_P10_NCU(pir_to_core_id(c->pir), P10_NCU_SPEC_BAR);
			val = (uint64_t)x->tm_base | P10_NCU_SPEC_BAR_ENABLE;
			/* Bail out if wakeup engine has already failed */
			if (wakeup_engine_state != WAKEUP_ENGINE_PRESENT) {
				prlog(PR_ERR, "XIVE proc_stop_api fail detected\n");
				break;
			}
			rc = proc_stop_save_scom((void *)chip->homer_base, xa, val,
				PROC_STOP_SCOM_REPLACE, PROC_STOP_SECTION_L3);
			if (rc) {
				xive_cpu_err(c, "proc_stop_save_scom failed for NCU_SPEC_BAR rc=%lld\n",
					     rc);
				wakeup_engine_state = WAKEUP_ENGINE_FAILED;
			}
		}
	}
}

static void xive_provision_cpu(struct xive_cpu_state *xs, struct cpu_thread *c)
{
	struct xive *x;

	/* VP ids for HW threads are pre-allocated */
	xs->vp_blk = PIR2VP_BLK(c->pir);
	xs->vp_idx = PIR2VP_IDX(c->pir);

	/* For now we use identical block IDs for VC and PC but that might
	 * change. We allocate the ENDs on the same XIVE as the VP.
	 */
	xs->end_blk = xs->vp_blk;

	/* Grab the XIVE where the END resides. It could be different from
	 * the local chip XIVE if not using block group mode
	 */
	x = xive_from_vc_blk(xs->end_blk);
	assert(x);

	/* Allocate a set of ENDs for that VP */
	xs->end_idx = xive_alloc_end_set(x, true);
	assert(!XIVE_ALLOC_IS_ERR(xs->end_idx));
}

static void xive_init_cpu(struct cpu_thread *c)
{
	struct proc_chip *chip = get_chip(c->chip_id);
	struct xive *x = chip->xive;
	struct xive_cpu_state *xs;

	if (!x)
		return;

	/*
	 * Each core pair (EX) needs this special BAR setup to have the
	 * right powerbus cycle for the TM area (as it has the same address
	 * on all chips so it's somewhat special).
	 *
	 * Because we don't want to bother trying to figure out which core
	 * of a pair is present we just do the setup for each of them, which
	 * is harmless.
	 */
	if (cpu_is_thread0(c) || cpu_is_core_chiplet_primary(c))
		xive_configure_ex_special_bar(x, c);

	/* Initialize the state structure */
	c->xstate = xs = local_alloc(c->chip_id, sizeof(struct xive_cpu_state), 1);
	assert(xs);
	memset(xs, 0, sizeof(struct xive_cpu_state));
	xs->xive = x;

	init_lock(&xs->lock);

	/* Shortcut to TM HV ring */
	xs->tm_ring1 = x->tm_base + (1u << x->tm_shift);

	/* Provision a VP id and some ENDs for a HW thread */
	xive_provision_cpu(xs, c);

	xive_init_cpu_exploitation(xs);
}

static uint64_t xive_convert_irq_flags(uint64_t iflags)
{
	uint64_t oflags = 0;

	if (iflags & XIVE_SRC_STORE_EOI)
		oflags |= OPAL_XIVE_IRQ_STORE_EOI2;

	/* OPAL_XIVE_IRQ_TRIGGER_PAGE is only meant to be set if
	 * the interrupt has a *separate* trigger page.
	 */
	if ((iflags & XIVE_SRC_EOI_PAGE1) &&
	    (iflags & XIVE_SRC_TRIGGER_PAGE))
		oflags |= OPAL_XIVE_IRQ_TRIGGER_PAGE;

	if (iflags & XIVE_SRC_LSI)
		oflags |= OPAL_XIVE_IRQ_LSI;

	return oflags;
}

static int64_t opal_xive_get_irq_info(uint32_t girq,
				      beint64_t *out_flags,
				      beint64_t *out_eoi_page,
				      beint64_t *out_trig_page,
				      beint32_t *out_esb_shift,
				      beint32_t *out_src_chip)
{
	struct irq_source *is = irq_find_source(girq);
	struct xive_src *s = container_of(is, struct xive_src, is);
	uint32_t idx;
	uint64_t mm_base;
	uint64_t eoi_page = 0, trig_page = 0;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;
	if (is == NULL || out_flags == NULL)
		return OPAL_PARAMETER;
	assert(is->ops == &xive_irq_source_ops);

	if (out_flags)
		*out_flags = cpu_to_be64(xive_convert_irq_flags(s->flags));

	idx = girq - s->esb_base;

	if (out_esb_shift)
		*out_esb_shift = cpu_to_be32(s->esb_shift);

	mm_base = (uint64_t)s->esb_mmio + (1ull << s->esb_shift) * idx;

	/* The EOI page can either be the first or second page */
	if (s->flags & XIVE_SRC_EOI_PAGE1) {
		uint64_t p1off = 1ull << (s->esb_shift - 1);
		eoi_page = mm_base + p1off;
	} else
		eoi_page = mm_base;

	/* The trigger page, if it exists, is always the first page */
	if (s->flags & XIVE_SRC_TRIGGER_PAGE)
		trig_page = mm_base;

	if (out_eoi_page)
		*out_eoi_page = cpu_to_be64(eoi_page);
	if (out_trig_page)
		*out_trig_page = cpu_to_be64(trig_page);
	if (out_src_chip)
		*out_src_chip = cpu_to_be32(GIRQ_TO_CHIP(girq));

	return OPAL_SUCCESS;
}

static int64_t opal_xive_get_irq_config(uint32_t girq,
					beint64_t *out_vp,
					uint8_t *out_prio,
					beint32_t *out_lirq)
{
	uint32_t vp;
	uint32_t lirq;
	uint8_t prio;

	if (xive_mode != XIVE_MODE_EXPL)
               return OPAL_WRONG_STATE;

	if (xive_get_irq_targetting(girq, &vp, &prio, &lirq)) {
		*out_vp = cpu_to_be64(vp);
		*out_prio = prio;
		*out_lirq = cpu_to_be32(lirq);
		return OPAL_SUCCESS;
	} else
		return OPAL_PARAMETER;
}

static int64_t opal_xive_set_irq_config(uint32_t girq,
					uint64_t vp,
					uint8_t prio,
					uint32_t lirq)
{
	/*
	 * This variant is meant for a XIVE-aware OS, thus it will
	 * *not* affect the ESB state of the interrupt. If used with
	 * a prio of FF, the EAS will be masked. In that case the
	 * races have to be handled by the OS.
	 */
	if (xive_mode != XIVE_MODE_EXPL)
               return OPAL_WRONG_STATE;

	return xive_set_irq_config(girq, vp, prio, lirq, false);
}

static int64_t opal_xive_get_queue_info(uint64_t vp, uint32_t prio,
					beint64_t *out_qpage,
					beint64_t *out_qsize,
					beint64_t *out_qeoi_page,
					beint32_t *out_escalate_irq,
					beint64_t *out_qflags)
{
	uint32_t blk, idx;
	struct xive *x;
	struct xive_end *end;

	if (xive_mode != XIVE_MODE_EXPL)
               return OPAL_WRONG_STATE;

	if (!xive_end_for_target(vp, prio, &blk, &idx))
		return OPAL_PARAMETER;

	x = xive_from_vc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;

	end = xive_get_end(x, idx);
	if (!end)
		return OPAL_PARAMETER;

	if (out_escalate_irq) {
		uint32_t esc_idx = idx;

		/* If escalations are routed to a single queue, fix up
		 * the escalation interrupt number here.
		 */
		if (xive_get_field32(END_W0_UNCOND_ESCALATE, end->w0))
			esc_idx |= xive_escalation_prio(x);
		*out_escalate_irq =
			cpu_to_be32(MAKE_ESCALATION_GIRQ(blk, esc_idx));
	}

	/* If this is a single-escalation gather queue, that's all
	 * there is to return
	 */
	if (xive_get_field32(END_W0_SILENT_ESCALATE, end->w0)) {
		if (out_qflags)
			*out_qflags = 0;
		if (out_qpage)
			*out_qpage = 0;
		if (out_qsize)
			*out_qsize = 0;
		if (out_qeoi_page)
			*out_qeoi_page = 0;
		return OPAL_SUCCESS;
	}

	if (out_qpage) {
		if (xive_get_field32(END_W0_ENQUEUE, end->w0))
			*out_qpage = cpu_to_be64(
				((uint64_t)xive_get_field32(END_W2_EQ_ADDR_HI, end->w2) << 32) |
				xive_get_field32(END_W3_EQ_ADDR_LO, end->w3));
		else
			*out_qpage = 0;
	}
	if (out_qsize) {
		if (xive_get_field32(END_W0_ENQUEUE, end->w0))
			*out_qsize = cpu_to_be64(xive_get_field32(END_W3_QSIZE, end->w3) + 12);
		else
			*out_qsize = 0;
	}
	if (out_qeoi_page) {
		*out_qeoi_page = cpu_to_be64(
			(uint64_t)x->end_base + idx * XIVE_ESB_PAGE_SIZE);
	}
	if (out_qflags) {
		*out_qflags = 0;
		if (xive_get_field32(END_W0_VALID, end->w0))
			*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ENABLED);
		if (xive_get_field32(END_W0_UCOND_NOTIFY, end->w0))
			*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ALWAYS_NOTIFY);
		if (xive_get_field32(END_W0_ESCALATE_CTL, end->w0))
			*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ESCALATE);
	}

	return OPAL_SUCCESS;
}

static void xive_cleanup_end(struct xive_end *end)
{
	end->w0 = xive_set_field32(END_W0_FIRMWARE1, 0, xive_end_is_firmware1(end));
	end->w1 = xive_set_field32(END_W1_ESe_Q, 0, 1) |
		  xive_set_field32(END_W1_ESn_Q, 0, 1);
	end->w2 = end->w3 = end->w4 = end->w5 = end->w6 = end->w7 = 0;
}

static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
					uint64_t qpage,
					uint64_t qsize,
					uint64_t qflags)
{
	uint32_t blk, idx;
	struct xive *x;
	struct xive_end *old_end;
	struct xive_end end;
	uint32_t vp_blk, vp_idx;
	bool group;
	int64_t rc;

	if (!xive_end_for_target(vp, prio, &blk, &idx))
		return OPAL_PARAMETER;

	x = xive_from_vc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;

	old_end = xive_get_end(x, idx);
	if (!old_end)
		return OPAL_PARAMETER;

	/* If this is a silent escalation queue, it cannot be
	 * configured directly
	 */
	if (xive_get_field32(END_W0_SILENT_ESCALATE, old_end->w0))
		return OPAL_PARAMETER;

	/* This shouldn't fail or xive_end_for_target would have
	 * failed already
	 */
	if (!xive_decode_vp(vp, &vp_blk, &vp_idx, NULL, &group))
		return OPAL_PARAMETER;

	/*
	 * Make a local copy which we will later try to commit using
	 * the cache watch facility
	 */
	end = *old_end;

	if (qflags & OPAL_XIVE_EQ_ENABLED) {
		switch(qsize) {
			/* Supported sizes */
		case 12:
		case 16:
		case 21:
		case 24:
			end.w3 = cpu_to_be32(qpage & END_W3_EQ_ADDR_LO);
			end.w2 = cpu_to_be32((qpage >> 32) & END_W2_EQ_ADDR_HI);
			end.w3 = xive_set_field32(END_W3_QSIZE, end.w3, qsize - 12);
			end.w0 = xive_set_field32(END_W0_ENQUEUE, end.w0, 1);
			break;
		case 0:
			end.w2 = end.w3 = 0;
			end.w0 = xive_set_field32(END_W0_ENQUEUE, end.w0, 0);
			break;
		default:
			return OPAL_PARAMETER;
		}

		/* Ensure the priority and target are correctly set (they will
		 * not be right after allocation
		 */
		end.w6 = xive_set_field32(END_W6_VP_BLOCK, 0, vp_blk) |
			xive_set_field32(END_W6_VP_OFFSET, 0, vp_idx);
		end.w7 = xive_set_field32(END_W7_F0_PRIORITY, 0, prio);
		/* XXX Handle group i bit when needed */

		/* Always notify flag */
		if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY)
			end.w0 = xive_set_field32(END_W0_UCOND_NOTIFY, end.w0, 1);
		else
			end.w0 = xive_set_field32(END_W0_UCOND_NOTIFY, end.w0, 0);

		/* Escalation flag */
		if (qflags & OPAL_XIVE_EQ_ESCALATE)
			end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 1);
		else
			end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 0);

		/* Unconditionally clear the current queue pointer, set
		 * generation to 1 and disable escalation interrupts.
		 */
		end.w1 = xive_set_field32(END_W1_GENERATION, 0, 1) |
			 xive_set_field32(END_W1_ES, 0, xive_get_field32(END_W1_ES, old_end->w1));

		/* Enable. We always enable backlog for an enabled queue
		 * otherwise escalations won't work.
		 */
		end.w0 = xive_set_field32(END_W0_VALID, end.w0, 1);
		end.w0 = xive_set_field32(END_W0_BACKLOG, end.w0, 1);
	} else
		xive_cleanup_end(&end);

	/* Update END, non-synchronous */
	lock(&x->lock);
	rc = xive_endc_cache_update(x, blk, idx, &end, false);
	unlock(&x->lock);

	return rc;
}

static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
					 beint32_t *out_qtoggle,
					 beint32_t *out_qindex)
{
	uint32_t blk, idx;
	struct xive *x;
	struct xive_end *end;
	int64_t rc;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;

	if (!out_qtoggle || !out_qindex ||
	    !xive_end_for_target(vp, prio, &blk, &idx))
		return OPAL_PARAMETER;

	x = xive_from_vc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;

	end = xive_get_end(x, idx);
	if (!end)
		return OPAL_PARAMETER;

	/* Scrub the queue */
	lock(&x->lock);
	rc = xive_endc_scrub(x, blk, idx);
	unlock(&x->lock);
	if (rc)
		return rc;

	/* We don't do disable queues */
	if (!xive_get_field32(END_W0_VALID, end->w0))
		return OPAL_WRONG_STATE;

	*out_qtoggle = cpu_to_be32(xive_get_field32(END_W1_GENERATION, end->w1));
	*out_qindex  = cpu_to_be32(xive_get_field32(END_W1_PAGE_OFF, end->w1));

	return OPAL_SUCCESS;
}

static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
					 uint32_t qtoggle, uint32_t qindex)
{
	uint32_t blk, idx;
	struct xive *x;
	struct xive_end *end, new_end;
	int64_t rc;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;

	if (!xive_end_for_target(vp, prio, &blk, &idx))
		return OPAL_PARAMETER;

	x = xive_from_vc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;

	end = xive_get_end(x, idx);
	if (!end)
		return OPAL_PARAMETER;

	/* We don't do disable queues */
	if (!xive_get_field32(END_W0_VALID, end->w0))
		return OPAL_WRONG_STATE;

	new_end = *end;

	new_end.w1 = xive_set_field32(END_W1_GENERATION, new_end.w1, qtoggle);
	new_end.w1 = xive_set_field32(END_W1_PAGE_OFF, new_end.w1, qindex);

	lock(&x->lock);
	rc = xive_endc_cache_update(x, blk, idx, &new_end, false);
	unlock(&x->lock);

	return rc;
}

static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
{
	struct proc_chip *c = get_chip(chip_id);
	struct list_node *n;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;
	if (!c)
		return OPAL_PARAMETER;
	if (!c->xive)
		return OPAL_PARAMETER;
	if (addr & 0xffff)
		return OPAL_PARAMETER;

	n = (struct list_node *)addr;
	lock(&c->xive->lock);
	list_add(&c->xive->donated_pages, n);
	unlock(&c->xive->lock);
	return OPAL_SUCCESS;
}

static int64_t opal_xive_get_vp_info(uint64_t vp_id,
				     beint64_t *out_flags,
				     beint64_t *out_cam_value,
				     beint64_t *out_report_cl_pair,
				     beint32_t *out_chip_id)
{
	struct xive *x;
	struct xive_nvp *vp;
	uint32_t blk, idx;
	bool group;

	if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
		return OPAL_PARAMETER;
	/* We don't do groups yet */
	if (group)
		return OPAL_PARAMETER;
	x = xive_from_pc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;
	vp = xive_get_vp(x, idx);
	if (!vp)
		return OPAL_PARAMETER;

	if (out_flags) {
		uint32_t end_blk, end_idx;
		struct xive_end *end;
		struct xive *end_x;
		*out_flags = 0;

		/*
		 * We would like to a way to stash a SW bit in the VP
		 * to know whether silent escalation is enabled or
		 * not, but unlike what happens with ENDs, the PC
		 * cache watch doesn't implement the reserved bit in
		 * the VPs... so we have to go look at END 7 instead.
		 */

		/* Grab END for prio 7 to check for silent escalation */
		if (!xive_end_for_target(vp_id, xive_escalation_prio(x),
					 &end_blk, &end_idx))
			return OPAL_PARAMETER;

		end_x = xive_from_vc_blk(end_blk);
		if (!end_x)
			return OPAL_PARAMETER;

		end = xive_get_end(x, end_idx);
		if (!end)
			return OPAL_PARAMETER;
		if (xive_get_field32(NVP_W0_VALID, vp->w0))
			*out_flags |= cpu_to_be64(OPAL_XIVE_VP_ENABLED);
		if (xive_cfg_save_restore(x))
			*out_flags |= cpu_to_be64(OPAL_XIVE_VP_SAVE_RESTORE);
		if (xive_get_field32(END_W0_SILENT_ESCALATE, end->w0))
			*out_flags |= cpu_to_be64(OPAL_XIVE_VP_SINGLE_ESCALATION);
	}

	if (out_cam_value) {
		uint64_t cam_value;

		cam_value = (blk << x->vp_shift) | idx;

		/*
		 * If save-restore is enabled, force the CAM line
		 * value with the H bit.
		 */
		if (xive_cfg_save_restore(x))
			cam_value |= TM10_QW1W2_HO;

		*out_cam_value = cpu_to_be64(cam_value);
	}

	if (out_report_cl_pair) {
		uint64_t report_cl_pair;

		report_cl_pair = ((uint64_t)(be32_to_cpu(vp->w6) & 0x0fffffff)) << 32;
		report_cl_pair |= be32_to_cpu(vp->w7) & 0xffffff00;

		*out_report_cl_pair = cpu_to_be64(report_cl_pair);
	}

	if (out_chip_id)
		*out_chip_id = cpu_to_be32(xive_block_to_chip[blk]);

	return OPAL_SUCCESS;
}

static int64_t xive_setup_silent_gather(uint64_t vp_id, bool enable)
{
	uint32_t blk, idx, i;
	struct xive_end *end_orig;
	struct xive_end end;
	struct xive *x;
	int64_t rc;

	/* Get base END block */
	if (!xive_end_for_target(vp_id, 0, &blk, &idx)) {
		prlog(PR_ERR, "%s: Invalid VP 0x%08llx\n", __func__, vp_id);
		return OPAL_PARAMETER;
	}
	x = xive_from_vc_blk(blk);
	if (!x) {
		prlog(PR_ERR, "%s: VP 0x%08llx has invalid block %d\n", __func__,
		      vp_id, blk);
		return OPAL_PARAMETER;
	}

	/* Grab prio 7 */
	end_orig = xive_get_end(x, idx + xive_escalation_prio(x));
	if (!end_orig) {
		xive_err(x, "Failed to get silent gather END 0x%x for VP 0x%08llx\n",
			 idx + xive_escalation_prio(x), vp_id);
		return OPAL_PARAMETER;
	}

	/* If trying to enable silent gather, make sure prio 7 is not
	 * already enabled as a normal queue
	 */
	if (enable && xive_get_field32(END_W0_VALID, end_orig->w0) &&
	    !xive_get_field32(END_W0_SILENT_ESCALATE, end_orig->w0)) {
		xive_err(x, "silent gather END 0x%x already in use\n",
			 idx + xive_escalation_prio(x));
		return OPAL_PARAMETER;
	}

	end = *end_orig;

	if (enable) {
		/* W0: Enabled and "s" set, no other bit */
		end.w0 = xive_set_field32(END_W0_FIRMWARE1, end.w0, 0);
		end.w0 = xive_set_field32(END_W0_VALID, end.w0, 1);
		end.w0 = xive_set_field32(END_W0_SILENT_ESCALATE, end.w0, 1);
		end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 1);
		end.w0 = xive_set_field32(END_W0_BACKLOG, end.w0, 1);

		/* Set new "N" for END escalation (vs. ESB)  */
		end.w0 = xive_set_field32(END_W0_ESCALATE_END, end.w0, 1);

		/* W1: Mark ESn as 01, ESe as 00 */
		end.w1 = xive_set_field32(END_W1_ESn_P, end.w1, 0);
		end.w1 = xive_set_field32(END_W1_ESn_Q, end.w1, 1);
		end.w1 = xive_set_field32(END_W1_ESe, end.w1, 0);
	} else if (xive_get_field32(END_W0_SILENT_ESCALATE, end.w0))
		xive_cleanup_end(&end);

	if (!memcmp(end_orig, &end, sizeof(end)))
		rc = 0;
	else
		rc = xive_endc_cache_update(x, blk, idx + xive_escalation_prio(x),
					    &end, false);
	if (rc)
		return rc;

	/* Mark/unmark all other prios with the new "u" bit and update
	 * escalation
	 */
	for (i = 0; i < xive_cfg_vp_prio(x); i++) {
		if (i == xive_escalation_prio(x))
			continue;
		end_orig = xive_get_end(x, idx + i);
		if (!end_orig)
			continue;
		end = *end_orig;
		if (enable) {
			/* Set "u" bit */
			end.w0 = xive_set_field32(END_W0_UNCOND_ESCALATE, end.w0, 1);

			/* Set new "N" for END escalation (vs. ESB)  */
			/* TODO (Gen2+) : use ESB escalation configuration */
			end.w0 = xive_set_field32(END_W0_ESCALATE_END, end.w0, 1);

			/* Re-route escalation interrupt (previous
			 * route is lost !) to the gather queue
			 */
			end.w4 = xive_set_field32(END_W4_END_BLOCK, end.w4, blk);
			end.w4 = xive_set_field32(END_W4_ESC_END_INDEX,
					  end.w4, idx + xive_escalation_prio(x));
		} else if (xive_get_field32(END_W0_UNCOND_ESCALATE, end.w0)) {
			/* Clear the "u" bit, disable escalations if it was set */
			end.w0 = xive_set_field32(END_W0_UNCOND_ESCALATE, end.w0, 0);
			end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 0);
		}
		if (!memcmp(end_orig, &end, sizeof(end)))
			continue;
		rc = xive_endc_cache_update(x, blk, idx + i, &end, false);
		if (rc)
			break;
	}

	return rc;
}

static int64_t opal_xive_set_vp_info(uint64_t vp_id,
				     uint64_t flags,
				     uint64_t report_cl_pair)
{
	struct xive *x;
	struct xive_nvp *vp, vp_new;
	uint32_t blk, idx;
	bool group;
	int64_t rc;

	if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
		return OPAL_PARAMETER;
	/* We don't do groups yet */
	if (group)
		return OPAL_PARAMETER;
	if (report_cl_pair & 0xff)
		return OPAL_PARAMETER;
	x = xive_from_pc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;
	vp = xive_get_vp(x, idx);
	if (!vp)
		return OPAL_PARAMETER;

	/* Consistency check. */
	if ((flags & OPAL_XIVE_VP_SAVE_RESTORE) && !xive_cfg_save_restore(x))
		return OPAL_PARAMETER;

	lock(&x->lock);

	vp_new = *vp;
	if (flags & OPAL_XIVE_VP_ENABLED) {
		vp_new.w0 = xive_set_field32(NVP_W0_VALID, vp_new.w0, 1);
		vp_new.w6 = cpu_to_be32(report_cl_pair >> 32);
		vp_new.w7 = cpu_to_be32(report_cl_pair & 0xffffffff);

		if (flags & OPAL_XIVE_VP_SINGLE_ESCALATION)
			rc = xive_setup_silent_gather(vp_id, true);
		else
			rc = xive_setup_silent_gather(vp_id, false);

		/*
		 * Prepare NVP to be HW owned for automatic save-restore
		 */
		if (xive_cfg_save_restore(x)) {
			/*
			 * Set NVP privilege level. Default to OS.
			 * This check only makes sense for KVM guests
			 * currently. We would need an extra flag to
			 * distinguish from pool level.
			 */
			vp_new.w0 = xive_set_field32(NVP_W0_VPRIV, vp_new.w0, 0);

			vp_new.w2 = xive_set_field32(NVP_W2_CPPR, vp_new.w2, 0xFF);
			vp_new.w0 = xive_set_field32(NVP_W0_HW, vp_new.w0, 1);
		}
	} else {
		/*
		 * TODO (kvm): disabling a VP invalidates the associated ENDs.
		 *
		 * The loads then return all 1s which can be an issue for the
		 * Linux code to handle.
		 */

		vp_new.w0 = vp_new.w6 = vp_new.w7 = 0;
		rc = xive_setup_silent_gather(vp_id, false);
	}

	if (rc) {
		if (rc != OPAL_BUSY)
			xive_dbg(x, "Silent gather setup failed with err %lld\n", rc);
		goto bail;
	}

	rc = xive_nxc_cache_update(x, blk, idx, &vp_new, false);
	if (rc)
		goto bail;

	/* When disabling, we scrub clean (invalidate the entry) so
	 * we can avoid cache ops in alloc/free
	 */
	if (!(flags & OPAL_XIVE_VP_ENABLED))
		xive_nxc_scrub_clean(x, blk, idx);

bail:
	unlock(&x->lock);
	return rc;
}

static int64_t opal_xive_get_vp_state(uint64_t vp_id, beint64_t *out_state)
{
	struct xive *x;
	struct xive_nvp *vp;
	uint32_t blk, idx;
	int64_t rc;
	bool group;

	if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
		return OPAL_PARAMETER;
	if (group)
		return OPAL_PARAMETER;
	x = xive_from_pc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;
	vp = xive_get_vp(x, idx);
	if (!vp)
		return OPAL_PARAMETER;

	/* Scrub the vp */
	lock(&x->lock);
	rc = xive_nxc_scrub(x, blk, idx);
	unlock(&x->lock);
	if (rc)
		return rc;

	if (!xive_get_field32(NVP_W0_VALID, vp->w0))
		return OPAL_WRONG_STATE;

	/*
	 * return a state matching the layout of WORD 0-1 of the TIMA
	 * as this is expected by current implementation.
	 */
	*out_state = cpu_to_be64(((uint64_t) 0x0) << 54 |
		(uint64_t)xive_get_field32(NVP_W2_CPPR, vp->w2)  << 48 |
		(uint64_t)xive_get_field32(NVP_W2_IPB,  vp->w2)  << 40 |
		(uint64_t)xive_get_field32(NVP_W2_LSMFB, vp->w2) << 32);

	return OPAL_SUCCESS;
}

static void *xive_cpu_get_tima(struct cpu_thread *c)
{
	struct xive_cpu_state *xs = c->xstate;
	struct xive *x = xs->xive;

	return x->ic_tm_direct_base + ((c->pir & 0xff) << x->ic_shift);
}

static void xive_cleanup_cpu_tima(struct cpu_thread *c)
{
	struct xive_cpu_state *xs __unused = c->xstate;
	void *cpu_tm_base = xive_cpu_get_tima(c);
	uint8_t old_w2 __unused, w2 __unused;

	/* Reset the HW context */
	xive_reset_enable_thread(c);

	/* Set VT to 1 */
	old_w2 = in_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2);
	out_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2, 0x80);
	w2 = in_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2);

	/* Dump HV state */
	xive_cpu_vdbg(c, "[reset] VP TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n",
		      xs->vp_blk, xs->vp_idx,
		      in_be64(cpu_tm_base + TM_QW3_HV_PHYS),
		      old_w2, w2);
}

static int64_t xive_vc_ind_cache_kill(struct xive *x, uint64_t type)
{
	uint64_t val;

	/* We clear the whole thing */
	xive_regw(x, VC_AT_MACRO_KILL_MASK, 0);
	xive_regw(x, VC_AT_MACRO_KILL, VC_AT_MACRO_KILL_VALID |
		  SETFIELD(VC_AT_MACRO_KILL_VSD, 0ull, type));

	/* XXX Add timeout */
	for (;;) {
		val = xive_regr(x, VC_AT_MACRO_KILL);
		if (!(val & VC_AT_MACRO_KILL_VALID))
			break;
	}
	return 0;
}

static int64_t xive_pc_ind_cache_kill(struct xive *x)
{
	uint64_t val;

	/* We clear the whole thing */
	xive_regw(x, PC_AT_KILL_MASK, 0);
	xive_regw(x, PC_AT_KILL, PC_AT_KILL_VALID |
		  SETFIELD(VC_AT_MACRO_KILL_VSD, 0ull, VST_NVP));

	/* XXX Add timeout */
	for (;;) {
		val = xive_regr(x, PC_AT_KILL);
		if (!(val & PC_AT_KILL_VALID))
			break;
	}
	return 0;
}

static void xive_cleanup_vp_ind(struct xive *x)
{
	int i;

	xive_dbg(x, "Cleaning up %d VP ind entries...\n", x->vp_ind_count);
	for (i = 0; i < x->vp_ind_count; i++) {
		if (be64_to_cpu(x->vp_ind_base[i]) & VSD_FIRMWARE) {
			xive_dbg(x, " %04x ... skip (firmware)\n", i);
			continue;
		}
		if (x->vp_ind_base[i] != 0) {
			x->vp_ind_base[i] = 0;
			xive_dbg(x, " %04x ... cleaned\n", i);
		}
	}
	xive_pc_ind_cache_kill(x);
}

static void xive_cleanup_end_ind(struct xive *x)
{
	int i;

	xive_dbg(x, "Cleaning up %d END ind entries...\n", x->end_ind_count);
	for (i = 0; i < x->end_ind_count; i++) {
		if (be64_to_cpu(x->end_ind_base[i]) & VSD_FIRMWARE) {
			xive_dbg(x, " %04x ... skip (firmware)\n", i);
			continue;
		}
		if (x->end_ind_base[i] != 0) {
			x->end_ind_base[i] = 0;
			xive_dbg(x, " %04x ... cleaned\n", i);
		}
	}
	xive_vc_ind_cache_kill(x, VST_END);
}

static void xive_reset_one(struct xive *x)
{
	struct cpu_thread *c;
	bool end_firmware;
	int i;

	xive_notice(x, "Resetting one xive...\n");

	lock(&x->lock);

	/* Check all interrupts are disabled */
	i = bitmap_find_one_bit(*x->int_enabled_map, 0, XIVE_INT_COUNT);
	if (i >= 0)
		xive_warn(x, "Interrupt %d (and maybe more) not disabled"
			  " at reset !\n", i);

	/* Reset IPI allocation */
	xive_dbg(x, "freeing alloc map %p/%p\n",
		 x->ipi_alloc_map, *x->ipi_alloc_map);
	memset(x->ipi_alloc_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));

	xive_dbg(x, "Resetting ENDs...\n");

	/* Reset all allocated ENDs and free the user ones */
	bitmap_for_each_one(*x->end_map, xive_end_bitmap_size(x), i) {
		struct xive_end end0;
		struct xive_end *end;
		int j;

		if (i == 0)
			continue;
		end_firmware = false;
		for (j = 0; j < xive_cfg_vp_prio(x); j++) {
			uint32_t idx = (i << xive_cfg_vp_prio_shift(x)) | j;

			end = xive_get_end(x, idx);
			if (!end)
				continue;

			/* We need to preserve the firmware bit, otherwise
			 * we will incorrectly free the ENDs that are reserved
			 * for the physical CPUs
			 */
			if (xive_get_field32(END_W0_VALID, end->w0)) {
				if (!xive_end_is_firmware1(end))
					xive_dbg(x, "END 0x%x:0x%x is valid at reset: %08x %08x\n",
						 x->block_id, idx, end->w0, end->w1);
				end0 = *end;
				xive_cleanup_end(&end0);
				xive_endc_cache_update(x, x->block_id, idx, &end0, true);
			}
			if (xive_end_is_firmware1(end))
				end_firmware = true;
		}
		if (!end_firmware)
			bitmap_clr_bit(*x->end_map, i);
	}

	/* Take out all VPs from HW and reset all CPPRs to 0 */
	for_each_present_cpu(c) {
		if (c->chip_id != x->chip_id)
			continue;
		if (!c->xstate)
			continue;
		xive_cleanup_cpu_tima(c);
	}

	/* Reset all user-allocated VPs. This is inefficient, we should
	 * either keep a bitmap of allocated VPs or add an iterator to
	 * the buddy which is trickier but doable.
	 */
	for (i = 0; i < XIVE_VP_COUNT(x); i++) {
		struct xive_nvp *vp;
		struct xive_nvp vp0 = {0};

		/* Ignore the physical CPU VPs */
		if (i >= xive_hw_vp_count &&
		    i < (xive_hw_vp_base + xive_hw_vp_count))
			continue;

		/* Is the VP valid ? */
		vp = xive_get_vp(x, i);
		if (!vp || !xive_get_field32(NVP_W0_VALID, vp->w0))
			continue;

		/* Clear it */
		xive_dbg(x, "VP 0x%x:0x%x is valid at reset\n", x->block_id, i);
		xive_nxc_cache_update(x, x->block_id, i, &vp0, true);
	}

	/* Forget about remaining donated pages */
	list_head_init(&x->donated_pages);

	/* And cleanup donated indirect VP and END pages */
	xive_cleanup_vp_ind(x);
	xive_cleanup_end_ind(x);

	/* The rest must not be called with the lock held */
	unlock(&x->lock);

	/* Re-configure VPs */
	for_each_present_cpu(c) {
		struct xive_cpu_state *xs = c->xstate;

		if (c->chip_id != x->chip_id || !xs)
			continue;

		xive_init_cpu_exploitation(xs);
	}
}

static void xive_reset_mask_source_cb(struct irq_source *is,
				      void *data __unused)
{
	struct xive_src *s = container_of(is, struct xive_src, is);
	struct xive *x;
	uint32_t isn;

	if (is->ops != &xive_irq_source_ops)
		return;

	/* Skip escalation sources */
	if (GIRQ_IS_ESCALATION(is->start))
		return;

	x = s->xive;

	/* Iterate all interrupts */
	for (isn = is->start; isn < is->end; isn++) {
		/* Has it ever been enabled ? */
		if (!bitmap_tst_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn)))
			continue;
		/* Mask it and clear the enabled map bit */
		xive_vdbg(x, "[reset] disabling source 0x%x\n", isn);
		__xive_set_irq_config(is, isn, 0, 0xff, isn, true, false);
		bitmap_clr_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn));
	}
}

void xive2_cpu_reset(void)
{
	struct cpu_thread *c = this_cpu();
	struct xive_cpu_state *xs = c->xstate;

	out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_CPPR, 0);

	in_be64(xs->tm_ring1 + TM_SPC_PULL_POOL_CTX);
}

static int64_t __xive_reset(uint64_t mode)
{
	struct proc_chip *chip;

	xive_mode = mode;

	/* Mask all interrupt sources */
	irq_for_each_source(xive_reset_mask_source_cb, NULL);

	/* For each XIVE do a sync... */
	for_each_chip(chip) {
		if (!chip->xive)
			continue;
		xive_sync(chip->xive);
	}

	/* For each XIVE reset everything else... */
	for_each_chip(chip) {
		if (!chip->xive)
			continue;
		xive_reset_one(chip->xive);
	}

	/* Cleanup global VP allocator */
	buddy_reset(xive_vp_buddy);

	/*
	 * We reserve the whole range of VP ids for HW threads.
	 */
	assert(buddy_reserve(xive_vp_buddy, xive_hw_vp_base, xive_threadid_shift));

	return OPAL_SUCCESS;
}

/* Called by fast reboot */
int64_t xive2_reset(void)
{
	if (xive_mode == XIVE_MODE_NONE)
		return OPAL_SUCCESS;
	return __xive_reset(XIVE_MODE_EXPL);
}

static int64_t opal_xive_reset(uint64_t mode)
{
	prlog(PR_DEBUG, "XIVE reset. mode = %llx\n", mode);

	if (!(mode & XIVE_MODE_EXPL)) {
		prlog(PR_NOTICE, "No emulation mode. XIVE exploitation mode "
		      "is the default\n");
	}

	xive_expl_options = mode & ~XIVE_MODE_EXPL;
	if (xive_expl_options & ~XIVE_EXPL_ALL_OPTIONS) {
		prerror("invalid XIVE exploitation mode option %016llx\n",
			xive_expl_options);
		return OPAL_PARAMETER;
	}

	return __xive_reset(XIVE_MODE_EXPL);
}

static int64_t opal_xive_free_vp_block(uint64_t vp_base)
{
	uint32_t blk, idx, i, j, count;
	uint8_t order;
	bool group;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;

	if (!xive_decode_vp(vp_base, &blk, &idx, &order, &group))
		return OPAL_PARAMETER;
	if (group)
		return OPAL_PARAMETER;
	if (blk)
		return OPAL_PARAMETER;
	if (order < (xive_chips_alloc_bits + 1))
		return OPAL_PARAMETER;
	if (idx & ((1 << (order - xive_chips_alloc_bits)) - 1))
		return OPAL_PARAMETER;

	count = 1 << order;
	for (i = 0; i < count; i++) {
		uint32_t vp_id = vp_base + i;
		uint32_t blk, idx, end_blk, end_idx;
		struct xive *x;
		struct xive_nvp *vp;

		if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) {
			prerror("Couldn't decode VP id %u\n", vp_id);
			return OPAL_INTERNAL_ERROR;
		}
		x = xive_from_pc_blk(blk);
		if (!x) {
			prerror("Instance not found for deallocated VP"
				" block %d\n", blk);
			return OPAL_INTERNAL_ERROR;
		}
		vp = xive_get_vp(x, idx);
		if (!vp) {
			prerror("VP not found for deallocation !");
			return OPAL_INTERNAL_ERROR;
		}

		/* VP must be disabled */
		if (xive_get_field32(NVP_W0_VALID, vp->w0)) {
			prlog(PR_ERR, "freeing active VP %d\n", vp_id);
			return OPAL_XIVE_FREE_ACTIVE;
		}

		/* Not populated */
		if (vp->w5 == 0)
			continue;

		end_blk = xive_get_field32(NVP_W5_VP_END_BLOCK, vp->w5);
		end_idx = xive_get_field32(NVP_W5_VP_END_INDEX, vp->w5);

		lock(&x->lock);

		/* Ensure ENDs are disabled and cleaned up. Ideally the caller
		 * should have done it but we double check it here
		 */
		for (j = 0; j < xive_cfg_vp_prio(x); j++) {
			struct xive *end_x = xive_from_vc_blk(end_blk);
			struct xive_end end, *orig_end = xive_get_end(end_x, end_idx + j);

			if (!xive_get_field32(END_W0_VALID, orig_end->w0))
				continue;

			prlog(PR_WARNING, "freeing VP %d with queue %d active\n",
			      vp_id, j);
			end = *orig_end;
			xive_cleanup_end(&end);
			xive_endc_cache_update(x, end_blk, end_idx + j, &end, true);
		}

		/* Mark it not populated so we don't try to free it again */
		vp->w5 = 0;

		if (end_blk != blk) {
			prerror("Block mismatch trying to free ENDs\n");
			unlock(&x->lock);
			return OPAL_INTERNAL_ERROR;
		}

		xive_free_end_set(x, end_idx);
		unlock(&x->lock);
	}

	xive_free_vps(vp_base);

	return OPAL_SUCCESS;
}

static int64_t opal_xive_alloc_vp_block(uint32_t alloc_order)
{
	uint32_t vp_base, ends, count, i;
	int64_t rc;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;

	prlog(PR_TRACE, "opal_xive_alloc_vp_block(%d)\n", alloc_order);

	vp_base = xive_alloc_vps(alloc_order);
	if (XIVE_ALLOC_IS_ERR(vp_base)) {
		if (vp_base == XIVE_ALLOC_NO_IND)
			return OPAL_XIVE_PROVISIONING;
		return OPAL_RESOURCE;
	}

	/* Allocate ENDs and initialize VPs */
	count = 1 << alloc_order;
	for (i = 0; i < count; i++) {
		uint32_t vp_id = vp_base + i;
		uint32_t blk, idx;
		struct xive *x;
		struct xive_nvp *vp;

		if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) {
			prerror("Couldn't decode VP id %u\n", vp_id);
			return OPAL_INTERNAL_ERROR;
		}
		x = xive_from_pc_blk(blk);
		if (!x) {
			prerror("Instance not found for allocated VP"
				" block %d\n", blk);
			rc = OPAL_INTERNAL_ERROR;
			goto fail;
		}
		vp = xive_get_vp(x, idx);
		if (!vp) {
			prerror("VP not found after allocation !");
			rc = OPAL_INTERNAL_ERROR;
			goto fail;
		}

		/* Allocate ENDs, if fails, free the VPs and return */
		lock(&x->lock);
		ends = xive_alloc_end_set(x, false);
		unlock(&x->lock);
		if (XIVE_ALLOC_IS_ERR(ends)) {
			if (ends == XIVE_ALLOC_NO_IND)
				rc = OPAL_XIVE_PROVISIONING;
			else
				rc = OPAL_RESOURCE;
			goto fail;
		}

		/* Initialize the VP structure. We don't use a cache watch
		 * as we have made sure when freeing the entries to scrub
		 * it out of the cache.
		 */
		memset(vp, 0, sizeof(*vp));

		/* Store the END base of the VP in W5 (new in p10) */
		xive_vp_set_end_base(vp, blk, ends);
	}
	return vp_base;
 fail:
	opal_xive_free_vp_block(vp_base);

	return rc;
}

static int64_t xive_try_allocate_irq(struct xive *x)
{
	int idx, base_idx, max_count, girq;
	struct xive_eas *eas;

	lock(&x->lock);

	base_idx = x->int_ipi_top - x->int_base;
	max_count = x->int_hw_bot - x->int_ipi_top;

	idx = bitmap_find_zero_bit(*x->ipi_alloc_map, base_idx, max_count);
	if (idx < 0) {
		unlock(&x->lock);
		return OPAL_RESOURCE;
	}
	bitmap_set_bit(*x->ipi_alloc_map, idx);
	girq = x->int_base + idx;

	/* Mark the EAS valid. Don't bother with the HW cache, it's
	 * still masked anyway, the cache will be updated when unmasked
	 * and configured.
	 */
	eas = xive_get_eas(x, girq);
	if (!eas) {
		bitmap_clr_bit(*x->ipi_alloc_map, idx);
		unlock(&x->lock);
		return OPAL_PARAMETER;
	}
	eas->w = xive_set_field64(EAS_VALID, 0, 1) |
		 xive_set_field64(EAS_MASKED, 0, 1) |
		 xive_set_field64(EAS_END_DATA, 0, girq);
	unlock(&x->lock);

	return girq;
}

static int64_t opal_xive_allocate_irq(uint32_t chip_id)
{
	struct proc_chip *chip;
	bool try_all = false;
	int64_t rc;

	if (xive_mode != XIVE_MODE_EXPL)
		return OPAL_WRONG_STATE;

	if (chip_id == OPAL_XIVE_ANY_CHIP) {
		try_all = true;
		chip_id = this_cpu()->chip_id;
	}
	chip = get_chip(chip_id);
	if (!chip)
		return OPAL_PARAMETER;

	/* Try initial target chip */
	if (!chip->xive)
		rc = OPAL_PARAMETER;
	else
		rc = xive_try_allocate_irq(chip->xive);
	if (rc >= 0 || !try_all)
		return rc;

	/* Failed and we try all... do so */
	for_each_chip(chip) {
		if (!chip->xive)
			continue;
		rc = xive_try_allocate_irq(chip->xive);
		if (rc >= 0)
			break;
	}
	return rc;
}

static int64_t opal_xive_free_irq(uint32_t girq)
{
	struct irq_source *is = irq_find_source(girq);
	struct xive_src *s = container_of(is, struct xive_src, is);
	struct xive *x = xive_from_isn(girq);
	struct xive_eas *eas;
	uint32_t idx;

	if (xive_mode != XIVE_MODE_EXPL)
               return OPAL_WRONG_STATE;
	if (!x || !is)
		return OPAL_PARAMETER;

	idx = GIRQ_TO_IDX(girq);

	lock(&x->lock);

	eas = xive_get_eas(x, girq);
	if (!eas) {
		unlock(&x->lock);
		return OPAL_PARAMETER;
	}

	/* Mask the interrupt source */
	xive_update_irq_mask(s, girq - s->esb_base, true);

	/* Mark the EAS masked and invalid */
	eas->w = xive_set_field64(EAS_VALID, 0, 1) |
		 xive_set_field64(EAS_MASKED, 0, 1);
	xive_easc_scrub(x, x->block_id, idx);

	/* Free it */
	if (!bitmap_tst_bit(*x->ipi_alloc_map, idx)) {
		unlock(&x->lock);
		return OPAL_PARAMETER;
	}
	bitmap_clr_bit(*x->ipi_alloc_map, idx);
	bitmap_clr_bit(*x->int_enabled_map, idx);
	unlock(&x->lock);

	return OPAL_SUCCESS;
}

static int64_t opal_xive_dump_tm(uint32_t offset, const char *n, uint32_t pir)
{
	struct cpu_thread *c = find_cpu_by_pir(pir);
	struct xive_cpu_state *xs;
	struct xive *x;
	void *cpu_tm_base;
	uint64_t v0,v1;

	if (!c)
		return OPAL_PARAMETER;
	xs = c->xstate;
	if (!xs || !xs->tm_ring1)
		return OPAL_INTERNAL_ERROR;
	x = xs->xive;
	cpu_tm_base = xive_cpu_get_tima(c);

	lock(&x->lock);
	v0 = in_be64(cpu_tm_base + offset);
	if (offset == TM_QW3_HV_PHYS) {
		v1 = in_8(cpu_tm_base + offset + 8);
		v1 <<= 56;
	} else {
		v1 = in_be32(cpu_tm_base + offset + 8);
		v1 <<= 32;
	}
	prlog(PR_INFO, "CPU[%04x]: TM state for QW %s\n", pir, n);
	prlog(PR_INFO, "CPU[%04x]: NSR CPPR IPB LSMFB ACK# INC AGE PIPR"
	      " W2       W3\n", pir);
	prlog(PR_INFO, "CPU[%04x]: %02x  %02x   %02x  %02x    %02x   "
	       "%02x  %02x  %02x   %08x %08x\n", pir,
	      (uint8_t)(v0 >> 56) & 0xff, (uint8_t)(v0 >> 48) & 0xff,
	      (uint8_t)(v0 >> 40) & 0xff, (uint8_t)(v0 >> 32) & 0xff,
	      (uint8_t)(v0 >> 24) & 0xff, (uint8_t)(v0 >> 16) & 0xff,
	      (uint8_t)(v0 >>  8) & 0xff, (uint8_t)(v0      ) & 0xff,
	      (uint32_t)(v1 >> 32) & 0xffffffff,
	      (uint32_t)(v1 & 0xffffffff));
	unlock(&x->lock);

	return OPAL_SUCCESS;
}

static int64_t opal_xive_dump_vp(uint32_t vp_id)
{
	uint32_t blk, idx;
	uint8_t order;
	bool group;
	struct xive *x;
	struct xive_nvp *vp;
	uint32_t *vpw;

	if (!xive_decode_vp(vp_id, &blk, &idx, &order, &group))
		return OPAL_PARAMETER;

	x = xive_from_vc_blk(blk);
	if (!x)
		return OPAL_PARAMETER;
	vp = xive_get_vp(x, idx);
	if (!vp)
		return OPAL_PARAMETER;
	lock(&x->lock);

	xive_nxc_scrub_clean(x, blk, idx);

	vpw = ((uint32_t *)vp) + (group ? 8 : 0);
	prlog(PR_INFO, "VP[%08x]: 0..3: %08x %08x %08x %08x\n", vp_id,
	      vpw[0], vpw[1], vpw[2], vpw[3]);
	prlog(PR_INFO, "VP[%08x]: 4..7: %08x %08x %08x %08x\n", vp_id,
	      vpw[4], vpw[5], vpw[6], vpw[7]);
	unlock(&x->lock);

	return OPAL_SUCCESS;
}

static int64_t opal_xive_sync_irq_src(uint32_t girq)
{
	struct xive *x = xive_from_isn(girq);

	if (!x)
		return OPAL_PARAMETER;
	return xive_sync(x);
}

static int64_t opal_xive_sync_irq_target(uint32_t girq)
{
	uint32_t target, vp_blk;
	struct xive *x;

	if (!xive_get_irq_targetting(girq, &target, NULL, NULL))
		return OPAL_PARAMETER;
	if (!xive_decode_vp(target, &vp_blk, NULL, NULL, NULL))
		return OPAL_PARAMETER;
	x = xive_from_pc_blk(vp_blk);
	if (!x)
		return OPAL_PARAMETER;
	return xive_sync(x);
}

static int64_t opal_xive_sync(uint32_t type, uint32_t id)
{
	int64_t rc = OPAL_SUCCESS;;

	if (type & XIVE_SYNC_EAS)
		rc = opal_xive_sync_irq_src(id);
	if (rc)
		return rc;
	if (type & XIVE_SYNC_QUEUE)
		rc = opal_xive_sync_irq_target(id);
	if (rc)
		return rc;

	/* Add more ... */

	return rc;
}

static int64_t opal_xive_dump(uint32_t type, uint32_t id)
{
	switch (type) {
	case XIVE_DUMP_TM_HYP:
		return opal_xive_dump_tm(TM_QW3_HV_PHYS, "PHYS", id);
	case XIVE_DUMP_TM_POOL:
		return opal_xive_dump_tm(TM_QW2_HV_POOL, "POOL", id);
	case XIVE_DUMP_TM_OS:
		return opal_xive_dump_tm(TM_QW1_OS, "OS  ", id);
	case XIVE_DUMP_TM_USER:
		return opal_xive_dump_tm(TM_QW0_USER, "USER", id);
	case XIVE_DUMP_VP:
		return opal_xive_dump_vp(id);
	default:
		return OPAL_PARAMETER;
	}
}

static void xive_init_globals(void)
{
	uint32_t i;

	for (i = 0; i < XIVE_MAX_CHIPS; i++)
		xive_block_to_chip[i] = XIVE_INVALID_CHIP;
}

/*
 * The global availability of some capabilities used in other drivers
 * (PHB, PSI) is deduced from the capabilities of the first XIVE chip
 * of the system. It should be common to all chips.
 */
bool xive2_cap_phb_pq_disable(void)
{
	return xive_has_cap(one_xive, CQ_XIVE_CAP_PHB_PQ_DISABLE);
}

bool xive2_cap_phb_abt(void)
{
	if (!xive_has_cap(one_xive, CQ_XIVE_CAP_PHB_ABT))
		return false;

	/*
	 * We need 'PQ disable' to use ABT mode, else the OS will use
	 * two different sets of ESB pages (PHB and IC) to control the
	 * interrupt sources. Can not work.
	 */
	if (!xive2_cap_phb_pq_disable()) {
		prlog_once(PR_ERR, "ABT mode is set without PQ disable. "
			   "Ignoring bogus configuration\n");
		return false;
	}

	return true;
}

bool xive2_cap_store_eoi(void)
{
	return xive_has_cap(one_xive, CQ_XIVE_CAP_STORE_EOI);
}

void xive2_init(void)
{
	struct dt_node *np;
	struct proc_chip *chip;
	struct cpu_thread *cpu;
	bool first = true;

	/* Look for xive nodes and do basic inits */
	dt_for_each_compatible(dt_root, np, "ibm,power10-xive-x") {
		struct xive *x;

		/* Initialize some global stuff */
		if (first)
			xive_init_globals();

		/* Create/initialize the xive instance */
		x = init_one_xive(np);
		if (first)
			one_xive = x;
		first = false;
	}
	if (first)
		return;

	/*
	 * P8 emulation is not supported on P10 anymore. Exploitation
	 * is the default XIVE mode. We might introduce a GEN2 mode.
	 */
	xive_mode = XIVE_MODE_EXPL;

	/* Init VP allocator */
	xive_init_vp_allocator();

	/* Create a device-tree node for Linux use */
	xive_create_mmio_dt_node(one_xive);

	/* Some inits must be done after all xive have been created
	 * such as setting up the forwarding ports
	 */
	for_each_chip(chip) {
		if (chip->xive)
			late_init_one_xive(chip->xive);
	}

	/* Initialize per-cpu structures */
	for_each_present_cpu(cpu) {
		xive_init_cpu(cpu);
	}

	/* Calling boot CPU */
	xive2_cpu_callin(this_cpu());

	/* Register XIVE exploitation calls */
	opal_register(OPAL_XIVE_RESET, opal_xive_reset, 1);
	opal_register(OPAL_XIVE_GET_IRQ_INFO, opal_xive_get_irq_info, 6);
	opal_register(OPAL_XIVE_GET_IRQ_CONFIG, opal_xive_get_irq_config, 4);
	opal_register(OPAL_XIVE_SET_IRQ_CONFIG, opal_xive_set_irq_config, 4);
	opal_register(OPAL_XIVE_GET_QUEUE_INFO, opal_xive_get_queue_info, 7);
	opal_register(OPAL_XIVE_SET_QUEUE_INFO, opal_xive_set_queue_info, 5);
	opal_register(OPAL_XIVE_DONATE_PAGE, opal_xive_donate_page, 2);
	opal_register(OPAL_XIVE_ALLOCATE_IRQ, opal_xive_allocate_irq, 1);
	opal_register(OPAL_XIVE_FREE_IRQ, opal_xive_free_irq, 1);
	opal_register(OPAL_XIVE_ALLOCATE_VP_BLOCK, opal_xive_alloc_vp_block, 1);
	opal_register(OPAL_XIVE_FREE_VP_BLOCK, opal_xive_free_vp_block, 1);
	opal_register(OPAL_XIVE_GET_VP_INFO, opal_xive_get_vp_info, 5);
	opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
	opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
	opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
	opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
	opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
	opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
}