x86: Implement a cache for Memory Reference Code parameters

The memory reference code takes a very long time to 'train' its SDRAM
interface, around half a second. To avoid this delay on every boot we can
store the parameters from the last training sessions to speed up the next.

Add an implementation of this, storing the training data in CMOS RAM and
SPI flash.

Signed-off-by: Simon Glass <sjg@chromium.org>

5 files changed, 464 insertions, 0 deletions

diff --git a/arch/x86/cpu/ivybridge/Makefile b/arch/x86/cpu/ivybridge/Makefile
index 0c7efae..3576b83 100644
--- a/arch/x86/cpu/ivybridge/Makefile
+++ b/arch/x86/cpu/ivybridge/Makefile
@@ -14,6 +14,7 @@ obj-y += lpc.o
 obj-y += me_status.o
 obj-y += model_206ax.o
 obj-y += microcode_intel.o
+obj-y += mrccache.o
 obj-y += northbridge.o
 obj-y += pch.o
 obj-y += pci.o
diff --git a/arch/x86/cpu/ivybridge/mrccache.c b/arch/x86/cpu/ivybridge/mrccache.c
new file mode 100644
index 0000000..0f1a64b
--- /dev/null
+++ b/arch/x86/cpu/ivybridge/mrccache.c
@@ -0,0 +1,156 @@
+/*
+ * From Coreboot src/southbridge/intel/bd82x6x/mrccache.c
+ *
+ * Copyright (C) 2014 Google Inc.
+ *
+ * SPDX-License-Identifier:	GPL-2.0
+ */
+
+#include <common.h>
+#include <errno.h>
+#include <fdtdec.h>
+#include <net.h>
+#include <spi.h>
+#include <spi_flash.h>
+#include <asm/arch/mrccache.h>
+#include <asm/arch/sandybridge.h>
+
+static struct mrc_data_container *next_mrc_block(
+	struct mrc_data_container *mrc_cache)
+{
+	/* MRC data blocks are aligned within the region */
+	u32 mrc_size = sizeof(*mrc_cache) + mrc_cache->data_size;
+	if (mrc_size & (MRC_DATA_ALIGN - 1UL)) {
+		mrc_size &= ~(MRC_DATA_ALIGN - 1UL);
+		mrc_size += MRC_DATA_ALIGN;
+	}
+
+	u8 *region_ptr = (u8 *)mrc_cache;
+	region_ptr += mrc_size;
+	return (struct mrc_data_container *)region_ptr;
+}
+
+static int is_mrc_cache(struct mrc_data_container *cache)
+{
+	return cache && (cache->signature == MRC_DATA_SIGNATURE);
+}
+
+/*
+ * Find the largest index block in the MRC cache. Return NULL if none is
+ * found.
+ */
+struct mrc_data_container *mrccache_find_current(struct fmap_entry *entry)
+{
+	struct mrc_data_container *cache, *next;
+	ulong base_addr, end_addr;
+	uint id;
+
+	base_addr = (1ULL << 32) - CONFIG_ROM_SIZE + entry->offset;
+	end_addr = base_addr + entry->length;
+	cache = NULL;
+
+	/* Search for the last filled entry in the region */
+	for (id = 0, next = (struct mrc_data_container *)base_addr;
+	     is_mrc_cache(next);
+	     id++) {
+		cache = next;
+		next = next_mrc_block(next);
+		if ((ulong)next >= end_addr)
+			break;
+	}
+
+	if (id-- == 0) {
+		debug("%s: No valid MRC cache found.\n", __func__);
+		return NULL;
+	}
+
+	/* Verify checksum */
+	if (cache->checksum != compute_ip_checksum(cache->data,
+						   cache->data_size)) {
+		printf("%s: MRC cache checksum mismatch\n", __func__);
+		return NULL;
+	}
+
+	debug("%s: picked entry %u from cache block\n", __func__, id);
+
+	return cache;
+}
+
+/**
+ * find_next_mrc_cache() - get next cache entry
+ *
+ * @entry:	MRC cache flash area
+ * @cache:	Entry to start from
+ *
+ * @return next cache entry if found, NULL if we got to the end
+ */
+static struct mrc_data_container *find_next_mrc_cache(struct fmap_entry *entry,
+		struct mrc_data_container *cache)
+{
+	ulong base_addr, end_addr;
+
+	base_addr = (1ULL << 32) - CONFIG_ROM_SIZE + entry->offset;
+	end_addr = base_addr + entry->length;
+
+	cache = next_mrc_block(cache);
+	if ((ulong)cache >= end_addr) {
+		/* Crossed the boundary */
+		cache = NULL;
+		debug("%s: no available entries found\n", __func__);
+	} else {
+		debug("%s: picked next entry from cache block at %p\n",
+		      __func__, cache);
+	}
+
+	return cache;
+}
+
+int mrccache_update(struct spi_flash *sf, struct fmap_entry *entry,
+		    struct mrc_data_container *cur)
+{
+	struct mrc_data_container *cache;
+	ulong offset;
+	ulong base_addr;
+	int ret;
+
+	/* Find the last used block */
+	base_addr = (1ULL << 32) - CONFIG_ROM_SIZE + entry->offset;
+	debug("Updating MRC cache data\n");
+	cache = mrccache_find_current(entry);
+	if (cache && (cache->data_size == cur->data_size) &&
+	    (!memcmp(cache, cur, cache->data_size + sizeof(*cur)))) {
+		debug("MRC data in flash is up to date. No update\n");
+		return -EEXIST;
+	}
+
+	/* Move to the next block, which will be the first unused block */
+	if (cache)
+		cache = find_next_mrc_cache(entry, cache);
+
+	/*
+	 * If we have got to the end, erase the entire mrc-cache area and start
+	 * again at block 0.
+	 */
+	if (!cache) {
+		debug("Erasing the MRC cache region of %x bytes at %x\n",
+		      entry->length, entry->offset);
+
+		ret = spi_flash_erase(sf, entry->offset, entry->length);
+		if (ret) {
+			debug("Failed to erase flash region\n");
+			return ret;
+		}
+		cache = (struct mrc_data_container *)base_addr;
+	}
+
+	/* Write the data out */
+	offset = (ulong)cache - base_addr + entry->offset;
+	debug("Write MRC cache update to flash at %lx\n", offset);
+	ret = spi_flash_write(sf, offset, cur->data_size + sizeof(*cur), cur);
+	if (ret) {
+		debug("Failed to write to SPI flash\n");
+		return ret;
+	}
+
+	return 0;
+}
diff --git a/arch/x86/cpu/ivybridge/sdram.c b/arch/x86/cpu/ivybridge/sdram.c
index 9504735..4963448 100644
--- a/arch/x86/cpu/ivybridge/sdram.c
+++ b/arch/x86/cpu/ivybridge/sdram.c
@@ -14,12 +14,17 @@
 #include <errno.h>
 #include <fdtdec.h>
 #include <malloc.h>
+#include <net.h>
+#include <rtc.h>
+#include <spi.h>
+#include <spi_flash.h>
 #include <asm/processor.h>
 #include <asm/gpio.h>
 #include <asm/global_data.h>
 #include <asm/mtrr.h>
 #include <asm/pci.h>
 #include <asm/arch/me.h>
+#include <asm/arch/mrccache.h>
 #include <asm/arch/pei_data.h>
 #include <asm/arch/pch.h>
 #include <asm/post.h>
@@ -27,6 +32,10 @@
 
 DECLARE_GLOBAL_DATA_PTR;
 
+#define CMOS_OFFSET_MRC_SEED		152
+#define CMOS_OFFSET_MRC_SEED_S3		156
+#define CMOS_OFFSET_MRC_SEED_CHK	160
+
 /*
  * This function looks for the highest region of memory lower than 4GB which
  * has enough space for U-Boot where U-Boot is aligned on a page boundary.
@@ -80,6 +89,202 @@ void dram_init_banksize(void)
 	}
 }
 
+static int get_mrc_entry(struct spi_flash **sfp, struct fmap_entry *entry)
+{
+	const void *blob = gd->fdt_blob;
+	int node, spi_node, mrc_node;
+	int upto;
+
+	/* Find the flash chip within the SPI controller node */
+	upto = 0;
+	spi_node = fdtdec_next_alias(blob, "spi", COMPAT_INTEL_ICH_SPI, &upto);
+	if (spi_node < 0)
+		return -ENOENT;
+	node = fdt_first_subnode(blob, spi_node);
+	if (node < 0)
+		return -ECHILD;
+
+	/* Find the place where we put the MRC cache */
+	mrc_node = fdt_subnode_offset(blob, node, "rw-mrc-cache");
+	if (mrc_node < 0)
+		return -EPERM;
+
+	if (fdtdec_read_fmap_entry(blob, mrc_node, "rm-mrc-cache", entry))
+		return -EINVAL;
+
+	if (sfp) {
+		*sfp = spi_flash_probe_fdt(blob, node, spi_node);
+		if (!*sfp)
+			return -EBADF;
+	}
+
+	return 0;
+}
+
+static int read_seed_from_cmos(struct pei_data *pei_data)
+{
+	u16 c1, c2, checksum, seed_checksum;
+
+	/*
+	 * Read scrambler seeds from CMOS RAM. We don't want to store them in
+	 * SPI flash since they change on every boot and that would wear down
+	 * the flash too much. So we store these in CMOS and the large MRC
+	 * data in SPI flash.
+	 */
+	pei_data->scrambler_seed = rtc_read32(CMOS_OFFSET_MRC_SEED);
+	debug("Read scrambler seed    0x%08x from CMOS 0x%02x\n",
+	      pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+
+	pei_data->scrambler_seed_s3 = rtc_read32(CMOS_OFFSET_MRC_SEED_S3);
+	debug("Read S3 scrambler seed 0x%08x from CMOS 0x%02x\n",
+	      pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+
+	/* Compute seed checksum and compare */
+	c1 = compute_ip_checksum((u8 *)&pei_data->scrambler_seed,
+				 sizeof(u32));
+	c2 = compute_ip_checksum((u8 *)&pei_data->scrambler_seed_s3,
+				 sizeof(u32));
+	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+
+	seed_checksum = rtc_read8(CMOS_OFFSET_MRC_SEED_CHK);
+	seed_checksum |= rtc_read8(CMOS_OFFSET_MRC_SEED_CHK + 1) << 8;
+
+	if (checksum != seed_checksum) {
+		debug("%s: invalid seed checksum\n", __func__);
+		pei_data->scrambler_seed = 0;
+		pei_data->scrambler_seed_s3 = 0;
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int prepare_mrc_cache(struct pei_data *pei_data)
+{
+	struct mrc_data_container *mrc_cache;
+	struct fmap_entry entry;
+	int ret;
+
+	ret = read_seed_from_cmos(pei_data);
+	if (ret)
+		return ret;
+	ret = get_mrc_entry(NULL, &entry);
+	if (ret)
+		return ret;
+	mrc_cache = mrccache_find_current(&entry);
+	if (!mrc_cache)
+		return -ENOENT;
+
+	/*
+	 * TODO(sjg@chromium.org): Skip this for now as it causes boot
+	 * problems
+	 */
+	if (0) {
+		pei_data->mrc_input = mrc_cache->data;
+		pei_data->mrc_input_len = mrc_cache->data_size;
+	}
+	debug("%s: at %p, size %x checksum %04x\n", __func__,
+	      pei_data->mrc_input, pei_data->mrc_input_len,
+	      mrc_cache->checksum);
+
+	return 0;
+}
+
+static int build_mrc_data(struct mrc_data_container **datap)
+{
+	struct mrc_data_container *data;
+	int orig_len;
+	int output_len;
+
+	orig_len = gd->arch.mrc_output_len;
+	output_len = ALIGN(orig_len, 16);
+	data = malloc(output_len + sizeof(*data));
+	if (!data)
+		return -ENOMEM;
+	data->signature = MRC_DATA_SIGNATURE;
+	data->data_size = output_len;
+	data->reserved = 0;
+	memcpy(data->data, gd->arch.mrc_output, orig_len);
+
+	/* Zero the unused space in aligned buffer. */
+	if (output_len > orig_len)
+		memset(data->data + orig_len, 0, output_len - orig_len);
+
+	data->checksum = compute_ip_checksum(data->data, output_len);
+	*datap = data;
+
+	return 0;
+}
+
+static int write_seeds_to_cmos(struct pei_data *pei_data)
+{
+	u16 c1, c2, checksum;
+
+	/* Save the MRC seed values to CMOS */
+	rtc_write32(CMOS_OFFSET_MRC_SEED, pei_data->scrambler_seed);
+	debug("Save scrambler seed    0x%08x to CMOS 0x%02x\n",
+	      pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+
+	rtc_write32(CMOS_OFFSET_MRC_SEED_S3, pei_data->scrambler_seed_s3);
+	debug("Save s3 scrambler seed 0x%08x to CMOS 0x%02x\n",
+	      pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+
+	/* Save a simple checksum of the seed values */
+	c1 = compute_ip_checksum((u8 *)&pei_data->scrambler_seed,
+				 sizeof(u32));
+	c2 = compute_ip_checksum((u8 *)&pei_data->scrambler_seed_s3,
+				 sizeof(u32));
+	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+
+	rtc_write8(CMOS_OFFSET_MRC_SEED_CHK, checksum & 0xff);
+	rtc_write8(CMOS_OFFSET_MRC_SEED_CHK + 1, (checksum >> 8) & 0xff);
+
+	return 0;
+}
+
+static int sdram_save_mrc_data(void)
+{
+	struct mrc_data_container *data;
+	struct fmap_entry entry;
+	struct spi_flash *sf;
+	int ret;
+
+	if (!gd->arch.mrc_output_len)
+		return 0;
+	debug("Saving %d bytes of MRC output data to SPI flash\n",
+	      gd->arch.mrc_output_len);
+
+	ret = get_mrc_entry(&sf, &entry);
+	if (ret)
+		goto err_entry;
+	ret = build_mrc_data(&data);
+	if (ret)
+		goto err_data;
+	ret = mrccache_update(sf, &entry, data);
+	if (!ret)
+		debug("Saved MRC data with checksum %04x\n", data->checksum);
+
+	free(data);
+err_data:
+	spi_flash_free(sf);
+err_entry:
+	if (ret)
+		debug("%s: Failed: %d\n", __func__, ret);
+	return ret;
+}
+
+/* Use this hook to save our SDRAM parameters */
+int misc_init_r(void)
+{
+	int ret;
+
+	ret = sdram_save_mrc_data();
+	if (ret)
+		printf("Unable to save MRC data: %d\n", ret);
+
+	return 0;
+}
+
 static const char *const ecc_decoder[] = {
 	"inactive",
 	"active on IO",
@@ -142,6 +347,11 @@ static asmlinkage void console_tx_byte(unsigned char byte)
 #endif
 }
 
+static int recovery_mode_enabled(void)
+{
+	return false;
+}
+
 /**
  * Find the PEI executable in the ROM and execute it.
  *
@@ -166,6 +376,17 @@ int sdram_initialise(struct pei_data *pei_data)
 
 	debug("Starting UEFI PEI System Agent\n");
 
+	/*
+	 * Do not pass MRC data in for recovery mode boot,
+	 * Always pass it in for S3 resume.
+	 */
+	if (!recovery_mode_enabled() ||
+	    pei_data->boot_mode == PEI_BOOT_RESUME) {
+		ret = prepare_mrc_cache(pei_data);
+		if (ret)
+			debug("prepare_mrc_cache failed: %d\n", ret);
+	}
+
 	/* If MRC data is not found we cannot continue S3 resume. */
 	if (pei_data->boot_mode == PEI_BOOT_RESUME && !pei_data->mrc_input) {
 		debug("Giving up in sdram_initialize: No MRC data\n");
@@ -216,6 +437,8 @@ int sdram_initialise(struct pei_data *pei_data)
 	debug("System Agent Version %d.%d.%d Build %d\n",
 	      version >> 24 , (version >> 16) & 0xff,
 	      (version >> 8) & 0xff, version & 0xff);
+	debug("MCR output data length %#x at %p\n", pei_data->mrc_output_len,
+	      pei_data->mrc_output);
 
 	/*
 	 * Send ME init done for SandyBridge here.  This is done inside the
@@ -231,6 +454,36 @@ int sdram_initialise(struct pei_data *pei_data)
 	post_system_agent_init(pei_data);
 	report_memory_config();
 
+	/* S3 resume: don't save scrambler seed or MRC data */
+	if (pei_data->boot_mode != PEI_BOOT_RESUME) {
+		/*
+		 * This will be copied to SDRAM in reserve_arch(), then written
+		 * to SPI flash in sdram_save_mrc_data()
+		 */
+		gd->arch.mrc_output = (char *)pei_data->mrc_output;
+		gd->arch.mrc_output_len = pei_data->mrc_output_len;
+		ret = write_seeds_to_cmos(pei_data);
+		if (ret)
+			debug("Failed to write seeds to CMOS: %d\n", ret);
+	}
+
+	return 0;
+}
+
+int reserve_arch(void)
+{
+	u16 checksum;
+
+	checksum = compute_ip_checksum(gd->arch.mrc_output,
+				       gd->arch.mrc_output_len);
+	debug("Saving %d bytes for MRC output data, checksum %04x\n",
+	      gd->arch.mrc_output_len, checksum);
+	gd->start_addr_sp -= gd->arch.mrc_output_len;
+	memcpy((void *)gd->start_addr_sp, gd->arch.mrc_output,
+	       gd->arch.mrc_output_len);
+	gd->arch.mrc_output = (char *)gd->start_addr_sp;
+	gd->start_addr_sp &= ~0xf;
+
 	return 0;
 }
 
diff --git a/arch/x86/include/asm/arch-ivybridge/mrccache.h b/arch/x86/include/asm/arch-ivybridge/mrccache.h
new file mode 100644
index 0000000..968b2ef
--- /dev/null
+++ b/arch/x86/include/asm/arch-ivybridge/mrccache.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2014 Google, Inc
+ *
+ * SPDX-License-Identifier:	GPL-2.0+
+ */
+
+#ifndef _ASM_ARCH_MRCCACHE_H
+#define _ASM_ARCH_MRCCACHE_H
+
+#define MRC_DATA_ALIGN		0x1000
+#define MRC_DATA_SIGNATURE	(('M' << 0) | ('R' << 8) | ('C' << 16) | \
+					('D'<<24))
+
+__packed struct mrc_data_container {
+	u32	signature;	/* "MRCD" */
+	u32	data_size;	/* Size of the 'data' field */
+	u32	checksum;	/* IP style checksum */
+	u32	reserved;	/* For header alignment */
+	u8	data[0];	/* Variable size, platform/run time dependent */
+};
+
+struct fmap_entry;
+struct spi_flash;
+
+/**
+ * mrccache_find_current() - find the latest MRC cache record
+ *
+ * This searches the MRC cache region looking for the latest record to use
+ * for setting up SDRAM
+ *
+ * @entry:	Information about the position and size of the MRC cache
+ * @return pointer to latest record, or NULL if none
+ */
+struct mrc_data_container *mrccache_find_current(struct fmap_entry *entry);
+
+/**
+ * mrccache_update() - update the MRC cache with a new record
+ *
+ * This writes a new record to the end of the MRC cache. If the new record is
+ * the same as the latest record then the write is skipped
+ *
+ * @sf:		SPI flash to write to
+ * @entry:	Position and size of MRC cache in SPI flash
+ * @cur:	Record to write
+ * @return 0 if updated, -EEXIST if the record is the same as the latest
+ * record, other error if SPI write failed
+ */
+int mrccache_update(struct spi_flash *sf, struct fmap_entry *entry,
+		    struct mrc_data_container *cur);
+
+#endif
diff --git a/arch/x86/include/asm/global_data.h b/arch/x86/include/asm/global_data.h
index 243ed5c..5ee06eb 100644
--- a/arch/x86/include/asm/global_data.h
+++ b/arch/x86/include/asm/global_data.h
@@ -65,6 +65,9 @@ struct arch_global_data {
 	struct mtrr_request mtrr_req[MAX_MTRR_REQUESTS];
 	int mtrr_req_count;
 	int has_mtrr;
+	/* MRC training data to save for the next boot */
+	char *mrc_output;
+	unsigned int mrc_output_len;
 };
 
 #endif