/* -*- linux-c -*- */
/*
 *   Copyright (c) International Business Machines  Corp., 2000
 *
 *   This program is free software;  you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
 *   the GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program;  if not, write to the Free Software
 *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
/* linux/driver/evms/evms_bbr.c
 *
 * EVMS - Bad Block Relocation (BBR) Feature Plugin
 *
 * BBR feature is designed to remap I/O write failures to another safe location
 * on disk. Note that most disk drives have BBR built into them, this means
 * that our software BBR will be only activated when all hardware BBR
 * replacement sectors have been used.
 */

#define LOG_PREFIX "bbr: "

#include <linux/config.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mempool.h>
#include <linux/init.h>
#include <asm/uaccess.h>

#include <linux/evms/evms.h>
#include <linux/evms/evms_bbr_k.h>

/* API prototypes. */
static int bbr_discover(struct evms_logical_node ** discover_list);
static int bbr_delete(struct evms_logical_node * node);
static void bbr_read(struct evms_logical_node * node, struct buffer_head * bh);
static void bbr_write(struct evms_logical_node * node, struct buffer_head * bh);
static int bbr_ioctl(struct evms_logical_node * bbr_node, struct inode * inode,
		     struct file * file, unsigned int cmd, unsigned long arg);
static int bbr_direct_ioctl(struct inode * inode, struct file * file,
			    unsigned int cmd, unsigned long arg);
static int bbr_init_io(struct evms_logical_node * bbr_node, int io_flag,
		       u64 startLSN, u64 nr_sects, void * bufptr);

/* Other function prototypes. */
static int bbr_create_pools(void);
static u32 bbr_table_to_remap_list(struct bbr_private * bbr_id);
static void bbr_io_handler(void * void_data);
static void bbr_free_private(struct bbr_private * bbr_id);
static inline void bbr_list_add(struct bbr_private * bbr_id);

/* List of all BBR nodes. */
static struct bbr_private * bbr_instances = NULL;

/* Data pertaining to the I/O thread. */
static struct evms_thread * bbr_io_thread = NULL;
static spinlock_t bbr_io_list_lock = SPIN_LOCK_UNLOCKED;
static LIST_HEAD(bbr_io_list);

/* Global pools for bbr_io_buf's and bbr_remap's. */
kmem_cache_t * bbr_io_buf_slab;
mempool_t * bbr_io_buf_pool;
kmem_cache_t * bbr_bh_slab;
mempool_t * bbr_bh_pool;
kmem_cache_t * bbr_remap_slab;
mempool_t * bbr_remap_pool;

/* Plugin function table and header. */
static struct evms_plugin_fops function_table = {
	.discover	= bbr_discover,
	.delete		= bbr_delete,
	.read		= bbr_read,
	.write		= bbr_write,
	.init_io	= bbr_init_io,
	.ioctl		= bbr_ioctl,
	.direct_ioctl	= bbr_direct_ioctl
};

static struct evms_plugin_header plugin_header = {
	.id = SetPluginID(IBM_OEM_ID,
			  EVMS_FEATURE,
			  EVMS_BBR_FEATURE_ID),
	.version = {
		.major		= EVMS_BBR_VERSION_MAJOR,
		.minor		= EVMS_BBR_VERSION_MINOR,
		.patchlevel	= EVMS_BBR_VERSION_PATCHLEVEL
	},
	.required_services_version = {
		.major		= EVMS_BBR_COMMON_SERVICES_MAJOR,
		.minor		= EVMS_BBR_COMMON_SERVICES_MINOR,
		.patchlevel	= EVMS_BBR_COMMON_SERVICES_PATCHLEVEL
	},
	.fops = &function_table
};

/**
 * le_meta_data_to_cpu
 *
 * Convert bbr meta data from on-disk (LE) format
 * to the native cpu endian format.
 */
void le_meta_data_to_cpu(struct evms_bbr_metadata * md)
{
	md->signature		   = le32_to_cpup(&md->signature);
	md->crc			   = le32_to_cpup(&md->crc);
	md->block_size		   = le32_to_cpup(&md->block_size);
	md->flags		   = le32_to_cpup(&md->flags);
	md->sequence_number	   = le64_to_cpup(&md->sequence_number);
	md->start_sect_bbr_table   = le64_to_cpup(&md->start_sect_bbr_table);
	md->nr_sects_bbr_table	   = le64_to_cpup(&md->nr_sects_bbr_table);
	md->start_replacement_sect = le64_to_cpup(&md->start_replacement_sect);
	md->nr_replacement_blks	   = le64_to_cpup(&md->nr_replacement_blks);
}

/**
 * le_bbr_table_sector_to_cpu 
 *
 * Convert bbr meta data from on-disk (LE) format
 * to the native cpu endian format.
 */
void le_bbr_table_sector_to_cpu(struct evms_bbr_table * p)
{
	int i;
	p->signature		= le32_to_cpup(&p->signature);
	p->crc			= le32_to_cpup(&p->crc);
	p->sequence_number	= le32_to_cpup(&p->sequence_number);
	p->in_use_cnt		= le32_to_cpup(&p->in_use_cnt);
	for ( i = 0; i < EVMS_BBR_ENTRIES_PER_SECT; i++ ) {
		p->entries[i].bad_sect =
			le64_to_cpup(&p->entries[i].bad_sect);
		p->entries[i].replacement_sect =
			le64_to_cpup(&p->entries[i].replacement_sect);
	}
}

/**
 * cpu_bbr_table_sector_to_le
 *
 * Convert bbr meta data from cpu endian format to on-disk (LE) format
 */
void cpu_bbr_table_sector_to_le(struct evms_bbr_table * p,
				struct evms_bbr_table * le)
{
	int i;
	le->signature		= cpu_to_le32p(&p->signature);
	le->crc			= cpu_to_le32p(&p->crc);
	le->sequence_number	= cpu_to_le32p(&p->sequence_number);
	le->in_use_cnt		= cpu_to_le32p(&p->in_use_cnt);
	for ( i = 0; i < EVMS_BBR_ENTRIES_PER_SECT; i++ ) {
		le->entries[i].bad_sect =
			cpu_to_le64p(&p->entries[i].bad_sect);
		le->entries[i].replacement_sect =
			cpu_to_le64p(&p->entries[i].replacement_sect);
	}
}

#ifdef EVMS_BBR_DEBUG
static void print_meta_data(struct evms_bbr_metadata * md)
{
	LOG_DEBUG("BBR Metadata Sector:\n"
		  "  signature               0x%08X\n"
		  "  crc                     0x%08X\n"
		  "  block_size              %u\n"
		  "  start_sect_bbr_table    "PFU64"\n"
		  "  nr_sects_bbr_table      "PFU64"\n"
		  "  start_replacement_sect  "PFU64"\n"
		  "  nr_replacement_blks     "PFU64"\n",
		  md->signature, md->crc, md->block_size,
		  md->start_sect_bbr_table, md->nr_sects_bbr_table,
		  md->start_replacement_sect, md->nr_replacement_blks);
}

static void print_bbr_table_sector(struct evms_bbr_table * p)
{
	int i;
	LOG_DEBUG("BBR Table Sector:\n"
		  "  sig          0x%08X\n"
		  "  crc          0x%08X\n"
		  "  sequence     %u\n"
		  "  in_use_cnt   %u\n"
		  "  Table Entries:\n",
		  p->signature, p->crc, p->sequence_number, p->in_use_cnt);
	for ( i = 0; i < EVMS_BBR_ENTRIES_PER_SECT; i++ ) {
		LOG_DEBUG("  [%d] bad_sect: "PFU64"     replacement_sect: "PFU64"\n",
			  i, p->entries[i].bad_sect,
			  p->entries[i].replacement_sect);
	}
}

void print_binary_tree(struct bbr_runtime_remap * node)
{
	if (node) {
		LOG_DEFAULT("["PFU64","PFU64"]\n", node->remap.bad_sect,
			    node->remap.replacement_sect);
		print_binary_tree(node->left);
		print_binary_tree(node->right);
	}
}

static void print_remap_list(struct bbr_private * bbr_id)
{
	if (bbr_id->remap_root) {
		LOG_DEFAULT("%s for %s\n", __FUNCTION__, bbr_id->node->name);
		print_binary_tree(bbr_id->remap_root);
	}
}
#endif

/**
 * validate_bbr_table_sector
 *
 * Check the specified BBR table sector for a valid signature and CRC.
 */
static int validate_bbr_table_sector(struct evms_bbr_table * p)
{
	int rc = 0;
	int org_crc, final_crc;

	if ( le32_to_cpup(&p->signature) != EVMS_BBR_TABLE_SIGNATURE ) {
		LOG_ERROR("BBR table signature doesn't match!\n");
		LOG_ERROR("Sector has (0x%08X) expected(0x%08X)\n",
			  le32_to_cpup(&p->signature),	
			  EVMS_BBR_TABLE_SIGNATURE);
		rc = -EINVAL;
	} else {
		if (p->crc) {
			org_crc = le32_to_cpup(&p->crc);
			p->crc = 0;
			final_crc = evms_cs_calculate_crc(EVMS_INITIAL_CRC, p,
							  sizeof(*p));
			if ( final_crc != org_crc ) {
				LOG_ERROR("CRC failed!\n");
				LOG_ERROR("Sector has (0x%08X) calculated(0x%08X)\n",
					  org_crc, final_crc);
				rc = -EINVAL;
			}
			p->crc = cpu_to_le32p(&org_crc);
		} else {
			LOG_ERROR("BBR table sector has no CRC!\n");
			rc = -EINVAL;
		}
	}
	if (rc)
		BBR_DEBUG_PRINT_TABLE_SECTOR(p);
	le_bbr_table_sector_to_cpu(p);
	return rc;
}

/**
 * update_invalid_bbr_table_sector
 *
 * If one copy of a BBR table sector is bad, replace it with the valid copy.
 */
void update_invalid_bbr_table_sector(struct evms_logical_node * node,
				     struct evms_bbr_table * valid,
				     struct evms_bbr_table * invalid,
				     u64 lsn)
{
	int rc;
	struct evms_bbr_table * tmp_bbr_table;

	/* Correct the invalid bbr table sector */
	memcpy(invalid, valid, sizeof(struct evms_bbr_table));

	/* Allocate memory for I/O */
	tmp_bbr_table = kmalloc(sizeof(struct evms_bbr_table), GFP_KERNEL);
	if (tmp_bbr_table) {
		memset(tmp_bbr_table, 0, sizeof(struct evms_bbr_table));
		cpu_bbr_table_sector_to_le(valid, tmp_bbr_table);
		LOG_WARNING("Correcting BBR table sector "PFU64"\n", lsn);
		rc = INIT_IO(node, 1, lsn, 1, tmp_bbr_table);
		if (rc) {
			LOG_ERROR("Could not correct BBR table sector "PFU64".\n",
				  lsn);
		}
		kfree(tmp_bbr_table);
	}
}

/**
 * validate_bbr_table
 *
 * Validate the entire range of sectors in the BBR table.
 */
static u32 validate_bbr_table(struct evms_bbr_metadata * md,
			      struct evms_bbr_table * p)
{
	u32 i, nr_sects;

	nr_sects = md->nr_sects_bbr_table;

	for ( i = 0; i < nr_sects; i++, p++ ) {
		if ( validate_bbr_table_sector(p) )
			break;
	}

	if ( i != nr_sects ) {
		LOG_SERIOUS("Stopped BBR table validation at sector %u.\n", i);
		nr_sects = i;
	}
	LOG_DEBUG("Validated %u BBR table sectors.\n", nr_sects);
	return nr_sects;
}

/**
 * validate_bbr_tables
 * @node:	BBR node to validate.
 * @MD1:	Primary metadata sector.
 * @MD2:	Secondary metadata sector.
 * @p1:		Primary BBR table.
 * @p2:		Secondary BBR table.
 *
 * Validate both copies of the BBR table. If one of them is invalid, 
 * try to correct the errors using the valid copy.
 */
static u32 validate_bbr_tables(struct evms_logical_node * node,
			       struct evms_bbr_metadata * MD1,
			       struct evms_bbr_metadata * MD2,
			       struct evms_bbr_table * p1,
			       struct evms_bbr_table * p2)
{
	u32 i, rc1, rc2, nr_sects;

	nr_sects = MD1->nr_sects_bbr_table;
	if ( nr_sects != MD2->nr_sects_bbr_table ) {
		nr_sects = (nr_sects < MD2->nr_sects_bbr_table) ?
			   nr_sects : MD2->nr_sects_bbr_table;
		LOG_SERIOUS("Size of BBR tables don't match. Using %u\n",
			    nr_sects);
	}

	for ( i = 0; i < nr_sects; i++, p1++, p2++ ) {
		rc1 = validate_bbr_table_sector(p1);
		if (rc1) {
			LOG_WARNING("Invalid BBR table sector at "PFU64".\n",
				    MD1->start_sect_bbr_table + i);
		}
		rc2 = validate_bbr_table_sector(p2);
		if (rc2) {
			LOG_WARNING("Invalid BBR table sector at "PFU64".\n",
				    MD2->start_sect_bbr_table + i);
		}

		/* Correct BBR table errors. */
		if (rc1 && rc2) {
			/* Cannot fix. */
			break;
		} else if (rc1) {
			update_invalid_bbr_table_sector(node, p2, p1,
							MD1->start_sect_bbr_table + i);
			continue;
		} else if (rc2) {
			update_invalid_bbr_table_sector(node, p1, p2,
							MD2->start_sect_bbr_table + i);
			continue;
		}

		if ( p1->sequence_number != p2->sequence_number ) {
			LOG_WARNING("Sequence numbers for BBR table index %u don't match.\n", i);
			LOG_WARNING("MD1 sequence_nr=%u, MD2 sequence_nr_2=%u\n",
				    p1->sequence_number, p2->sequence_number);
			if ( p1->sequence_number < p2->sequence_number ) {
				update_invalid_bbr_table_sector(node, p2, p1,
								MD1->start_sect_bbr_table + i);
			} else {
				update_invalid_bbr_table_sector(node, p1, p2,
								MD2->start_sect_bbr_table + i);
			}
		}
	}
	if ( i != nr_sects ) {
		LOG_SERIOUS("Stopped validation at sector %u\n", i);
		nr_sects = i;
	}
	LOG_DEBUG("Validated %u BBR table sectors.\n", nr_sects);
	return nr_sects;
}

/**
 * validate_meta_data
 *
 * Check the specified BBR metadata sector for a valid signature and CRC.
 */
static int validate_meta_data(struct evms_bbr_metadata * md)
{
	int org_crc, final_crc;

	BBR_DEBUG_PRINT_META_DATA(md);

	if ( le32_to_cpup(&md->signature) != EVMS_BBR_SIGNATURE ) {
		LOG_SERIOUS("BBR signature doesn't match!\n");
		LOG_SERIOUS("Found: 0x%08X  Expecting: 0x%08X\n",
			    le32_to_cpup(&md->signature), EVMS_BBR_SIGNATURE);
		return -EINVAL;
	}

	if (md->crc) {
		org_crc = le32_to_cpup(&md->crc);
		md->crc = 0;
		final_crc = evms_cs_calculate_crc(EVMS_INITIAL_CRC, md,
						  sizeof(*md));
		if ( final_crc != org_crc ) {
			LOG_ERROR("CRC failed!\n");
			LOG_ERROR("Sector has (0x%08X) calculated(0x%08X)\n",
				    org_crc, final_crc);
			return -EINVAL;
		}
		md->crc = cpu_to_le32p(&org_crc);
	} else {
		LOG_WARNING("Metadata sector has no CRC!\n");
	}

	le_meta_data_to_cpu(md);
	return 0;
}

/**
 * bbr_load_meta_data
 * @node:	BBR node to read metadata from.
 * @lsn:	Sector to read metadata from.
 * @md:		Pointer to return metadata structure.
 * @bbr_table:	Pointer to return BBR table.
 *
 * Load one copy of the BBR metadata. If the metadata is valid, load the
 * corresponding copy of the BBR table.
 */
static int load_meta_data(struct evms_logical_node * node,
			  u64 lsn,
			  struct evms_bbr_metadata ** md,
			  struct evms_bbr_table ** bbr_table)
{
	int rc;

	*md = NULL;
	*bbr_table = NULL;

	if (!lsn) {
		LOG_WARNING("No sector specified for BBR metadata on %s.\n",
			    node->name);
		return -ENODATA;
	}

	/* Allocate a buffer for the metadata sector. */
	*md = kmalloc(sizeof(struct evms_bbr_metadata), GFP_KERNEL);
	if (!*md) {
		LOG_ERROR("kmalloc error creating metadata buffer for %s.\n",
			  node->name);
		return -ENOMEM;
	}

	/* Read the metadata sector. */
	rc = INIT_IO(node, 0, lsn, 1, *md);
	if (rc) {
		LOG_ERROR("init_io error on %s.\n", node->name);
		kfree(*md);
		*md = NULL;
		return rc;
	}

	/* Validate the metadata sector. */
	rc = validate_meta_data(*md);
	if (rc) {
		LOG_ERROR("Error validating metadata for %s.\n", node->name);
		kfree(*md);
		*md = NULL;
		return rc;
	}

	/* Allocate a buffer for the BBR table. */
	*bbr_table = kmalloc((*md)->nr_sects_bbr_table <<
			     EVMS_VSECTOR_SIZE_SHIFT, GFP_KERNEL);
	if (!*bbr_table) {
		LOG_ERROR("kmalloc error creating BBR table buffer for %s.\n",
			  node->name);
		kfree(*md);
		*md = NULL;
		return -ENOMEM;
	}

	/* Read the BBR table but don't validate here. */
	rc = INIT_IO(node, 0, (*md)->start_sect_bbr_table,
		     (*md)->nr_sects_bbr_table, *bbr_table);
	if (rc) {
		LOG_ERROR("init_io error on %s.\n", node->name);
		kfree(*md);
		*md = NULL;
		kfree(*bbr_table);
		*bbr_table = NULL;
	}

	return rc;
}

/**
 * bbr_load_feature_data
 * @node:	BBR node
 * @ID:		Return pointer to BBR private data.
 *
 * Load both copies of the BBR metadata and table. If one is invalid, try
 * to correct is using the valid copy. When a valid copy is found, create
 * a private data structure for the specified node.
 */
static int load_feature_data(struct evms_logical_node * node,
			     struct bbr_private ** ID)
{
	struct evms_bbr_metadata * md1 = NULL;
	struct evms_bbr_metadata * md2 = NULL;
	struct evms_bbr_table * table1 = NULL;
	struct evms_bbr_table * table2 = NULL;
	u64 lba_table1 = 0, lba_table2 = 0;
	u32 nr_sects = 0;
	int rc = 0, rc1, rc2;

	*ID = NULL;

	/* Load metadata 1 */
	rc1 = load_meta_data(node,
			     node->feature_header->feature_data1_start_lsn,
			     &md1, &table1);
	/* Load metadata 2 */
	rc2 = load_meta_data(node,
			     node->feature_header->feature_data2_start_lsn,
			     &md2, &table2);

	if (rc1 && rc2) {
		/* Both copies are bad? Cannot continue. */
		rc = -ENODATA;
	} else if (rc1 || rc2) {
		/* One copy is bad. Use the good copy. */
		if (rc1) {
			lba_table2 = md2->start_sect_bbr_table;
			kfree(table1);
			kfree(md1);
			table1 = table2;
			table2 = NULL;
			md1 = md2;
			md2 = NULL;
		} else {
			lba_table1 = md1->start_sect_bbr_table;
		}

		nr_sects = validate_bbr_table(md1, table1);
		if ( nr_sects == 0 ) {
			rc = -ENODATA;
		}
	} else {
		lba_table1 = md1->start_sect_bbr_table;
		lba_table2 = md2->start_sect_bbr_table;
		nr_sects = validate_bbr_tables(node, md1, md2, table1, table2);
		if ( nr_sects == 0 ) {
			rc = -ENODATA;
		}
	}

	if (!rc && nr_sects) {
		*ID = kmalloc(sizeof(struct bbr_private), GFP_KERNEL);
		if (*ID) {
			memset(*ID, 0, sizeof(struct bbr_private));
			(*ID)->source = node;
			(*ID)->blksize_in_sects = md1->block_size >>
						  EVMS_VSECTOR_SIZE_SHIFT;
			(*ID)->remap_root = NULL;
			(*ID)->lba_table1 = lba_table1;
			(*ID)->lba_table2 = lba_table2;
			(*ID)->bbr_table = table1;
			(*ID)->nr_sects_bbr_table = nr_sects;
			if ( nr_sects < md1->nr_sects_bbr_table ) {
				LOG_WARNING("Making BBR node read-only\n");
				(*ID)->flag |= EVMS_VOLUME_READ_ONLY;
			}
			(*ID)->nr_replacement_blks = nr_sects *
						     EVMS_BBR_ENTRIES_PER_SECT;
			(*ID)->start_replacement_sect = md1->start_replacement_sect;
			atomic_set(&(*ID)->in_use_replacement_blks, 0);
			(*ID)->bbr_id_lock = SPIN_LOCK_UNLOCKED;
			if ( !bbr_remap_pool || !bbr_io_buf_pool ) {
				rc = bbr_create_pools();
			}
			if (!rc) {
				atomic_set(&(*ID)->in_use_replacement_blks,
					   bbr_table_to_remap_list(*ID));
			}
		} else {
			rc = -ENOMEM;
		}
	}

	if (!rc) {
		if (!bbr_io_thread) {
			const char * name = "evms_bbr_io";
			bbr_io_thread = evms_cs_register_thread(bbr_io_handler,
								NULL, name);
			if (!bbr_io_thread) {
				rc = -EINVAL;
			}
		}
	}

	/* If error, free table1. */
	if (rc)	{
		if (table1) {
			kfree(table1);
		}
		if (*ID) {
			(*ID)->bbr_table = NULL;
			bbr_free_private(*ID);
			(*ID) = NULL;
		}
	}

	/* Will never use md1, md2 and table2 again */
	if (md1) {
		kfree(md1);
	}
	if (md2) {
		kfree(md2);
	}
	if (table2) {
		kfree(table2);
	}

	return rc;
}

/**
 * bbr_binary_tree_insert
 *
 * Insert a node into the binary tree.
 */
void bbr_binary_tree_insert(struct bbr_runtime_remap ** root,
			    struct bbr_runtime_remap * newnode)
{
	struct bbr_runtime_remap ** node = root;
	while (node && *node) {
		if ( newnode->remap.bad_sect > (*node)->remap.bad_sect ) {
			node = &((*node)->right);
		} else {
			node = &((*node)->left);
		}
	}
	
	newnode->left = newnode->right = NULL;
	*node = newnode;
}

/**
 * bbr_binary_search
 *
 * Search for a node that contains bad_sect = lsn.
 */
struct bbr_runtime_remap * bbr_binary_search(struct bbr_runtime_remap * root,
					     u64 lsn)
{
	struct bbr_runtime_remap * node = root;
	while (node) {
		if (node->remap.bad_sect == lsn) {
			break;
		}
		if ( lsn > node->remap.bad_sect ) {
			node = node->right;
		} else {
			node = node->left;
		}
	}
	return node;
}

/**
 * bbr_binary_tree_destroy
 *
 * Destroy the binary tree.
 */
void bbr_binary_tree_destroy(struct bbr_runtime_remap * root,
			     struct bbr_private * bbr_id)
{
	struct bbr_runtime_remap ** link = NULL;
	struct bbr_runtime_remap * node = root;

	while (node) {
		if (node->left) {
			link = &(node->left);
			node = node->left;
			continue;
		}
		if (node->right) {
			link = &(node->right);
			node = node->right;
			continue;
		}

		mempool_free(node, bbr_remap_pool);
		if (node == root) {
			/* If root is deleted, we're done. */
			break;
		}

		/* Back to root. */
		node = root;
		*link = NULL;
	}
}

static void bbr_free_remap(struct bbr_private * bbr_id)
{
	unsigned long flags;
	spin_lock_irqsave(&bbr_id->bbr_id_lock, flags);	
	bbr_binary_tree_destroy(bbr_id->remap_root, bbr_id);
	bbr_id->remap_root = NULL;
	spin_unlock_irqrestore(&bbr_id->bbr_id_lock, flags);
}

/**
 * bbr_insert_remap_entry
 *
 * Create a new remap entry and add it to the binary tree for this node.
 */
static int bbr_insert_remap_entry(struct bbr_private * bbr_id,
				  struct evms_bbr_table_entry * new_bbr_entry)
{
	struct bbr_runtime_remap * newnode = NULL;
	unsigned long flags;
	int rc;

	newnode = mempool_alloc(bbr_remap_pool, GFP_NOIO);
	if (!newnode) {
		rc = -ENOMEM;
		LOG_SERIOUS("Could not allocate from remap pool! (rc=%d)\n", rc);
		return rc;
	}
	newnode->remap.bad_sect  = new_bbr_entry->bad_sect;
	newnode->remap.replacement_sect = new_bbr_entry->replacement_sect;
	spin_lock_irqsave(&bbr_id->bbr_id_lock, flags);	
	bbr_binary_tree_insert(&bbr_id->remap_root, newnode);
	spin_unlock_irqrestore(&bbr_id->bbr_id_lock, flags);
	return 0;
}

/**
 * bbr_table_to_remap_list
 *
 * The on-disk bbr table is sorted by the replacement sector LBA. In order to
 * improve run time performance, the in memory remap list must be sorted by
 * the bad sector LBA. This function is called at discovery time to initialize
 * the remap list. This function assumes that at least one copy of meta data
 * is valid.
 */
static u32 bbr_table_to_remap_list(struct bbr_private * bbr_id)
{
	u32 in_use_blks = 0;
	int i, j;
	struct evms_bbr_table * p;
	

	for ( i = 0, p = bbr_id->bbr_table;
	      i < bbr_id->nr_sects_bbr_table;
	      i++, p++ ) {
		if (!p->in_use_cnt) {
			break;
		}
		in_use_blks += p->in_use_cnt;
		for ( j = 0; j < p->in_use_cnt; j++ ) {
			bbr_insert_remap_entry(bbr_id, &p->entries[j]);
		}
	}

	return in_use_blks;
}

/**
 * bbr_search_remap_entry
 *
 * Search remap entry for the specified sector. If found, return a pointer to
 * the table entry. Otherwise, return NULL.
 */
static struct evms_bbr_table_entry * bbr_search_remap_entry(struct bbr_private * bbr_id,
							    u64 lsn)
{
	struct bbr_runtime_remap * p;
	unsigned long flags;

	spin_lock_irqsave(&bbr_id->bbr_id_lock, flags);
	p = bbr_binary_search(bbr_id->remap_root, lsn);
	spin_unlock_irqrestore(&bbr_id->bbr_id_lock, flags);
	if (p) {
		return (&p->remap);
	} else {
		return NULL;
	}
}

/**
 * bbr_remap
 *
 * If *lsn is in the remap table, return TRUE and modify *lsn,
 * else, return FALSE.
 */
static inline int bbr_remap(struct bbr_private * bbr_id,
			    u64 * lsn)
{
	struct evms_bbr_table_entry *e;

	if ( atomic_read(&bbr_id->in_use_replacement_blks) && 
	     ! (bbr_id->flag & BBR_STOP_REMAP) ) {
		e = bbr_search_remap_entry(bbr_id, *lsn);
		if (e) {
			*lsn = e->replacement_sect;
			LOG_EXTRA("%s replacement sector (LSN="PFU64")\n",
				  __FUNCTION__, *lsn);
			return TRUE;
		}
	}
	return FALSE;
}

/**
 * bbr_remap_probe
 *
 * If any of the sectors in the range [lsn, lsn+nr_sects] are in the remap
 * table return TRUE, Else, return FALSE.
 */
static inline int bbr_remap_probe(struct bbr_private * bbr_id,
				  u64 lsn, u64 nr_sects)
{
	u64 tmp, cnt;

	if ( atomic_read(&bbr_id->in_use_replacement_blks) &&
	     ! (bbr_id->flag & BBR_STOP_REMAP) ) {
		for ( cnt = 0, tmp = lsn;
		      cnt < nr_sects;
		      cnt += bbr_id->blksize_in_sects, tmp = lsn + cnt) {
			if ( bbr_remap(bbr_id,&tmp) ) {
				return TRUE;
			}
		}
	}
	return FALSE;
}

static void *bbr_slab_pool_alloc(int gfp_mask, void * data)
{
	return kmem_cache_alloc(data, gfp_mask);
}

static void bbr_slab_pool_free(void *ptr, void * data)
{
	kmem_cache_free(data, ptr);
}

static void bbr_destroy_pools(void)
{
	if (bbr_io_buf_pool) {
		mempool_destroy(bbr_io_buf_pool);
		bbr_io_buf_pool = NULL;
	}
	if (bbr_io_buf_slab) {
		kmem_cache_destroy(bbr_io_buf_slab);
		bbr_io_buf_slab = NULL;
	}
	if (bbr_bh_pool) {
		mempool_destroy(bbr_bh_pool);
		bbr_bh_pool = NULL;
	}
	if (bbr_bh_slab) {
		kmem_cache_destroy(bbr_bh_slab);
		bbr_bh_slab = NULL;
	}
	if (bbr_remap_pool) {
		mempool_destroy(bbr_remap_pool);
		bbr_remap_pool = NULL;
	}
	if (bbr_remap_slab) {
		kmem_cache_destroy(bbr_remap_slab);
		bbr_remap_slab = NULL;
	}
}

static int bbr_create_pools(void)
{
	/* Create a memory pool for the remap list. */
	bbr_remap_slab = kmem_cache_create("BBR_Remap_Slab",
					   sizeof(struct bbr_runtime_remap),
					   0, SLAB_HWCACHE_ALIGN, NULL, NULL);
	if (!bbr_remap_slab) {
		LOG_ERROR("Unable to create BBR remap cache.");
		goto error;
	}

	bbr_remap_pool = mempool_create(64, bbr_slab_pool_alloc,
					bbr_slab_pool_free, bbr_remap_slab);
	if (!bbr_remap_pool) {
		LOG_ERROR("Unable to create BBR remap pool.");
		goto error;
	}

	/* Create a memory pool for cloning buffer-heads. */
	bbr_bh_slab = kmem_cache_create("BBR_BH_Slab",
					sizeof(struct buffer_head),
					0, SLAB_HWCACHE_ALIGN, NULL, NULL);
	if (!bbr_bh_slab) {
		LOG_ERROR("Unable to create BBR BH cache.");
		goto error;
	}

	bbr_bh_pool = mempool_create(256, bbr_slab_pool_alloc,
				     bbr_slab_pool_free, bbr_bh_slab);
	if (!bbr_bh_pool) {
		LOG_ERROR("Unable to create BBR BH pool.");
		goto error;
	}

	/* Create a memory pool for the BBR I/O anchors. */
	bbr_io_buf_slab = kmem_cache_create("BBR_IO_Buf_Slab",
					    sizeof(struct bbr_io_buffer),
					    0, SLAB_HWCACHE_ALIGN, NULL, NULL);
	if (!bbr_io_buf_slab) {
		LOG_ERROR("Unable to create BBR I/O buffer cache.");
		goto error;
	}

	bbr_io_buf_pool = mempool_create(256, bbr_slab_pool_alloc,
					 bbr_slab_pool_free, bbr_io_buf_slab);
	if (!bbr_io_buf_pool) {
		LOG_ERROR("Unable to create BBR I/O buffer pool.");
		goto error;
	}

	return 0;

error:
	bbr_destroy_pools();
	return -ENOMEM;
}

/**
 * bbr_discover
 *
 * Search through the discover list looking for object with BBR metadata.
 * Remove them from the list and replace with a new BBR node.
 */
static int bbr_discover(struct evms_logical_node ** discover_list)
{
        struct evms_logical_node * node, * next_node;
        struct evms_logical_node * bbr_node = NULL;
        struct bbr_private * bbr_id;
	int bad_blocks, rc = 0;
	
	MOD_INC_USE_COUNT;

	next_node = *discover_list;
	while (next_node) {
		node = next_node;
		next_node = node->next;

		/* The node must have a BBR feature-header. */
       		if ( ! node->feature_header ||
		     node->feature_header->feature_id != plugin_header.id ) {
       			continue;
		}

		rc = load_feature_data(node, &bbr_id);
		if (rc) {
			/* Error loading feature data.
			 * This node belongs to us, but metadata is invalid,
			 * - remove it from the discovery list
			 * - delete it
			 * - clear error code then continue.
			 * Will consider creating a read only BBR node in
			 * the future.
			 */
			LOG_SERIOUS("Error in node (%s) with "PFU64" sectors.\n",
				    node->name, node->total_vsectors);
			evms_cs_remove_logical_node_from_list(discover_list,
							      node);
			DELETE(node);
			rc = 0;
			continue;
		}

		rc = evms_cs_allocate_logical_node(&bbr_node);
		if (rc) {
			LOG_SERIOUS("Could not allocate logical node! rc=%d\n", rc);
			bbr_free_private(bbr_id);
			continue;
		}

		MOD_INC_USE_COUNT;
		bbr_node->volume_info = node->volume_info;
		bbr_node->flags |= node->flags;
		bbr_node->plugin = &plugin_header;
		strcpy(bbr_node->name,
		       node->feature_header->object_name);
		bbr_node->hardsector_size = node->hardsector_size;
		bbr_node->total_vsectors = node->total_vsectors - 2 -
					   node->feature_header->feature_data1_size -
					   node->feature_header->feature_data2_size;
		bbr_node->block_size = node->block_size;
		bbr_node->private = bbr_id;
		bbr_id->node = bbr_node;

		/* Free the feature header */
		kfree(node->feature_header);
		node->feature_header = NULL;
		evms_cs_remove_logical_node_from_list(discover_list, node);

		/* If bad blocks exist, give warning */
		bad_blocks = atomic_read(&bbr_id->in_use_replacement_blks);
		if (bad_blocks) {
			BBR_DEBUG_PRINT_REMAP_LIST(bbr_id);
			LOG_WARNING("%s has %d bad blocks.\n",
				    bbr_id->source->name, bad_blocks);
			LOG_WARNING("There are "PFU64" total replacement blocks.\n",
				    bbr_id->nr_replacement_blks);
			LOG_WARNING("There are "PFU64" remaining replacement blocks.\n",
				    bbr_id->nr_replacement_blks -
				    bad_blocks);
		}

		evms_cs_add_logical_node_to_list(discover_list, bbr_node);
		bbr_list_add(bbr_id);
	}

	MOD_DEC_USE_COUNT;
	return rc;
}

static inline void bbr_list_add(struct bbr_private * bbr_id)
{
       	bbr_id->next = bbr_instances;
	bbr_instances = bbr_id;
}

static void bbr_list_remove(struct bbr_private * bbr_id)
{
	struct bbr_private ** p;

	for ( p = &bbr_instances; *p; p = &(*p)->next ) {
		if ( *p == bbr_id ) {
			*p = (*p)->next;
			break;
		}
	}
}

static struct bbr_private * bbr_find_private(char * object_name)
{
	struct bbr_private * p;

	for ( p = bbr_instances; p; p = p->next ) {
		if ( ! strncmp(p->node->name, object_name,
			       EVMS_VOLUME_NAME_SIZE) ) {
			return p;
		}
	}
	return NULL;
}

static void bbr_free_private(struct bbr_private * bbr_id)
{
	if (bbr_id->remap_root) {
		bbr_free_remap(bbr_id);
	}
	if (bbr_id->bbr_table) {
		kfree(bbr_id->bbr_table);
	}
	bbr_list_remove(bbr_id);
	kfree(bbr_id);
}

/**
 * bbr_delete
 *
 * Delete the specified BBR node and the node it is built on. If the last BBR
 * node is deleted, shut down the I/O thread.
 */
static int bbr_delete(struct evms_logical_node * bbr_node)
{
	struct bbr_private * bbr_id;
	int rc;
	
        bbr_id = bbr_node->private;

        rc = DELETE(bbr_id->source);
	if (!rc) {
		/* Now cleanup and go away */
		bbr_free_private(bbr_id);
		evms_cs_deallocate_logical_node(bbr_node);
		if (!bbr_instances) {
			bbr_destroy_pools();
			if (bbr_io_thread) {
				evms_cs_unregister_thread(bbr_io_thread);
				bbr_io_thread = NULL;
			}
		}
		MOD_DEC_USE_COUNT;
	}
        return rc;
}

static struct buffer_head * allocate_bbr_bh(struct buffer_head * org_bh)
{
	struct buffer_head * new_bh;

	new_bh = mempool_alloc(bbr_bh_pool, GFP_NOIO);
	if (!new_bh) {
		LOG_WARNING("Could not allocate from BBR BH pool!\n");
		return NULL;
	}

	memset(new_bh, 0, sizeof(*new_bh));
	new_bh->b_size = org_bh->b_size;
	new_bh->b_list = BUF_LOCKED;
	new_bh->b_dev = org_bh->b_dev;
	atomic_set(&new_bh->b_count, 1);
	new_bh->b_rdev = org_bh->b_rdev;
	set_bit(BH_Dirty, &new_bh->b_state);
	set_bit(BH_Lock, &new_bh->b_state);
	set_bit(BH_Req, &new_bh->b_state);
	set_bit(BH_Mapped, &new_bh->b_state);
	set_bit(BH_Uptodate, &new_bh->b_state);
	new_bh->b_this_page = (struct buffer_head *)1;
	new_bh->b_data = org_bh->b_data;
	new_bh->b_page = org_bh->b_page;
	new_bh->b_rsector = org_bh->b_rsector;
	init_waitqueue_head(&new_bh->b_wait);

	return new_bh;
}

static void free_bbr_bh(struct buffer_head * bh)
{
	mempool_free(bh, bbr_bh_pool);
}

static struct bbr_io_buffer * allocate_bbr_io_buf(struct bbr_private * bbr_id,
						  struct buffer_head * org_bh,
						  int rw)
{
	struct bbr_io_buffer * bbr_io_buf;

	bbr_io_buf = mempool_alloc(bbr_io_buf_pool, GFP_NOIO);
	if (bbr_io_buf) {
		memset(bbr_io_buf, 0, sizeof(struct bbr_io_buffer));
		INIT_LIST_HEAD(&bbr_io_buf->bbr_io_list);
		bbr_io_buf->bbr_id = bbr_id;
		bbr_io_buf->org_bh = org_bh;
		bbr_io_buf->rw = rw;
	} else {
		LOG_WARNING("Could not allocate from BBR I/O buffer pool!\n");
	}
	return bbr_io_buf;
}

static void free_bbr_io_buf(struct bbr_io_buffer * bbr_io_buf)
{
	mempool_free(bbr_io_buf, bbr_io_buf_pool);
}

/**
 * bbr_io_remap_error
 * @bbr_id:		Private data for the BBR node.
 * @rw:			READ or WRITE.
 * @starting_lsn:	Starting sector of request to remap.
 * @count:		Number of sectors in the request.
 * @buffer:		Data buffer for the request.
 *
 * For the requested range, try to write each sector individually. For each
 * sector that fails, find the next available remap location and write the
 * data to that new location. Then update the table and write both copies
 * of the table to disk. Finally, update the in-memory mapping and do any
 * other necessary bookkeeping.
 */
static int bbr_io_remap_error(struct bbr_private * bbr_id,
			      int rw,
			      u64 starting_lsn,
			      u64 count,
			      char * buffer )
{
	struct evms_bbr_table * bbr_table;
	unsigned long table_sector_index;
	unsigned long table_sector_offset;
	unsigned long index;
	u64 lsn, new_lsn;
	int rc;

	if ( rw == READ ) {
		/* Nothing can be done about read errors. */
		return -EIO;
	}

	/* For each sector in the request. */
	for ( lsn = 0; lsn < count; lsn++, buffer += EVMS_VSECTOR_SIZE ) {
		rc = INIT_IO(bbr_id->source, rw, starting_lsn + lsn, 1, buffer);
		while (rc) {
			if ( bbr_id->flag & BBR_STOP_REMAP ) {
				/* Can't allow new remaps if the
				 * engine told us to stop.
				 */
				LOG_ERROR("Object %s: Bad sector ("PFU64"), but remapping is turned off.\n",
					  bbr_id->node->name, starting_lsn+lsn);
				return -EIO;
			}

			/* Find the next available relocation sector. */
			new_lsn = atomic_read(&bbr_id->in_use_replacement_blks);
			if ( new_lsn >= bbr_id->nr_replacement_blks ) {
				/* No more replacement sectors available. */
				return -EIO;
			}
			new_lsn += bbr_id->start_replacement_sect;

			/* Write the data to its new location. */
			LOG_WARNING("Object %s: Trying to remap bad sector ("PFU64") to sector ("PFU64")\n",
				    bbr_id->node->name, starting_lsn + lsn,
				    new_lsn);
			rc = INIT_IO(bbr_id->source, rw, new_lsn, 1, buffer);
			if (rc) {
				/* This replacement sector is bad.
				 * Try the next one.
				 */
				LOG_ERROR("Object %s: Replacement sector ("PFU64") is bad. Skipping.\n",
					bbr_id->node->name, new_lsn);
				atomic_inc(&bbr_id->in_use_replacement_blks);
				continue;
			}

			/* Add this new entry to the on-disk table. */
			table_sector_index = new_lsn -
					     bbr_id->start_replacement_sect;
			table_sector_offset = table_sector_index /
					      EVMS_BBR_ENTRIES_PER_SECT;
			index = table_sector_index % EVMS_BBR_ENTRIES_PER_SECT;

			bbr_table = &bbr_id->bbr_table[table_sector_offset];
			bbr_table->entries[index].bad_sect = starting_lsn + lsn;
			bbr_table->entries[index].replacement_sect = new_lsn;
			bbr_table->in_use_cnt++;
			bbr_table->sequence_number++;
			bbr_table->crc = 0;
			bbr_table->crc = evms_cs_calculate_crc(EVMS_INITIAL_CRC,
							       bbr_table,
							       sizeof(struct evms_bbr_table));

			/* Write the table to disk. */
			cpu_bbr_table_sector_to_le(bbr_table, bbr_table);
			if ( bbr_id->lba_table1 ) {
				rc = INIT_IO(bbr_id->source, WRITE,
					     bbr_id->lba_table1 +
					     table_sector_offset,
					     1, bbr_table);
			}
			if ( bbr_id->lba_table2 ) {
				rc |= INIT_IO(bbr_id->source, WRITE,
					      bbr_id->lba_table2 +
					      table_sector_offset,
					      1, bbr_table);
			}
			le_bbr_table_sector_to_cpu(bbr_table);

			if (rc) {
				/* Error writing one of the tables to disk. */
				LOG_ERROR("Object %s: Error updating BBR tables on disk.\n",
					  bbr_id->node->name);
				return rc;
			}

			/* Insert a new entry in the remapping binary-tree. */
			rc = bbr_insert_remap_entry(bbr_id,
						    &bbr_table->entries[index]);
			if (rc) {
				LOG_ERROR("Object %s: Error adding new entry to remap tree.\n",
					  bbr_id->node->name);
				return rc;
			}

			atomic_inc(&bbr_id->in_use_replacement_blks);
		}
	}

	return 0;
}

/**
 * bbr_io_process_request
 *
 * For each sector in this request, check if the sector has already
 * been remapped. If so, process all previous sectors in the request,
 * followed by the remapped sector. Then reset the starting lsn and
 * count, and keep going with the rest of the request as if it were
 * a whole new request. If any of the INIT_IO's return an error,
 * call the remapper to relocate the bad sector(s).
 */
static int bbr_io_process_request(struct bbr_io_buffer * bbr_io_buf)
{
	struct bbr_private * bbr_id = bbr_io_buf->bbr_id;
	u64 starting_lsn = bbr_io_buf->org_bh->b_rsector;
	u64 count = bbr_io_buf->org_bh->b_size >> EVMS_VSECTOR_SIZE_SHIFT;
	u64 lsn, remapped_lsn;
	char * buffer = bbr_io_buf->org_bh->b_data;
	int rc = 0, rw = bbr_io_buf->rw;

	/* For each sector in this request, check if this sector has already
	 * been remapped. If so, process all previous sectors in this request,
	 * followed by the remapped sector. Then reset the starting lsn and
	 * count and keep going with the rest of the request as if it were
	 * a whole new request.
	 */
	for ( lsn = 0; lsn < count && !(bbr_id->flag & BBR_STOP_REMAP); lsn++ ) {
		remapped_lsn = starting_lsn + lsn;
		rc = bbr_remap(bbr_id, &remapped_lsn);
		if (!rc) {
			/* This sector is fine. */
			continue;
		}

		/* Process all sectors in the request up to this one. */
		if ( lsn > 0 ) {
			rc = INIT_IO(bbr_id->source, rw,
				     starting_lsn, lsn, buffer);
			if (rc) {
				/* If this I/O failed, then one of the sectors
				 * in this request needs to be relocated.
				 */
				rc = bbr_io_remap_error(bbr_id, rw, starting_lsn,
							lsn, buffer);
				if (rc) {
					return rc;
				}
			}
			buffer += (lsn << EVMS_VSECTOR_SIZE_SHIFT);
		}

		/* Process the remapped sector. */
		rc = INIT_IO(bbr_id->source, rw, remapped_lsn, 1, buffer);
		if (rc) {
			/* BUGBUG - Need more processing if this caused an
			 * an error. If this I/O failed, then the existing
			 * remap is now bad, and we need to find a new remap.
			 * Can't use bbr_io_remap_error(), because the existing
			 * map entry needs to be changed, not added again, and
			 * the original table entry also needs to be changed.
			 */
			return rc;
		}

		buffer		+= EVMS_VSECTOR_SIZE;
		starting_lsn	+= (lsn + 1);
		count		-= (lsn + 1);
		lsn		= -1;
	}

	/* Check for any remaining sectors after the last split. This could
	 * potentially be the whole request, but that should be a rare case
	 * because requests should only be processed by the thread if we know
	 * an error occurred or they contained one or more remapped sectors.
	 */
	if ( count ) {
		rc = INIT_IO(bbr_id->source, rw, starting_lsn, count, buffer);
		if (rc) {
			/* If this I/O failed, then one of the sectors in this
			 * request needs to be relocated.
			 */
			rc = bbr_io_remap_error(bbr_id, rw, starting_lsn,
						count, buffer);
			if (rc) {
				return rc;
			}
		}
	}

	return 0;
}

/**
 * bbr_io_handler
 *
 * This is the handler for the bbr_io_thread. It continuously loops,
 * taking I/O requests off its list and processing them. If nothing
 * is on the list, the thread goes back to sleep until specifically
 * woken up.
 *
 * I/O requests should only be sent to this thread if we know that:
 * a) the request contains at least one remapped sector.
 *   or
 * b) the request caused an error on the normal I/O path.
 * This function uses synchronous I/O, so sending a request to this
 * thread that doesn't need special processing will cause severe
 * performance degredation.
 */
static void bbr_io_handler(void * void_data)
{
	struct bbr_io_buffer * bbr_io_buf;
	struct buffer_head * bh;
	unsigned long flags;
	int rc = 0;

	while (1) {
		/* Process bbr_io_list, one entry at a time. */
		spin_lock_irqsave(&bbr_io_list_lock, flags);
		if (list_empty(&bbr_io_list)) {
			/* No more items on the list. */
			spin_unlock_irqrestore(&bbr_io_list_lock, flags);
			break;
		}
		bbr_io_buf = list_entry(bbr_io_list.next,
					struct bbr_io_buffer, bbr_io_list);
		list_del(&bbr_io_buf->bbr_io_list);
		spin_unlock_irqrestore(&bbr_io_list_lock, flags);

		rc = bbr_io_process_request(bbr_io_buf);

		/* Clean up and complete the original I/O. */
		bh = bbr_io_buf->org_bh;
		if (bh->b_end_io) {
			free_bbr_io_buf(bbr_io_buf);
			evms_cs_volume_request_in_progress(bh->b_rdev, -1, NULL);
			bh->b_end_io(bh, rc ? 0 : 1);
		} else {
			/* A request that originated from bbr_init_io. */
			bbr_io_buf->rc = rc;
			complete(bbr_io_buf->complete);
		}
	}
}

/**
 * bbr_schedule_io
 *
 * Place the specified bbr_io_buf on the thread's processing list.
 */
static void bbr_schedule_io(struct bbr_io_buffer * bbr_io_buf)
{
	unsigned long flags;

	spin_lock_irqsave(&bbr_io_list_lock, flags);
	list_add_tail(&bbr_io_buf->bbr_io_list, &bbr_io_list);
	spin_unlock_irqrestore(&bbr_io_list_lock, flags);
	evms_cs_wakeup_thread(bbr_io_thread);
}

/**
 * bbr_read
 *
 * If there are any remapped sectors on this object, send this request over
 * to the thread for processing. Otherwise send it down the stack normally.
 */
static void bbr_read(struct evms_logical_node * bbr_node,
		     struct buffer_head * bh)
{
        struct bbr_private * bbr_id = bbr_node->private;
	struct bbr_io_buffer * bbr_io_buf;

	if ( bh->b_rsector + (bh->b_size >> EVMS_VSECTOR_SIZE_SHIFT) >
	     bbr_node->total_vsectors ) {
		/* Request is off the end of the object. */
		bh->b_end_io(bh, 0);
		return;
	}

	if ( atomic_read(&bbr_id->in_use_replacement_blks) == 0 ||
	     bbr_id->flag & BBR_STOP_REMAP ||
	     ! bbr_remap_probe(bbr_id, bh->b_rsector,
			       bh->b_size >> EVMS_VSECTOR_SIZE_SHIFT) ) {
		/* No existing remaps, this request doesn't contain any
		 * remapped sectors, or the engine told us not to remap.
		 */
		R_IO(bbr_id->source, bh);
		return;
	}

	/* This request has at least one remapped sector. */
	bbr_io_buf = allocate_bbr_io_buf(bbr_id, bh, READ);
	if (!bbr_io_buf) {
		/* Can't get memory to track the I/O. */
		bh->b_end_io(bh, 0);
		return;
	}

	evms_cs_volume_request_in_progress(bh->b_rdev, +1, NULL);
	bbr_schedule_io(bbr_io_buf);
}

/**
 * bbr_write_callback
 *
 * This is the callback for normal write requests. Check for an error
 * during the I/O, and send to the thread for processing if necessary.
 */
static void bbr_write_callback(struct buffer_head * bh,
			       int uptodate)
{
	struct bbr_io_buffer * bbr_io_buf = bh->b_private;
	struct buffer_head * org_bh = bbr_io_buf->org_bh;

	free_bbr_bh(bh);

	if (!(bbr_io_buf->bbr_id->flag & BBR_STOP_REMAP) &&
	    !uptodate) {
		LOG_ERROR("Object %s: Write failure on sector ("PFU64"). Scheduling for retry.\n",
			  bbr_io_buf->bbr_id->node->name,
			  (u64)org_bh->b_rsector);
		bbr_schedule_io(bbr_io_buf);
	} else {
		free_bbr_io_buf(bbr_io_buf);
		evms_cs_volume_request_in_progress(org_bh->b_rdev, -1, NULL);
		org_bh->b_end_io(org_bh, uptodate);
	}
}

/**
 * bbr_write
 *
 * If there are any remapped sectors on this object, send the request over
 * to the thread for processing. Otherwise, register for callback
 * notification, and send the request down normally.
 */
static void bbr_write(struct evms_logical_node * bbr_node,
		      struct buffer_head * bh)
{
        struct bbr_private * bbr_id = bbr_node->private;
	struct bbr_io_buffer * bbr_io_buf;
	struct buffer_head * new_bh;

	if ( bh->b_rsector + (bh->b_size >> EVMS_VSECTOR_SIZE_SHIFT) >
	     bbr_node->total_vsectors ||
	     bbr_id->flag & EVMS_VOLUME_READ_ONLY ) {
		/* Request is off the end of the object, or this
		 * is a read-only object.
		 */
		bh->b_end_io(bh, 0);
		return;
	}

	bbr_io_buf = allocate_bbr_io_buf(bbr_id, bh, WRITE);
	if (!bbr_io_buf) {
		/* Can't get memory to track the I/O. */
		bh->b_end_io(bh, 0);
		return;
	}

	new_bh = allocate_bbr_bh(bh);
	if (!new_bh) {
		/* Can't get memory to track the I/O. */
		free_bbr_io_buf(bbr_io_buf);
		bh->b_end_io(bh, 0);
		return;
	}

	evms_cs_volume_request_in_progress(bh->b_rdev, +1, NULL);

	if ( atomic_read(&bbr_id->in_use_replacement_blks) == 0 ||
	     bbr_id->flag & BBR_STOP_REMAP ||
	     ! bbr_remap_probe(bbr_id, bh->b_rsector,
			       bh->b_size >> EVMS_VSECTOR_SIZE_SHIFT) ) {
		/* No existing remaps, this request contains no remapped
		 * sectors, or the engine said to stop remapping.
		 */
		new_bh->b_private = bbr_io_buf;
		new_bh->b_end_io = bbr_write_callback;
		W_IO(bbr_id->source, new_bh);
	} else {
		/* This request contains at least one remapped sector. */
		free_bbr_bh(new_bh);
		bbr_schedule_io(bbr_io_buf);
	}
}

/**
 * bbr_init_io_schedule_io
 * @bbr_id:	Private data for the BBR node.
 * @rw:		READ or WRITE.
 * @lsn:	Starting sector for the request.
 * @count:	Number of sectors in the request.
 * @buffer:	Data buffer for the request.
 *
 * During init_io, failures must still be handled by the I/O thread. Create
 * a bbr_io_buf, and schedule it to be handled by the thread. Then wait until
 * the request is complete.
 */
static int bbr_init_io_schedule_io(struct bbr_private * bbr_id,
				   int rw,
				   u64 lsn,
				   u64 count,
				   void * buffer)
{
	struct bbr_io_buffer * bbr_io_buf;
	struct buffer_head bh;
	struct completion complete;
	int rc = 0;

	if ( rw != WRITE ) {
		/* Nothing can be done about read failures. */
		return -EIO;
	}

	LOG_ERROR("Object %s: init_io write failure (sector "PFU64": count "PFU64"). Scheduling for retry.\n",
		  bbr_id->node->name, lsn, count);
	bbr_io_buf = allocate_bbr_io_buf(bbr_id, &bh, rw);
	if (!bbr_io_buf) {
		return -ENOMEM;
	}

	memset(&bh, 0, sizeof(struct buffer_head));
	init_waitqueue_head(&bh.b_wait);
	bh.b_rsector = lsn;
	bh.b_size = count << EVMS_VSECTOR_SIZE_SHIFT;
	bh.b_data = buffer;
	bh.b_end_io = NULL;

	/* Schedule the I/O and wait for it to finish. */
	bbr_io_buf->complete = &complete;
	init_completion(bbr_io_buf->complete);
	bbr_schedule_io(bbr_io_buf);
	wait_for_completion(bbr_io_buf->complete);

	rc = bbr_io_buf->rc;
	free_bbr_io_buf(bbr_io_buf);

	return rc;
}

/**
 * bbr_init_io
 * @bbr_node:	BBR node.
 * @rw:		READ or WRITE.
 * @lsn:	Starting sector for I/O request.
 * @count:	Number of sectors in the I/O request.
 * @buffer:	Data buffer for the I/O request.
 *
 * Synchronous I/O requests.
 */
static int bbr_init_io(struct evms_logical_node * bbr_node,
		       int rw,
		       u64 start_lsn,
		       u64 count,
		       void * buffer )
{
        struct bbr_private * bbr_id = bbr_node->private;
	u64 lsn;
	int rc = 0;

	if ( start_lsn + count > bbr_node->total_vsectors ) {
		/* Request is off the end of the object. */
		return -EINVAL;
	}

	if ( rw == WRITE && (bbr_id->flag & EVMS_VOLUME_READ_ONLY) ) {
		/* Can't write to a read-only object. */
		return -EINVAL;
	}

	if ( bbr_id->flag & BBR_STOP_REMAP ||
	     atomic_read(&bbr_id->in_use_replacement_blks) == 0 ||
	     ! bbr_remap_probe(bbr_id, start_lsn, count) ) {
		/* Normal case (no existing remaps). */
		rc = INIT_IO(bbr_id->source, rw, start_lsn, count, buffer);
		if (rc && ! (bbr_id->flag & BBR_STOP_REMAP) ) {
			/* Init_io error. Send request over to
			 * thread for further processing.
			 */
			rc = bbr_init_io_schedule_io(bbr_id, rw, start_lsn,
						     count, buffer);
		}
	} else {
		/* At least one sector in this request needs to be remapped.
		 * Test and send each one down individually.
		 */
		for ( lsn = start_lsn;
		      lsn < start_lsn + count;
		      lsn++, buffer += EVMS_VSECTOR_SIZE ) {
			bbr_remap(bbr_id, &lsn);
			rc = INIT_IO(bbr_id->source, rw, lsn, 1, buffer);
			if (rc) {
				/* Init_io error. Send request
				 * to thread for processing.
				 */
				rc = bbr_init_io_schedule_io(bbr_id, rw,
							     lsn, 1, buffer);
				if (rc) {
					break;
				}
			}
		}
	}

	return rc;
}

/**
 * bbr_direct_ioctl_sector_io
 *
 * Process an I/O from the engine on an active BBR object.
 */
static int bbr_direct_ioctl_sector_io(struct bbr_private * bbr_id,
				      struct evms_notify_bbr * notify)
{
	char * buffer, * user_buffer;
	u64 lsn;
	int rc = 0;

	buffer = kmalloc(EVMS_VSECTOR_SIZE, GFP_NOIO);
	if (!buffer) {
		return -ENOMEM;
	}

	user_buffer = (char*)notify->buffer;

	for ( lsn = 0;
	      lsn < notify->nr_sect;
	      lsn++, user_buffer += EVMS_VSECTOR_SIZE ) {
		if ( notify->rw == WRITE ) {
			if ( copy_from_user(buffer, user_buffer,
					    EVMS_VSECTOR_SIZE) ) {
				rc = -EFAULT;
				break;
			}
		}

		rc = bbr_init_io(bbr_id->node, notify->rw,
				 notify->start_sect + lsn, 1, buffer);
		if (rc) {
			break;
		}

		if ( notify->rw == READ ) {
			if ( copy_to_user(user_buffer, buffer,
					  EVMS_VSECTOR_SIZE) ) {
				rc = -EFAULT;
				break;
			}
		}
	}

	kfree(buffer);
	return rc;
}

/**
 * bbr_direct_ioctl
 * @inode:	N/A
 * @file:	N/A
 * @cmd:	N/A
 * @arg:	Pointer to an evms_plugin_ioctl_pkt.
 *
 * BBR-specific ioctls from the engine. Currently handles:
 *   BBR_STOP_REMAP_CMD
 *   BBR_GET_INFO_CMD
 *   BBR_SECTOR_IO_CMD
 */
static int bbr_direct_ioctl(struct inode * inode,
			    struct file * file,
			    unsigned int cmd,
			    unsigned long arg)
{
	int rc = 0;
	struct bbr_private * bbr_id;
	struct evms_plugin_ioctl_pkt pkt, * user_pkt;
	struct evms_notify_bbr notify, * user_notify;

	MOD_INC_USE_COUNT;

	user_pkt = (struct evms_plugin_ioctl_pkt *)arg;
	if ( copy_from_user(&pkt, user_pkt, sizeof(pkt)) ) {
		MOD_DEC_USE_COUNT;
		return -EFAULT;
	}

	if ( pkt.feature_id != plugin_header.id ) {
		MOD_DEC_USE_COUNT;
		return -EINVAL;
	}

	user_notify = (struct evms_notify_bbr *)pkt.feature_ioctl_data;
	if ( copy_from_user(&notify, user_notify, sizeof(notify)) ) {
		rc = -EFAULT;
	} else {
		bbr_id = bbr_find_private(notify.object_name);
		if (!bbr_id) {
			rc = -ENODEV;
		} else {

			switch(pkt.feature_command) {

			case BBR_STOP_REMAP_CMD:
				bbr_id->flag |= BBR_STOP_REMAP;
				/* Fall through. */

			case BBR_GET_INFO_CMD:
				notify.count = atomic_read(&bbr_id->in_use_replacement_blks);
				if ( copy_to_user(&user_notify->count,
						  &notify.count,
						  sizeof(user_notify->count))) {
					rc = -EFAULT;
				}
				break;

			case BBR_SECTOR_IO_CMD:
				rc = bbr_direct_ioctl_sector_io(bbr_id,
								&notify);
				break;

			default:
				rc = -ENOSYS;
			}
		}
	}

	pkt.status = rc;
	copy_to_user(user_pkt, &pkt, sizeof(pkt));
	MOD_DEC_USE_COUNT;
	return rc;
}

/**
 * bbr_ioctl
 * @bbr_node:	BBR node.
 * @inode:	N/A
 * @file:	N/A
 * @cmd:	ioctl command to process.
 * @arg:	ioctl-specific data pointer.
 *
 * IOCTL handler. Currently BBR handles plugin-specific ioctls, as well as
 * EVMS_GET_BMAP. All others are passed to the child node.
 */
static int bbr_ioctl (struct evms_logical_node * bbr_node,
		      struct inode * inode,
		      struct file * file,
		      unsigned int cmd,
		      unsigned long arg)
{
        struct bbr_private * bbr_id = bbr_node->private;
	struct evms_get_bmap_pkt * bmap;
        int rc = 0;

        switch (cmd) {
		case EVMS_PLUGIN_IOCTL:
			rc = bbr_direct_ioctl(inode, file, cmd, arg);
			break;

		case EVMS_GET_BMAP:
			bmap = (struct evms_get_bmap_pkt *)arg;
			bbr_remap(bbr_id, &bmap->rsector);
			/* fall thru */
	
		default:
			rc = IOCTL(bbr_id->source, inode, file, cmd, arg);
        }
        return rc;
}

static int __init bbr_init(void)
{
        return evms_cs_register_plugin(&plugin_header);
}

static void __exit bbr_exit(void)
{
	evms_cs_unregister_plugin(&plugin_header);
}

module_init(bbr_init);
module_exit(bbr_exit);
#ifdef MODULE_LICENSE
MODULE_LICENSE("GPL");
#endif

