Skip to content

Ghidra is a software reverse engineering (SRE) framework

45.1 KB
Raw Blame
/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package db.buffers;

import java.io.*;
import java.util.*;

import ghidra.framework.Application;
import ghidra.util.BigEndianDataConverter;
import ghidra.util.Msg;
import ghidra.util.datastruct.IntSet;
import ghidra.util.exception.*;
import ghidra.util.task.CancelledListener;
import ghidra.util.task.TaskMonitor;

/**
 * <code>LocalBufferFile</code> implements a BufferFile as block-oriented
 * random-access file.  This type of buffer file supports save-as but does
 * not support the save operation.
 */
public class LocalBufferFile implements BufferFile {

	static final long MAGIC_NUMBER = 0x2f30312c34292c2aL;

	public static final String BUFFER_FILE_EXTENSION = ".gbf";
	public static final String PRESAVE_FILE_EXT = ".ps";
	public static final String PRESAVE_FILE_PREFIX = "tmp";
	public static final String TEMP_FILE_EXT = ".tmp";

	private static final String STRING_ENCODING = "UTF-8";

	private static final int MINIMUM_BLOCK_SIZE = 512;
	private static final int MAXIMUM_BLOCK_SIZE = DataBuffer.MAX_BUFFER_SIZE;

	private static final Random random = new Random();

	/**
	 * Current file header format version number.
	 * The third field of the file header indicates a
	 * format version which indicates how the header
	 * is formatted.
	 */
	private static final int HEADER_FORMAT_VERSION = 1;

	//
	// The first block is reserved for use by the BlockFile header.
	// The format of this header is determined by the Field #3.
	//
	// FILE FORMAT - VERSION 1
	//
	// A file header is contained within the first buffer at offset 0 within the file.
	// This header contains the following fields:
	//
	// 1. Magic number (8,long)
	// 2. File ID (8,long)
	// 3. Block file header format version (4,int) = 1
	// 5. Block size (4,int)
	// 6. First free block index (4,int)
	// 7. User-defined parameter count (4,int)
	// 8. User-defined parameters stored as:
	//       * Parm name length (4,int)
	//       * Parm name (?,char[])
	//       * Parm value (4,int)
	//    Parameter space is limited by buffer size.
	//
	// In Version 1 - the first available user buffer immediately follows the file
	// header block (i.e., buffer index 0 corresponds to block index 1).
	//
	// Each user block has the following prefix data:
	// 1. Flags (1,byte)
	//   * Bit 0: 1=empty block, 0=not empty
	// 2. DataBuffer ID (4,int) if not empty, or next empty buffer index if empty
	//        (-1 indicates last empty buffer)
	//

	// File Header offset where File ID is stored
	private static final int FILE_ID_OFFSET = 8;

	// Number of bytes added to buffer to allow for storage of flag bits.
	private static final int BUFFER_PREFIX_SIZE = 5;

	// Number of header bytes before the user-defined parameter area
	private static final int VER1_FIXED_HEADER_LENGTH = 32;

	// Buffer Flags bits
	private static final byte EMPTY_BUFFER = 0x01;

	static final int MAX_BUFFER_INDEX = Integer.MAX_VALUE - 1;

	/**
	 * <code>userParms</code> contains application parameters which correspond to
	 * this buffer file.  This list is established for
	 * an existing buffer file when the readHeader is invoked.  For a non-temporary
	 * writable buffer file, this list is flushed to the file when either close or
	 * setReadOnly is invoked.
	 */
	private Hashtable<String, Integer> userParms = new Hashtable<>();

	/**
	 * <code>freeIndexes</code> contains those buffer indexes which are free/empty
	 * and may be re-used in an update of this file.  This list is established for
	 * an existing buffer file when the readHeader is invoked.  For a non-temporary
	 * writable buffer file, this list is flushed to the file when either close or
	 * setReadOnly is invoked.
	 */
	private int[] freeIndexes = new int[0];

	/**
	 * The underlying storage file for this buffer file.
	 */
	private File file;

	/**
	 * The random-access object for the underlying file.
	 */
	private RandomAccessFile raf;

	/**
	 * Provides a handle to the active OutputBlockStream used to update file via this class's
	 * random access file.  This should be checked during {@link #close()} to guard against
	 * partially written file.  A LocalOutputBlockStream should only be used for files open for
	 * writing (i.e., !readOnly).
	 */
	private volatile LocalOutputBlockStream activeOutputBlockStream;

	/**
	 * A simple factory that allows this class to support dependency injection.
	 */
	private static InputBlockStreamFactory inputBlockStreamFactory =
		new DefaultInputBlockStreamFactory();

	/**
	 * When <code>temporary</code> is true and this file is writable (!readOnly)
	 * it will be deleted when disposed or closed.
	 */
	private boolean temporary = false;

	/**
	 * <code>fileId</code> corresponds to a random file ID assigned to this file.
	 * This file ID can be used as an integrity check when applying version files
	 * to a specific buffer file.
	 */
	private long fileId;

	/**
	 * If <code>readOnly</code> is true, this file may not be modified
	 * via the buffer put method.
	 * A read-only file may be considered "updatable" if the canSave
	 * method returns true.  The term "updatable" means that a Save file
	 * can be obtained via the getSaveFile method.
	 */
	private boolean readOnly;

	/**
	 * <code>blockSize</code> is the "actual" size of each block within this
	 * file. The <code>blockSize</code> equals the <code>bufferSize</code> plus
	 * a few bytes used for flags and to identify the user-level buffer ID.
	 */
	private int blockSize;

	/**
	 * <code>bufferSize</code> is the "usable" buffer space within each
	 * block of this file.
	 */
	private int bufferSize;

	/**
	 * <code>bufferCount</code> indicates the number of buffer which have been
	 * allocated within this file and directly reflects the size of the file.
	 * The value corresponds to the next buffer index which can be allocated once all
	 * free indexes have been utilized.  When an existing file is opened, this value
	 * is computed based upon the file length and the buffer size.
	 */
	private int bufferCount = 0;

	/**
	 * Create a temporary read/write block file.
	 * @param bufferSize user buffer size
	 * @param tmpPrefix temporary file prefix
	 * @param tmpExtension temporary file extension
	 * @throws IOException if there is an exception creating underlying files
	 */
	LocalBufferFile(int bufferSize, String tmpPrefix, String tmpExtension) throws IOException {
		this.bufferSize = bufferSize;
		this.blockSize = bufferSize + BUFFER_PREFIX_SIZE;
		if (blockSize < MINIMUM_BLOCK_SIZE || blockSize > MAXIMUM_BLOCK_SIZE) {
			throw new IllegalArgumentException("Unsupported buffer size: " + bufferSize);
		}
		this.readOnly = false;
		this.temporary = true;
		file = Application.createTempFile(tmpPrefix, tmpExtension);
		raf = new RandomAccessFile(file, "rw");
	}

	/**
	 * Create a new buffer file for writing.
	 * If the file does not exist and create is true, a new buffer file will
	 * be created.
	 * The file will be saved when closed.
	 * @param file buffer file
	 * @param bufferSize user buffer size
	 * @throws DuplicateFileException if file already exists
	 * @throws IOException if an I/O error occurs during file creation
	 */
	public LocalBufferFile(File file, int bufferSize) throws IOException {
		if (file.exists()) {
			throw new DuplicateFileException("File " + file + " already exists");
		}
		this.file = file;
		this.bufferSize = bufferSize;
		this.blockSize = bufferSize + BUFFER_PREFIX_SIZE;

		if (blockSize < MINIMUM_BLOCK_SIZE || blockSize > MAXIMUM_BLOCK_SIZE) {
			throw new IllegalArgumentException(
				"Unsupported buffer size: " + bufferSize);
		}

		this.readOnly = false;
		raf = new RandomAccessFile(file, "rw");

		fileId = random.nextLong();
	}

	/**
	 * Open an existing block file.
	 * @param file block file
	 * @param readOnly if true the file will be opened read-only
	 * @throws IOException if an error occurs or the incorrect magicNumber was read from the file.
	 */
	public LocalBufferFile(File file, boolean readOnly) throws IOException {
		this.file = file;
		this.readOnly = readOnly;
		raf = new RandomAccessFile(file, readOnly ? "r" : "rw");

		readHeader();
	}

	/**
	 * Modify an existing buffer file.
	 * <p>
	 * WARNING! Use with extreme caution since this modifies the original file and could destroy
	 * data if used improperly.
	 *
	 * @param file block file
	 * @param bufferIndex the index at which to place the buffer
	 * @param buf the buffer add to the file
	 * @throws IOException if an I/O error occurs
	 */
	public static void poke(File file, int bufferIndex, DataBuffer buf) throws IOException {
		LocalBufferFile bf = new LocalBufferFile(file, false);
		try {
			bf.put(buf, bufferIndex);
		}
		finally {
			bf.close();
		}
	}

	/**
	 * Read a buffer from an existing buffer file.
	 * @param file block file
	 * @param bufferIndex the index from which to read the buffer
	 * @return the buffer
	 * @throws IOException if an I/O error occurs
	 */
	public static DataBuffer peek(File file, int bufferIndex) throws IOException {
		LocalBufferFile bf = new LocalBufferFile(file, false);
		try {
			DataBuffer buf = new DataBuffer(bf.getBufferSize());
			bf.get(buf, bufferIndex);
			return buf;
		}
		finally {
			bf.close();
		}
	}

	/**
	 * Returns the physical file associated with this BufferFile.
	 * @return the file
	 */
	public File getFile() {
		return file;
	}

	@Override
	public boolean isReadOnly() {
		return readOnly;
	}

	@Override
	public int getParameter(String name) throws NoSuchElementException {
		Object obj = userParms.get(name);
		if (obj == null) {
			throw new NoSuchElementException(name);
		}
		return ((Integer) obj).intValue();
	}

	@Override
	public void setParameter(String name, int value) {
		try {
			if (name.getBytes(STRING_ENCODING).length > MAX_PARAM_NAME_LENGTH) {
				throw new IllegalArgumentException("Parameter name is too long");
			}
		}
		catch (UnsupportedEncodingException e) {
			throw new IllegalArgumentException("Unsupported parameter name", e);
		}
		userParms.put(name, value);
	}

	@Override
	public void clearParameters() {
		userParms.clear();
	}

	@Override
	public String[] getParameterNames() {
		ArrayList<String> list = new ArrayList<>();
		Enumeration<String> it = userParms.keys();
		while (it.hasMoreElements()) {
			list.add(it.nextElement());
		}
		String[] names = new String[list.size()];
		list.toArray(names);
		return names;
	}

	@Override
	public int[] getFreeIndexes() {
		return freeIndexes.clone();
	}

	@Override
	public void setFreeIndexes(int[] indexes) {
		freeIndexes = indexes.clone();
		Arrays.sort(freeIndexes);
	}

	/**
	 * @return the file ID assigned to this file.
	 */
	long getFileId() {
		return fileId;
	}

	/**
	 * Assigns the file ID for this file.  This should only be done when reconstructing a file.
	 * @param id file ID
	 */
	void setFileId(long id) {
		fileId = id;
	}

	/**
	 * The number of buffers contained within this file.
	 * @return the number of buffers contained within this file
	 */
	int getBufferCount() {
		return bufferCount;
	}

	/**
	 * Assigns the buffer count for this file.  This should only be done when reconstructing a
	 * file.
	 * @param count number of valid buffers contained within this file
	 */
	void setBufferCount(int count) {
		bufferCount = count;
	}

	/**
	 * Set the temporary status of this buffer file.  Temporary buffer files are deleted when
	 * closed.
	 * @param isTemporary true if temporary
	 */
	void setTemporary(boolean isTemporary) {
		temporary = isTemporary;
	}

	/**
	 * Rename underlying file
	 * @param newFile the new file for this class
	 * @return true if rename successful
	 * @throws IOException if an I/O error occurs
	 */
	boolean renameFile(File newFile) throws IOException {
		if (raf != null) {
			raf.close();
		}
		if (file.renameTo(newFile)) {
			file = newFile;
			if (raf != null) {
				raf = new RandomAccessFile(file, "rw");
			}
			return true;
		}
		return false;
	}

	/**
	 * Set the random access file position to the file block containing the specified buffer
	 * identified by its bufferIndex.  It is important to understand the distinction between
	 * blocks and buffers, where buffers are stored within file blocks which are slightly larger.
	 * In addition, the first file block stores the file header and is not used to store a buffer.
	 * @param bufferIndex buffer index
	 * @return file block index (=bufferIndex+1)
	 * @throws IOException if an I/O error occurs
	 */
	private int seekBufferBlock(int bufferIndex) throws IOException {
		// Perform long multiplication to support file sizes greater than 2-GBytes
		// Add 1 to buffer index to obtain block index (first usable buffer, buffer#0, is
		// contained within block#1 since block#0 contains file header)
		int blockIndex = bufferIndex + 1;
		long offset = (long) blockIndex * (long) blockSize;
		raf.seek(offset);
		return blockIndex;
	}

	/**
	 * Set the random access file position to the specified file block and offset within that
	 * block.
	 *
	 * <p>NOTE: block#0 contains the file header.
	 * @param blockIndex block index
	 * @param offsetWithinBlock offset within file block
	 * @throws IOException if there is an exception reading the file
	 */
	private void seekBlock(int blockIndex, int offsetWithinBlock) throws IOException {
		// Perform long multiplication to support file sizes greater than 2-GBytes
		long offset = ((long) blockIndex * (long) blockSize) + offsetWithinBlock;
		raf.seek(offset);
	}

	/**
	 * Read file header and initialize the user parameter and free buffer index lists.
	 * @throws IOException if an I/O error occurs
	 */
	private void readHeader() throws IOException {

		seekBlock(0, 0);

		// Check magic number
		long magicNumber = raf.readLong();
		if (magicNumber != MAGIC_NUMBER) {
			throw new IOException("Unrecognized file format");
		}

		// Read file ID
		fileId = raf.readLong();

		// Check file format version
		int headerFormatVersion = raf.readInt();
		if (headerFormatVersion != HEADER_FORMAT_VERSION) {
			throw new IOException("Unrecognized file format");
		}

		// Read block size, free buffer count, and first free buffer index
		blockSize = raf.readInt();
		if (blockSize < MINIMUM_BLOCK_SIZE || blockSize > MAXIMUM_BLOCK_SIZE) {
			throw new IOException("Unsupported block size: " + blockSize);
		}

		bufferSize = blockSize - BUFFER_PREFIX_SIZE;
		int firstFreeBufferIndex = raf.readInt();
		long len = raf.length();
		if ((len % blockSize) != 0) {
			throw new IOException("Corrupt file");
		}
		bufferCount = (int) (len / blockSize) - 1;

		// Read user-defined integer parameters values
		int cnt = raf.readInt();
		clearParameters();
		for (int i = 0; i < cnt; i++) {
			int nameLen = raf.readInt();
			byte[] nameBytes = new byte[nameLen];
			raf.read(nameBytes);
			setParameter(new String(nameBytes, STRING_ENCODING), raf.readInt());
		}
		buildFreeIndexList(firstFreeBufferIndex);
	}

	/**
	 * Store the user parameter and free buffer index lists and write the file header.
	 * @throws IOException if an I/O error occurs
	 */
	private void writeHeader() throws IOException {

		if (readOnly) {
			throw new IOException("File is read-only");
		}

		// Output free list
		int prev = -1;
		for (int index : freeIndexes) {
			putFreeBlock(index, prev);
			prev = index;
		}

		seekBlock(0, 0);

		// Write Header values
		raf.writeLong(MAGIC_NUMBER);
		raf.writeLong(fileId);
		raf.writeInt(HEADER_FORMAT_VERSION);
		raf.writeInt(blockSize);
		raf.writeInt(prev);

		// Write user parameter count and values
		String[] parmNames = getParameterNames();
		raf.writeInt(parmNames.length);
		int cnt = VER1_FIXED_HEADER_LENGTH;
		for (String parmName : parmNames) {
			byte[] nameBytes = parmName.getBytes(STRING_ENCODING);
			cnt += 8 + nameBytes.length;
			if (cnt > bufferSize) {
				throw new IOException("Buffer size too small");
			}
			raf.writeInt(nameBytes.length);
			raf.write(nameBytes);
			raf.writeInt(getParameter(parmName));
		}
	}

	/**
	 * Build free index stack from file.
	 */
	private void buildFreeIndexList(int firstFreeBufferIndex) throws IOException {

		ArrayList<Integer> freeIndexList = new ArrayList<>();
		int nextIndex = firstFreeBufferIndex;
		while (nextIndex >= 0) {

			// Push index on stack
			freeIndexList.add(nextIndex);

			// Read block to get next index
			seekBufferBlock(nextIndex);

			// Read version 1 buffer prefix and next empty index
			byte flags = raf.readByte();
			if ((flags & EMPTY_BUFFER) == 0) {
				throw new IOException("Corrupt file");
			}
			nextIndex = raf.readInt();
		}

		int[] newFreeIndexes = new int[freeIndexList.size()];
		for (int i = 0; i < newFreeIndexes.length; i++) {
			newFreeIndexes[i] = freeIndexList.get(i).intValue();
		}
		setFreeIndexes(newFreeIndexes);
	}

	/**
	 * Update a storage block as free and link to the next free block.
	 * @param index block index of free block
	 * @param nextFreeIndex block index of next free block, a -1 should be specified to mark the
	 * end of the linked list.
	 * @throws IOException if an I/O error occurs
	 */
	void putFreeBlock(int index, int nextFreeIndex) throws IOException {

		if (index > bufferCount) {
			throw new EOFException(
				"Free buffer index too large (" + index + " > " + bufferCount + ")");
		}
		if (index == bufferCount) {
			++bufferCount;
		}

		// Write version 1 buffer prefix
		seekBufferBlock(index);

		raf.writeByte(EMPTY_BUFFER);
		raf.writeInt(nextFreeIndex);
	}

	/**
	 * Generate a DataBuffer instance which corresponds to the specified block
	 * based upon LocalBufferFile block usage.
	 * @param block the buffer file block to be converted
	 * @return DataBuffer instance or null if head block.  If empty block
	 * DataBuffer will have null data
	 */
	public static DataBuffer getDataBuffer(BufferFileBlock block) {
		int blockIndex = block.getIndex();
		if (blockIndex <= 0) {
			return null; // head or invalid block
		}
		byte[] blockData = block.getData();
		byte flags = blockData[0];

		DataBuffer buf = new DataBuffer();

		if (flags == EMPTY_BUFFER) {
			buf.setId(-1);
			buf.setEmpty(true);
			return buf;
		}

		int bufferId = BigEndianDataConverter.INSTANCE.getInt(blockData, 1);
		buf.setId(bufferId);

		byte[] bufData = new byte[blockData.length - BUFFER_PREFIX_SIZE];
		System.arraycopy(blockData, BUFFER_PREFIX_SIZE, bufData, 0, bufData.length);
		buf.setData(bufData);

		return buf;
	}

	/**
	 * Generate a BufferFileBlock instance which corresponds to the specified DataBuffer
	 * based upon LocalBufferFile block usage.  This should generally not be used for writing
	 * empty blocks since they will not be properly linked which is normally handled during
	 * header flush which is performed by BufferFile close on files being written.
	 * @param buf the data buffer to be converted
	 * @param bufferSize data buffer size used for integrity check and generating empty buffer
	 * @return BufferFileBlock instance.
	 */
	public static BufferFileBlock getBufferFileBlock(DataBuffer buf, int bufferSize) {

		byte[] data = buf.data;
		boolean empty = buf.isEmpty();
		if (!empty && data.length != bufferSize) {
			throw new IllegalArgumentException("Bad buffer size");
		}

		int blockIndex = buf.getId() + 1;

		byte[] bytes = new byte[bufferSize + BUFFER_PREFIX_SIZE];
		bytes[0] = empty ? EMPTY_BUFFER : 0; // set empty flag
		BigEndianDataConverter.INSTANCE.putInt(bytes, 1, buf.getId());
		if (!empty) {
			System.arraycopy(data, 0, bytes, BUFFER_PREFIX_SIZE, data.length);
		}
		return new BufferFileBlock(blockIndex, bytes);
	}

	@Override
	public synchronized DataBuffer get(DataBuffer buf, int index) throws IOException {

		if (index > bufferCount) {
			throw new EOFException("Buffer index too large (" + index + " > " + bufferCount + ")");
		}
		if (raf == null) {
			throw new ClosedException();
		}

		seekBufferBlock(index);

		// Read version 1 buffer prefix
		byte flags = raf.readByte();

		// Read buffer ID
		buf.setId(raf.readInt());

		if ((flags & EMPTY_BUFFER) != 0) {
			buf.setEmpty(true);
			buf.setId(-1);
		}
		else {
			buf.setEmpty(false);
			byte[] data = buf.data;
			if (data == null) {
				data = new byte[bufferSize];
				buf.data = data;
			}
			else if (data.length != bufferSize) {
				throw new IllegalArgumentException("Bad buffer size");
			}
			// Non-empty Buffer - read data
			raf.readFully(data);
		}
		buf.setDirty(false);
		return buf;
	}

	@Override
	public synchronized void put(DataBuffer buf, int index) throws IOException {

		if (readOnly) {
			throw new IOException("File is read-only");
		}
		if (raf == null) {
			throw new ClosedException();
		}

		if (index > MAX_BUFFER_INDEX) {
			throw new EOFException("Buffer index too large, exceeds max-int");
		}

		byte[] data = buf.data;
		boolean empty = buf.isEmpty();

		if (data != null && data.length != bufferSize) {
			throw new IllegalArgumentException("Bad buffer size");
		}

		seekBufferBlock(index);

		// Write block
		if (empty) {
			raf.writeByte(EMPTY_BUFFER); // Empty flag only
			raf.writeInt(buf.getId()); // ID
		}
		else {
			raf.writeByte(0); // Clear Flags
			raf.writeInt(buf.getId()); // ID

			// Write data
			raf.write(data, 0, bufferSize);
		}

		if (index >= bufferCount) {
			// we must assume that any buffers starting with bufferCount upto index will
			// be accounted for in the ultimate free-list maintained outside the buffer file.
			bufferCount = index + 1;
		}
	}

	@Override
	public int getBufferSize() {
		return bufferSize;
	}

	@Override
	public int getIndexCount() {
		return bufferCount;
	}

	/**
	 * Truncate the buffer file length to the specified index count.
	 * @param indexCount the index count
	 * @throws IOException if an I/O error occurs
	 */
	void truncate(int indexCount) throws IOException {

		if (readOnly) {
			throw new IOException("File is read-only");
		}

		long size = (indexCount + 1) * blockSize;
		raf.setLength(size);
		this.bufferCount = indexCount;
	}

	/**
	 * Write all unwritten data to the file prior to closing.
	 * @return true if flush was performed, false if not required.
	 * @throws IOException thrown if flush failed
	 */
	boolean flush() throws IOException {

		if (raf == null || readOnly || temporary) {
			return false;
		}

		// write header
		writeHeader();

		// Adjust file length
		long len = raf.length();
		long d = len % blockSize;
		if (d != 0) {
			raf.setLength(len - d + blockSize);
		}

		try {
			raf.getFD().sync();
		}
		catch (SyncFailedException e) {
			// Sync not supported - we tried our best
		}

		return true;
	}

	@Override
	public void dispose() {
		try {
			if (!readOnly) {
				delete();
			}
			else {
				close();
			}
		}
		catch (Throwable t) {
			Msg.error(this, t);
		}
		finally {
			if (activeOutputBlockStream != null) {
				try {
					// force closure of active block stream
					activeOutputBlockStream.close();
				}
				catch (IOException e) {
					// ignore
				}
			}
			activeOutputBlockStream = null;
		}
	}

	@Override
	public synchronized boolean setReadOnly() throws IOException {

		if (!flush()) {
			return false;
		}

		raf.close();
		raf = new RandomAccessFile(file, "r");
		readOnly = true;

		return true;
	}

	/**
	 * True if this is a temporary buffer file
	 * @return true if this is a temporary buffer file
	 */
	boolean isTemporary() {
		return temporary;
	}

	/**
	 * True if buffer file is closed
	 * @return true if buffer file is closed
	 */
	boolean isClosed() {
		return raf == null;
	}

	@Override
	public synchronized void close() throws IOException {
		if (raf == null) {
			return;
		}

		boolean commit = false;
		try {
			if (activeOutputBlockStream != null) {
				// active block stream was not closed properly
				try {
					activeOutputBlockStream.close();
				}
				catch (IOException e) {
					// ignore
				}
			}
			else {
				commit = flush();
			}
			raf.close();
		}
		finally {
			raf = null;
			if (!readOnly && !commit) {
				// commit failed
				file.delete();
				if (activeOutputBlockStream != null) {
					// block stream forced closed above but need to throw critical error
					activeOutputBlockStream = null;
					throw new IOException("active block stream was not closed properly");
				}
			}
		}
	}

	@Override
	public synchronized boolean delete() {

		if (raf == null || readOnly) {
			return false;
		}

		boolean success = false;
		try {
			try {
				raf.close();
			}
			catch (IOException e) {
				// ignored
			}
			raf = null;
		}
		finally {
			success = file.delete();
		}
		return success;
	}

	/**
	 * Clone this buffer file to the specified file.  The file must not already exist.  If the
	 * operation is cancelled or an error occurs the file is not created.
	 * @param destinationFile destination file
	 * @param monitor progress monitor
	 * @throws IOException if IO error occurs.
	 * @throws CancelledException if the monitor cancels the operation.
	 */
	public void clone(File destinationFile, TaskMonitor monitor)
			throws IOException, CancelledException {
		LocalBufferFile destBf = new LocalBufferFile(destinationFile, bufferSize);
		boolean success = false;
		try {
			copyFile(this, destBf, null, monitor);
			destBf.setFileId(fileId);
			destBf.close();
			success = true;
		}
		finally {
			if (!success) {
				destBf.delete();
			}
		}
	}

	@Override
	public String toString() {
		return file.toString();
	}

	/**
	 * Obtain a direct stream to read all blocks of this buffer file
	 * @return input block stream
	 * @throws IOException if there is an exception creating the stream
	 */
	public InputBlockStream getInputBlockStream() throws IOException {
		return inputBlockStreamFactory.createInputBlockStream(this);
	}

	/**
	 * Obtain a direct stream to write blocks to this buffer file
	 * @param blockCount number of blocks to be transferred
	 * @return output block stream
	 * @throws IOException if an I/O error occurs
	 */
	public OutputBlockStream getOutputBlockStream(int blockCount) throws IOException {
		return new LocalOutputBlockStream(blockCount);
	}

	/**
	 * Factory method for generating the appropriate type of {@link InputBlockStream}
	 * for the specified read-only bufferFile.  Input stream may not supply header block in which case
	 * free list and file parameters may need to be set separately.
	 * @param bufferFile buffer file opened read-only
	 * @param monitor task monitor
	 * @return input block stream object
	 * @throws IOException if an I/O error occurs
	 */
	private static InputBlockStream getInputBlockStream(BufferFile bufferFile, TaskMonitor monitor)
			throws IOException {
		// This method is used so we can utilize a package method and avoid putting
		// it on the BufferFile interface
		if (bufferFile instanceof BufferFileAdapter) {
			return ((BufferFileAdapter) bufferFile).getInputBlockStream(monitor);
		}
		if (bufferFile instanceof LocalBufferFile) {
			return ((LocalBufferFile) bufferFile).getInputBlockStream();
		}
		throw new IllegalArgumentException(
			"Unsupported buffer file implementation: " + bufferFile.getClass().getName());
	}

	/**
	 * Factory method for generating the appropriate type of {@link InputBlockStream}
	 * for the specified read-only bufferFile with an optional changeMap used
	 * to select which buffer should be transferred.  Input stream may not supply header block
	 * in which case free list and file parameters may need to be set separately.
	 * @param bufferFile buffer file opened read-only
	 * @param monitor task monitor
	 * @return input block stream object
	 * @throws IOException if an I/O error occurs
	 */
	private static InputBlockStream getInputBlockStream(BufferFile bufferFile, ChangeMap changeMap,
			TaskMonitor monitor)
			throws IOException {
		if (changeMap == null) {
			return getInputBlockStream(bufferFile, monitor);
		}
		// This method is used so we can utilize a package method and avoid putting
		// it on the BufferFile interface
		if (bufferFile instanceof ManagedBufferFileAdapter) {
			return ((ManagedBufferFileAdapter) bufferFile).getInputBlockStream(changeMap.getData(),
				monitor);
		}
		if (bufferFile instanceof LocalManagedBufferFile) {
			return ((LocalManagedBufferFile) bufferFile).getInputBlockStream(changeMap.getData());
		}
		throw new IllegalArgumentException(
			"Unsupported buffer file implementation: " + bufferFile.getClass().getName());
	}

	/**
	 * Factory method for generating the appropriate type of {@link OutputBlockStream}
	 * for the specified write-able bufferFile.
	 * @param bufferFile write-able buffer file
	 * @param blockCount number of blocks to be written.  This should be available from
	 * the corresponding {@link InputBlockStream}.
	 * @param monitor task monitor
	 * @return output block stream object
	 * @throws IOException if an I/O error occurs
	 */
	static OutputBlockStream getOutputBlockStream(BufferFile bufferFile, int blockCount,
			TaskMonitor monitor)
			throws IOException {
		// This method is used so we can utilize a package method and avoid putting
		// it on the BufferFile interface
		if (bufferFile instanceof BufferFileAdapter) {
			return ((BufferFileAdapter) bufferFile).getOutputBlockStream(blockCount, monitor);
		}
		if (bufferFile instanceof LocalBufferFile) {
			return ((LocalBufferFile) bufferFile).getOutputBlockStream(blockCount);
		}
		throw new IllegalArgumentException(
			"Unsupported buffer file implementation: " + bufferFile.getClass().getName());
	}

	/**
	 * Copy the complete content of a specified srcFile into a destFile excluding file ID.  Both
	 * files remain open.
	 * @param srcFile open buffer file
	 * @param destFile empty buffer file which is open for writing.
	 * @param changeMap optional change map which indicates those buffers which must be copied.
	 * Any buffer index outside the range of the change map will also be copied.
	 * @param monitor progress monitor
	 * @throws IOException if IO error occurs.
	 * @throws CancelledException if the monitor cancels the operation.
	 */
	public static void copyFile(BufferFile srcFile, BufferFile destFile, ChangeMap changeMap,
			TaskMonitor monitor) throws IOException, CancelledException {
		if (destFile.isReadOnly()) {
			throw new IOException("File is read-only");
		}

		if (srcFile.getBufferSize() != destFile.getBufferSize()) {
			throw new IOException("Buffer sizes differ");
		}

		monitor = TaskMonitor.dummyIfNull(monitor);

		int srcBlockCnt;
		boolean headerTransferRequired;

		try (InputBlockStream in = getInputBlockStream(srcFile, changeMap, monitor)) {
			headerTransferRequired = !in.includesHeaderBlock();
			srcBlockCnt = in.getBlockCount();
			monitor.initialize(srcBlockCnt + 2);
			try (OutputBlockStream out =
				getOutputBlockStream(destFile, in.getBlockCount(), monitor)) {
				completeBlockStreamTransfer(in, out, monitor);
			}
		}
		finally {
			// circumvent other exceptions if cancelled
			monitor.checkCancelled();
		}

		if (headerTransferRequired) {
			destFile.clearParameters();
			String[] parmNames = srcFile.getParameterNames();
			for (String name : parmNames) {
				destFile.setParameter(name, srcFile.getParameter(name));
			}
			monitor.setProgress(srcBlockCnt + 1);

			// Copy free index list
			destFile.setFreeIndexes(srcFile.getFreeIndexes());
		}
		monitor.setProgress(srcBlockCnt + 2);
	}

	/**
	 * Perform a complete block stream transfer from in to out
	 * @param in input block stream
	 * @param out output block stream
	 * @param monitor progress and cancel monitor
	 * @throws CancelledException the operation is cancelled
	 * @throws IOException if an I/O error occurs
	 */
	static void completeBlockStreamTransfer(InputBlockStream in, OutputBlockStream out,
			TaskMonitor monitor) throws CancelledException, IOException {
		int count = 0;
		try (BlockStreamCancelMonitor cancelMonitor =
			new BlockStreamCancelMonitor(monitor, in, out)) {
			int srcBlockCnt = in.getBlockCount();
			BufferFileBlock block;
			while ((block = in.readBlock()) != null) {
				monitor.checkCancelled();
				out.writeBlock(block);
				monitor.setProgress(count++);
			}
			if (count != srcBlockCnt) {
				throw new IOException("unexpected block transfer count");
			}
		}
	}

	/**
	 * Attempt to remove all pre-save files.
	 * Those still open by an existing process should
	 * not be removed by the operating system.
	 * @param dir data directory containing pre-save files
	 * @param beforeNow if not 0, file mod time must be less than the specified time
	 */
	public static void cleanupOldPreSaveFiles(File dir, long beforeNow) {
		File[] oldFiles = dir.listFiles(new BufferFileFilter(null, PRESAVE_FILE_EXT));
		if (oldFiles == null) {
			return;
		}
		for (File oldFile : oldFiles) {
			if ((beforeNow == 0 || oldFile.lastModified() < beforeNow) && oldFile.delete()) {
				Msg.info(LocalBufferFile.class, "Removed old presave file: " + oldFile);
			}
		}
	}

	/**
	 * Get the recommended buffer size given a target buffer size.
	 * @param requestedBufferSize target buffer size
	 * @return recommended buffer size
	 */
	static int getRecommendedBufferSize(int requestedBufferSize) {
		int size = (requestedBufferSize + BUFFER_PREFIX_SIZE) & -MINIMUM_BLOCK_SIZE;
		if (size <= 0) {
			size = MINIMUM_BLOCK_SIZE;
		}
		return size - BUFFER_PREFIX_SIZE;
	}

//=================================================================================================
// Inner Classes
//=================================================================================================

	/**
	 * <code>LocalBufferInputBlockStream</code> provides an input BlockStream for
	 * transferring the entire file content associated with a read-only buffer
	 * file use a buffer-based transfer.
	 */
	class LocalBufferInputBlockStream implements InputBlockStream {

		private int nextBufferIndex = 0;
		private DataBuffer buf;

		/**
		 * Construct input block stream for this buffer file.
		 * @throws IOException	if an I/O error occurs
		 */
		LocalBufferInputBlockStream() throws IOException {
			if (!isReadOnly()) {
				throw new IOException("Read stream only permitted on read-only buffer file");
			}
			buf = new DataBuffer();
		}

		@Override
		public boolean includesHeaderBlock() {
			return false;
		}

		@Override
		public int getBlockCount() {
			// header block is excluded
			return getBufferCount();
		}

		@Override
		public int getBlockSize() {
			return blockSize;
		}

		@Override
		public void close() throws IOException {
			// raf remains open
		}

		@Override
		public BufferFileBlock readBlock() throws IOException {
			synchronized (LocalBufferFile.this) {

				// Must add 1 to buffer index values to produce block index value.

				if (nextBufferIndex == getBufferCount()) {
					return null;
				}

				get(buf, nextBufferIndex);

				int blockIndex = (nextBufferIndex++) + 1;

				byte[] block = new byte[getBlockSize()];
				if (buf.isEmpty()) {
					block[0] = EMPTY_BUFFER; // Empty flag only
				}
				BigEndianDataConverter.INSTANCE.putInt(block, 1, buf.getId());
				if (!buf.isEmpty()) {
					byte[] data = buf.getData();
					System.arraycopy(data, 0, block, BUFFER_PREFIX_SIZE, data.length);
				}

				return new BufferFileBlock(blockIndex, block);
			}
		}
	}

	/**
	 * <code>LocalFileInputBlockStream</code> provides an input BlockStream for
	 * transferring the entire file content associated with a read-only file.
	 * This implementation reads the data directly from a single local file
	 * and must not be used when performing version reconstruction or
	 * change-map driven streams.
	 */
	private class LocalFileInputBlockStream implements InputBlockStream {

		private InputStream fin;
		private int blockCount;
		private int nextIndex = 0;

		/**
		 * Construct input block stream for entire file
		 * @throws IOException if an I/O error occurs
		 */
		LocalFileInputBlockStream() throws IOException {
			if (!readOnly) {
				throw new IOException("Read stream only permitted on read-only buffer file");
			}

			long fileLength = file.length();
			if (fileLength <= 0 || (fileLength % blockSize) != 0) {
				throw new IOException("Corrupt file");
			}
			blockCount = (int) (fileLength / blockSize);
			fin = new BufferedInputStream(new FileInputStream(file));
		}

		@Override
		public boolean includesHeaderBlock() {
			return true;
		}

		@Override
		public int getBlockCount() {
			return blockCount;
		}

		@Override
		public int getBlockSize() {
			return blockSize;
		}

		@Override
		public void close() throws IOException {
			fin.close();
		}

		@Override
		public BufferFileBlock readBlock() throws IOException {
			byte[] data = new byte[blockSize];
			int total = 0;
			while (total < blockSize) {
				int readlen = fin.read(data, total, blockSize - total);
				if (readlen < 0) {
					if (total == 0 && nextIndex == blockCount) {
						return null;
					}
					throw new EOFException("unexpected end of file");
				}
				total += readlen;
			}
			return new BufferFileBlock(nextIndex++, data);
		}
	}

	/**
	 * <code>LocalRandomInputBlockStream</code> provides ability to
	 * selectively read a select set of buffers from the LocalBufferFile
	 * based upon a specified ChangeMap.
	 */
	class LocalRandomInputBlockStream implements InputBlockStream {

		/**
		 * The <code>bufferIndexList</code> contains the DataBuffer indexes
		 * which are to be transferred.  Buffer index values plus 1 equal the
		 * file block index.
		 */
		private List<Integer> bufferIndexList;

		/**
		 * If the bufferIndexList is not null, this index identifies the next
		 * entry within the list whose block should be transferred, otherwise
		 * the index directly identifies the next DataBuffer to be transferred.
		 */
		private int nextIndex;

		/**
		 * Construct input block stream for specific block indexes
		 * within the file based upon a changeMap.  The head buffer
		 * will never be included in the transfer.
		 * @param changeMapData ChangeMap data which is used in determining which
		 * buffers should be streamed
		 * @throws IOException if an I/O error occurs
		 */
		LocalRandomInputBlockStream(byte[] changeMapData) throws IOException {
			if (!readOnly) {
				throw new IOException("Read stream only permitted on read-only buffer file");
			}
			buildBufferIndexList(changeMapData);
		}

		private void buildBufferIndexList(byte[] changeMapData) {

			ChangeMap changeMap = new ChangeMap(changeMapData);

			bufferIndexList = new ArrayList<>();

			IntSet emptySet = new IntSet(getFreeIndexes());

			for (int bufferIndex = 0; bufferIndex < bufferCount; bufferIndex++) {
				if (!emptySet.contains(bufferIndex) &&
					(changeMap.hasChanged(bufferIndex) || !changeMap.containsIndex(bufferIndex))) {
					// Add block index if block is not empty and block has either been
					// modified in versioned srcFile since checkout to initial destFile, or
					// block was added in most recent version of srcFile
					bufferIndexList.add(bufferIndex);
				}
			}
		}

		@Override
		public boolean includesHeaderBlock() {
			return false;
		}

		@Override
		public int getBlockCount() {
			return bufferIndexList != null ? bufferIndexList.size() : bufferCount;
		}

		@Override
		public int getBlockSize() {
			return blockSize;
		}

		@Override
		public void close() throws IOException {
			// raf remains open
		}

		@Override
		public BufferFileBlock readBlock() throws IOException {
			synchronized (LocalBufferFile.this) {
				if (raf == null) {
					throw new ClosedException();
				}

				if (nextIndex == bufferIndexList.size()) {
					return null; // no more buffers
				}

				int blockIndex = seekBufferBlock(bufferIndexList.get(nextIndex++));

				byte[] block = new byte[blockSize];
				raf.readFully(block);

				return new BufferFileBlock(blockIndex, block);
			}
		}
	}

	/**
	 * <code>LocalOutputBlockStream</code> provides an OutputBlockStream for
	 * updating specific buffers of a non-read-only file.
	 */
	class LocalOutputBlockStream implements OutputBlockStream {

		private final long orignalFileId = fileId;
		private final int blockCount;

		private boolean isClosed = false;
		private boolean refreshOnClose;

		public LocalOutputBlockStream(int blockCount) throws IOException {
			if (readOnly) {
				throw new IOException("Write stream only permitted on updateable buffer file");
			}
			if (activeOutputBlockStream != null) {
				throw new IOException("Active block stream already exists");
			}
			this.blockCount = blockCount;
			activeOutputBlockStream = this;
		}

		@Override
		public void close() throws IOException {
			synchronized (LocalBufferFile.this) {

				if (isClosed) {
					return;
				}
				isClosed = true;

				if (raf == null) {
					throw new ClosedException();
				}

				try {
					if (refreshOnClose) {

						// read file header required
						readHeader();

						// restore fileId within file
						fileId = orignalFileId;
						seekBlock(0, FILE_ID_OFFSET);
						raf.writeLong(fileId);
					}
				}
				finally {
					activeOutputBlockStream = null;
				}

				// don't close file
			}
		}

		@Override
		public void writeBlock(BufferFileBlock block) throws IOException {
			if (blockSize != block.size()) {
				throw new IOException("incompatible block size");
			}
			synchronized (LocalBufferFile.this) {
				if (raf == null) {
					throw new ClosedException();
				}
				int blockIndex = block.getIndex();
				if (blockIndex == 0) {
					// must refresh buffer file if head block is written
					refreshOnClose = true;
				}
				seekBlock(blockIndex, 0);
				raf.write(block.getData());
				if (blockIndex > bufferCount) {
					// we must assume that any buffers starting with bufferCount upto blockIndex will
					// be accounted for in the ultimate free-list maintained outside the buffer file.
					// NOTE: bufferCount reflects DataBuffers which excludes the first block at blockIndex=0
					bufferCount = blockIndex; // 
				}
			}
		}

		@Override
		public int getBlockCount() {
			return blockCount;
		}

		@Override
		public int getBlockSize() {
			return blockSize;
		}
	}

	/**
	 * <code>BlockStreamCancelMonitor</code> is used to close associated BlockStreams
	 * when a TaskMonitor is cancelled
	 */
	private static class BlockStreamCancelMonitor implements Closeable, CancelledListener {

		private TaskMonitor monitor;
		private BlockStream[] blockStreams;

		BlockStreamCancelMonitor(TaskMonitor monitor, BlockStream... blockStreams) {
			this.monitor = monitor;
			this.blockStreams = blockStreams;
			monitor.addCancelledListener(this);
		}

		@Override
		public void close() throws IOException {
			monitor.removeCancelledListener(this);
		}

		@Override
		public void cancelled() {
			for (BlockStream blockStream : blockStreams) {
				try {
					blockStream.close();
				}
				catch (IOException e) {
					// ignore
				}
			}
		}

	}

	/**
	 * File filter to identify various files
	 */
	static class BufferFileFilter implements FileFilter {
		private final String ext;
		private final String prefix;

		BufferFileFilter(String prefix, String ext) {
			this.prefix = prefix;
			this.ext = ext;
		}

		@Override
		public boolean accept(File file) {
			if (file.isFile()) {
				String name = file.getName();
				if (prefix == null || name.indexOf(prefix) == 0) {
					if (ext == null || name.endsWith(ext)) {
						return true;
					}
				}
			}
			return false;
		}
	}

	/**
	 * A simple interface that allows for dependency injection
	 */
	public static interface InputBlockStreamFactory {
		public InputBlockStream createInputBlockStream(LocalBufferFile bf) throws IOException;
	}

	/**
	 * A factory to supply the default implementation for create an {@link InputBlockStream}
	 */
	private static class DefaultInputBlockStreamFactory implements InputBlockStreamFactory {
		@Override
		public InputBlockStream createInputBlockStream(LocalBufferFile bf) throws IOException {
			if (!bf.readOnly) {
				throw new IOException("Read stream only permitted on read-only buffer file");
			}
			return bf.new LocalFileInputBlockStream();
		}
	}

//	private void checkSameContent(BufferFile expectedBf, BufferFile inspectedBf) throws IOException {
//
//		//assertEquals(inspectedBf.getIndexCount(), expectedBf.getIndexCount());
//		if(inspectedBf.getIndexCount() != expectedBf.getIndexCount()) {
//			System.err.println("Buffer count mismatch following update");
//		}
//
//		int[] vFreeList = inspectedBf.getFreeIndexes();
//		int[] pFreeList = inspectedBf.getFreeIndexes();
//		Arrays.sort(vFreeList);
//		Arrays.sort(pFreeList);
//		if (!Arrays.equals(vFreeList, pFreeList)) {
//			System.err.println("Freelist mismatch following update");
//		}
//
//		if (!Arrays.equals(expectedBf.getParameterNames(), inspectedBf.getParameterNames())) {
//			System.err.println("Parameter list mismatch following update");
//		}
//
//		for (String name : expectedBf.getParameterNames()) {
//			if (expectedBf.getParameter(name) != inspectedBf.getParameter(name)) {
//				System.err.println("Parameter " + name + " mismatch following update");
//			}
//		}
//
//		DataBuffer pbuf = new DataBuffer();
//		DataBuffer vbuf = new DataBuffer();
//		int cnt = inspectedBf.getIndexCount();
//		for (int i = 0; i < cnt; i++) {
//			checkSameContent(i, expectedBf.get(vbuf, i), inspectedBf.get(pbuf, i));
//		}
//	}
//
//	private void checkSameContent(int index, DataBuffer expectedBuf, DataBuffer inspectedBuf) {
//		if (expectedBuf.isEmpty() != inspectedBuf.isEmpty()) {
//			System.err.println("Buffer " + index + " empty flag mismatch following update");
//		}
//		if (expectedBuf.getId() != inspectedBuf.getId()) {
//			System.err.println("Buffer " + index + " has unexpected ID following update");
//		}
//		if (!expectedBuf.isEmpty()) {
//			if (!Arrays.equals(expectedBuf.data, inspectedBuf.data)) {
//				System.err.println("Buffer " + index + " content differ following update");
//			}
//		}
//	}

}