44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
63 #define F3D_SHORT_MUTEX_ARRAY 1
64 #define F3D_MUTEX_ARRAY_SIZE 1000
65 #define F3D_NO_BLOCKS_ARRAY 1
77 template <
typename Data_T>
82 template <
typename Data_T>
85 template <
typename Data_T>
103 template <
class Data_T>
113 typedef boost::shared_ptr<Reference>
Ptr;
118 #if F3D_NO_BLOCKS_ARRAY
121 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
156 #if F3D_SHORT_MUTEX_ARRAY
170 static Ptr create(
const std::string a_filename,
171 const std::string a_layerPath);
285 template <
class Data_T>
290 template <
class Data_T>
294 template <
class Data_T>
401 template <
class Data_T>
415 void setLimitMemUse(
bool enabled);
419 bool doLimitMemUse()
const;
422 void setMaxMemUse(
float maxMemUse);
429 long long totalLoads();
432 long long numLoadedBlocks();
435 long long totalLoadedBlocks();
439 float cacheFractionLoaded();
442 float cacheLoadsPerBlock();
446 float cacheEfficiency();
449 void resetCacheStatistics();
461 template <
class Data_T>
468 template <
class Data_T>
474 template <
class Data_T>
480 template <
class Data_T>
485 template <
class Data_T>
486 int getNextId(
const std::string filename,
const std::string layerPath);
488 template <
class Data_T>
489 void removeFieldFromCache(
int refIdx);
500 void addBlockToCache(
DataTypeEnum blockType,
int fileId,
int blockIdx);
504 void deallocateBlocks(int64_t bytesNeeded);
508 template <
class Data_T>
512 template <
class Data_T>
513 void deallocateBlock(CacheList::iterator &it);
557 template <
class Data_T>
559 const std::string a_layerPath)
560 : filename(a_filename), layerPath(a_layerPath),
561 valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
562 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
570 template <
class Data_T>
580 delete [] blockMutex;
585 template <
class Data_T>
588 m_ogReaderPtr.reset();
597 template <
class Data_T>
619 #if F3D_SHORT_MUTEX_ARRAY
620 # if F3D_NO_BLOCKS_ARRAY
623 blockMutexSize =
std::min(
static_cast<int>(blocks.size()),
626 blockMutex =
new boost::mutex[blockMutexSize];
628 blockMutex =
new boost::mutex[blocks.size()];
637 if (m_fileHandle >= 0) {
638 m_layerGroup.open(m_fileHandle, layerPath.c_str());
649 m_ogReaderPtr.reset();
657 template <
class Data_T>
660 const std::string a_layerPath)
667 template <
class Data_T>
670 return m_fileHandle >= 0;
675 template <
class Data_T>
678 boost::mutex::scoped_lock lock(m_mutex);
681 numBlocks = a_numBlocks;
683 fileBlockIndices.resize(numBlocks);
684 blockLoaded.resize(numBlocks, 0);
685 #if !F3D_NO_BLOCKS_ARRAY
686 blocks.resize(numBlocks, 0);
688 blockUsed.resize(numBlocks,
false);
689 loadCounts.resize(numBlocks, 0);
690 refCounts.resize(numBlocks, 0);
693 #if F3D_SHORT_MUTEX_ARRAY
694 # if F3D_NO_BLOCKS_ARRAY
697 blockMutexSize =
std::min(blocks.size(),
700 blockMutex =
new boost::mutex[blockMutexSize];
702 blockMutex =
new boost::mutex[numBlocks];
708 template <
class Data_T>
711 if (m_fileHandle >= 0) {
712 if (H5Fclose(m_fileHandle) < 0) {
713 Msg::print(
"In ~Reference(): Error closing file " + filename);
720 template <
class Data_T>
724 #if F3D_NO_BLOCKS_ARRAY
725 blocks[blockIdx].clear();
727 blocks[blockIdx]->clear();
730 blockLoaded[blockIdx] = 0;
735 if (m_numActiveBlocks == 0) {
743 template <
class Data_T>
746 #if F3D_SHORT_MUTEX_ARRAY
747 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
749 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
751 ++refCounts[blockIdx];
756 template <
class Data_T>
759 #if F3D_SHORT_MUTEX_ARRAY
760 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
762 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
764 --refCounts[blockIdx];
769 template <
class Data_T>
772 return numVoxels *
sizeof(Data_T);
777 template <
class Data_T>
780 std::vector<int>::const_iterator i = loadCounts.begin();
781 std::vector<int>::const_iterator end = loadCounts.end();
783 for (; i != end; ++i)
791 template <
class Data_T>
794 std::vector<int>::const_iterator i = blockLoaded.begin();
795 std::vector<int>::const_iterator end = blockLoaded.end();
796 int numBlockCounter = 0;
797 for (; i != end; ++i)
801 return numBlockCounter;
806 template <
class Data_T>
809 std::vector<int>::const_iterator i = loadCounts.begin();
810 std::vector<int>::const_iterator li = blockLoaded.begin();
811 std::vector<int>::const_iterator end = loadCounts.end();
812 int numBlockCounter = 0;
814 if (blockLoaded.size() == 0) {
815 for (; i != end; ++i)
819 assert(loadCounts.size() == blockLoaded.size());
821 for (; i != end; ++i, ++li)
826 return numBlockCounter;
831 template <
class Data_T>
834 std::vector<int>::const_iterator i = loadCounts.begin();
835 std::vector<int>::const_iterator end = loadCounts.end();
836 int numLoads = 0, numBlockCounter = 0;
837 for (; i != end; ++i) {
844 return (
float)numLoads /
std::max(1, numBlockCounter);
849 template <
class Data_T>
852 std::vector<int>::iterator li = loadCounts.begin();
853 std::vector<int>::iterator lend = loadCounts.end();
854 for (; li != lend; ++li)
860 template <
class Data_T>
864 boost::mutex::scoped_lock lock(m_mutex);
866 return sizeof(*this) +
867 fileBlockIndices.capacity() *
sizeof(int) +
868 blockLoaded.capacity() *
sizeof(int) +
872 blockUsed.capacity() *
sizeof(bool) +
873 loadCounts.capacity() *
sizeof(int) +
874 refCounts.capacity() *
sizeof(int) +
876 blockMutexSize *
sizeof(boost::mutex) +
878 numBlocks *
sizeof(boost::mutex) +
898 for (
size_t i = 0, end =
m_hRefs.size(); i < end; ++i) {
901 for (
size_t i = 0, end =
m_fRefs.size(); i < end; ++i) {
904 for (
size_t i = 0, end =
m_dRefs.size(); i < end; ++i) {
907 for (
size_t i = 0, end =
m_vhRefs.size(); i < end; ++i) {
910 for (
size_t i = 0, end =
m_vfRefs.size(); i < end; ++i) {
913 for (
size_t i = 0, end =
m_vdRefs.size(); i < end; ++i) {
948 inline Reference<float>*
961 inline Reference<V3f>*
974 inline Reference<double>*
987 inline Reference<V3d>*
1002 Mutex::scoped_lock lock(
m_mutex);
1013 Mutex::scoped_lock lock(
m_mutex);
1024 Mutex::scoped_lock lock(
m_mutex);
1035 Mutex::scoped_lock lock(
m_mutex);
1046 Mutex::scoped_lock lock(
m_mutex);
1057 Mutex::scoped_lock lock(
m_mutex);
1066 inline size_t FileReferences::numRefs<half>()
const
1068 Mutex::scoped_lock lock(
m_mutex);
1076 inline size_t FileReferences::numRefs<V3h>()
const
1078 Mutex::scoped_lock lock(
m_mutex);
1086 inline size_t FileReferences::numRefs<float>()
const
1088 Mutex::scoped_lock lock(
m_mutex);
1096 inline size_t FileReferences::numRefs<V3f>()
const
1098 Mutex::scoped_lock lock(
m_mutex);
1106 inline size_t FileReferences::numRefs<double>()
const
1108 Mutex::scoped_lock lock(
m_mutex);
1116 inline size_t FileReferences::numRefs<V3d>()
const
1118 Mutex::scoped_lock lock(
m_mutex);
1129 template <
class Data_T>
1132 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1133 "simple scalars or vectors!");
1135 "FileReferences::ref(): Do not use memory limiting on sparse "
1136 "fields that aren't simple scalars or vectors!");
1143 template <
class Data_T>
1146 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1147 "simple scalars or vectors!");
1149 "FileReferences::append(): Do not use memory limiting on sparse "
1150 "fields that aren't simple scalars or vectors!");
1156 template <
class Data_T>
1159 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1160 "simple scalars or vectors!");
1162 "FileReferences::numRefs(): "
1163 "Do not use memory limiting on sparse "
1164 "fields that aren't "
1165 "simple scalars or vectors!");
1179 template <
class Data_T>
1182 const std::string layerPath)
1187 boost::mutex::scoped_lock lock(
m_mutex);
1196 template <
class Data_T>
1200 boost::mutex::scoped_lock lock(
m_mutex);
1207 CacheList::iterator next;
1209 int64_t bytesFreed = 0;
1212 if (it->blockType == blockType && it->refIdx == refIdx) {
1218 bytesFreed +=
reference->blockSize(it->blockIdx);
1227 std::vector<int>().swap(
reference->fileBlockIndices);
1228 #if F3D_NO_BLOCKS_ARRAY
1235 std::vector<int>().swap(
reference->blockLoaded);
1236 std::vector<bool>().swap(
reference->blockUsed);
1237 std::vector<int>().swap(
reference->loadCounts);
1238 std::vector<int>().swap(
reference->refCounts);
1245 template <
class Data_T>
1254 template <
class Data_T>
1260 if (
reference->fileBlockIndices[blockIdx] >= 0) {
1261 if (!
reference->blockLoaded[blockIdx]) {
1262 int blockSize =
reference->blockSize(blockIdx);
1273 boost::mutex::scoped_lock lock_A(
m_mutex);
1274 #if F3D_SHORT_MUTEX_ARRAY
1275 boost::mutex::scoped_lock
1278 boost::mutex::scoped_lock lock_B(
reference->blockMutex[blockIdx]);
1282 if (!
reference->blockLoaded[blockIdx]) {
1295 template <
class Data_T>
1301 if (
reference->fileBlockIndices[blockIdx] >= 0) {
1308 template <
class Data_T>
1314 if (
reference->fileBlockIndices[blockIdx] >= 0) {
Contains Exception base class.
Contains various utility functions for Hdf5.
Contains forward declarations for Ogawa classes.
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
boost::shared_ptr< OgIGroup > OgIGroupPtr
Contains functions controlling the loading of sparse fields.
#define F3D_SHORT_MUTEX_ARRAY
#define F3D_NO_BLOCKS_ARRAY
#define F3D_MUTEX_ARRAY_SIZE
Scoped object - opens a group on creation and closes it on destruction.
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
This Field subclass stores voxel data in block-allocated arrays.
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
void activateBlock(const int blockId) const
Activates a given block.
virtual long long int memSize() const
Returns the memory usage (in bytes)
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
int64_t m_memUse
Current amount of memory in use in bytes.
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user,...
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
static SparseFileManager * ms_singleton
Pointer to singleton.
void removeFieldFromCache(int refIdx)
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
std::list< SparseFile::CacheBlock > CacheList
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
float m_maxMemUse
Max amount om memory to use in megabytes.
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
long long int memSize() const
Returns the memory use for the refs.
std::deque< Reference< double >::Ptr > m_dRefs
std::deque< Reference< float >::Ptr > m_fRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
size_t numRefs() const
Returns the number of file references of the corresponding collection.
std::deque< Reference< V3f >::Ptr > m_vfRefs
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
std::deque< Reference< half >::Ptr > m_hRefs
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
std::deque< Reference< V3h >::Ptr > m_vhRefs
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
std::vector< int > fileBlockIndices
Index in file for each block.
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
OgIGroupPtr m_ogRoot
Ogawa archive root.
Sparse::SparseBlock< Data_T > * BlockPtrs
long long int memSize() const
Memory use for the Reference.
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
IArchivePtr m_ogArchive
Ogawa archive.
void closeFile()
Closes the file. Will be re-opened as needed.
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
int blockMutexSize
Size of the mutex array. Used as modulus base.
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks),...
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
void decBlockRef(int blockIdx)
Decrement reference count on a block.
size_t m_numActiveBlocks
Number of currently active blocks.
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
void resetCacheStatistics()
Resets counts of total block loads.
bool fileIsOpen()
Checks if the file used by this reference is open already.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Namespace for sparse field specifics.
T max(const T a, const T2 b)
Max operation on mixed types.
T min(const T a, const T2 b)
Min operation on mixed types.
#define FIELD3D_NAMESPACE_HEADER_CLOSE
static DataTypeEnum typeEnum()
Storage for one individual block of a SparseField.