00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #ifndef OPENMS_SYSTEM_EXTERNALALLOCATOR_H
00028 #define OPENMS_SYSTEM_EXTERNALALLOCATOR_H
00029
00030 #include <limits>
00031 #include <iostream>
00032 #include <stdio.h>
00033 #include <stdlib.h>
00034 #include <sys/stat.h>
00035 #include <sys/types.h>
00036 #include <fcntl.h>
00037 #include <math.h>
00038
00039
00040
00041 #include <OpenMS/config.h>
00042 #include <OpenMS/CONCEPT/Exception.h>
00043 #include <OpenMS/DATASTRUCTURES/String.h>
00044 #include <OpenMS/SYSTEM/File.h>
00045 #include <OpenMS/SYSTEM/ExternalAllocatorUnique.h>
00046 #include <OpenMS/SYSTEM/MemoryMap.h>
00047 #include <boost/shared_ptr.hpp>
00048
00049
00050 namespace OpenMS
00051 {
00052
00062 template <class T>
00063 class ExternalAllocator {
00064
00065 protected:
00066
00068 boost::shared_ptr<ExternalAllocatorUnique> shared_extalloc_;
00069
00070 public:
00071
00073 template < typename T_ > friend class ExternalAllocator;
00074
00075
00076 typedef T value_type;
00077 typedef T* pointer;
00078 typedef const T* const_pointer;
00079 typedef T& reference;
00080 typedef const T& const_reference;
00081 typedef std::size_t size_type;
00082 typedef std::ptrdiff_t difference_type;
00083
00085 template <class U>
00086 struct rebind
00087 {
00088 typedef ExternalAllocator<U> other;
00089 };
00090
00092 pointer address (reference value) const
00093 {
00094 std::cout << "address of " << value << " is " << &value << "\n";
00095 return &value;
00096 }
00098 const_pointer address (const_reference value) const
00099 {
00100 std::cout << "address of " << value << " is " << &value << " (c)\n";
00101 return &value;
00102 }
00103
00104
00105
00106
00108 ExternalAllocator(const String& filename = File::getUniqueName(), const Offset64Int &filesize = 1)
00109 {
00110 #ifdef DEBUG_ALLOC
00111 std::cout << "<<-->> 2-tuple Ctor called \n";
00112 #endif
00113
00114
00115 ExternalAllocatorUnique * ea = new ExternalAllocatorUnique(filename, filesize);
00116 shared_extalloc_.reset( ea );
00117
00118 }
00119
00121 ExternalAllocator(const ExternalAllocator& rhs) throw()
00122 :
00123 shared_extalloc_(rhs.shared_extalloc_)
00124 {
00125 #ifdef DEBUG_ALLOC
00126 std::cerr << "<<-->> Copy Ctor called with nextfree_ " << shared_extalloc_->getNextfree() << "\n";
00127 #endif
00128 }
00129
00131 template <class U>
00132 ExternalAllocator (const ExternalAllocator<U>& rhs) throw()
00133 :
00134 shared_extalloc_(rhs.shared_extalloc_)
00135 {
00136 #ifdef DEBUG_ALLOC
00137 std::cerr << "<<-->> !!strange!! Ctor called \n";
00138 #endif
00139 }
00140
00142 ~ExternalAllocator() throw()
00143 {
00144
00145 }
00146
00148 size_type max_size () const throw() {
00149 return shared_extalloc_->getFilesize() / sizeof(T);
00150 }
00151
00152
00154 pointer allocate (size_type num, const void* = 0)
00155 {
00156 pointer map = 0;
00157
00158
00159 if (num==0)
00160 {
00161 return map;
00162 }
00163
00164 size_type alloc_bytes = num*sizeof(T);
00165
00166 ldiv_t ldiff = ldiv( alloc_bytes, MemoryMap::OpenMS_getFileBlocksize() );
00167
00168 #ifdef DEBUG_ALLOC
00169 std::cout << "\n\n in:" << alloc_bytes << "/" << MemoryMap::OpenMS_getFileBlocksize() << "\n out:"<< ldiff.rem << " & " << ldiff.quot << "\n\n";
00170 #endif
00171
00172
00173 size_type block_bytes = (ldiff.rem > 0 ? ldiff.quot + 1 : ldiff.quot) * MemoryMap::OpenMS_getFileBlocksize();
00174
00175
00176 #ifdef DEBUG_ALLOC
00177 std::cout << "file space required: " << block_bytes << " blocksize: " << MemoryMap::OpenMS_getFileBlocksize() << std::endl;
00178 #endif
00179
00180
00181
00182
00183
00184
00185 #ifdef DEBUG_ALLOC
00186 std::cerr << "allocate " << num << " element(s)"
00187 << " of size " << sizeof(T) << "(( " << shared_extalloc_->getMmapHandle() << " & " << shared_extalloc_->getNextfree() << "))"
00188 << std::endl;
00189 #endif
00190
00191
00192 if (!shared_extalloc_->hasFreeSwap(block_bytes))
00193 {
00194
00195
00196 if (File::extendSparseFile(shared_extalloc_->getMmapHandle(), shared_extalloc_->getFilesize()+block_bytes))
00197 {
00198 shared_extalloc_->advanceFilesize(block_bytes);
00199 }
00200 else
00201 {
00202 std::cerr << "ExternalAllocator: Extending the swap file failed. Maybe you ran out of disk space!" << std::endl;
00203 return 0;
00204 }
00205
00206
00207 }
00208
00209 map = static_cast<pointer> (MemoryMap::OpenMS_mmap(alloc_bytes,
00210 shared_extalloc_->getMmapHandle(),
00211 shared_extalloc_->getNextfree()
00212 )
00213 );
00214
00215
00216
00217 if (map == MAP_FAILED) {
00218 std::cerr << "MAPPING FAILED: \n"
00219 << " blocksize " << block_bytes << "\n"
00220 << " nextfree: " << shared_extalloc_->getNextfree() << "( of allowed " << (shared_extalloc_->getFilesize() - block_bytes) << ")\n"
00221 << " totally mapped: " << shared_extalloc_->getTotalmappingsize() << std::endl;
00222 #ifndef OPENMS_64BIT_ARCHITECTURE
00223
00224 std::cerr << "The most common cause on 32-bit systems (like this one)"
00225 << " is lack of virtual address space, which is usually 2-3 GB large. See the 'totally mapped' for information about your system."
00226 << "\nUpdate to a 64-bit OS to circumvent this restriction or use smaller datasets."
00227 << std::endl;
00228
00229
00230
00231 #endif
00232 return 0;
00233 }
00234
00235 #ifdef DEBUG_ALLOC
00236 std::cerr << " allocated at MEM: " << (void*)map << " <-> filepos: " << shared_extalloc_->getNextfree() << std::endl;
00237 #endif
00238
00239
00240
00241 shared_extalloc_->advanceNextfree(block_bytes);
00242
00243 shared_extalloc_->setTotalmappingsize(shared_extalloc_->getTotalmappingsize() + block_bytes);
00244
00245
00246 #ifdef DEBUG_ALLOC
00247 FILE * pFile;
00248 String file = shared_extalloc_->getFilename() + ".log";
00249 pFile = fopen (file.c_str(),"a");
00250 if (pFile!=NULL)
00251 {
00252 String s((long long unsigned int)shared_extalloc_->getTotalmappingsize());
00253 String s2((long long unsigned int)shared_extalloc_->getNextfree());
00254 s = "totally mapped: " + s + " offset: " + s2 + "\n";
00255
00256 fputs (s.c_str(),pFile);
00257 fclose (pFile);
00258 }
00259
00260 std::cerr << " new filepos: " << shared_extalloc_->getNextfree() << std::endl;
00261 #endif
00262
00263 return map;
00264 }
00265
00267 void construct (pointer p, const T& value) {
00268
00269
00270
00271 new((void*)p)T(value);
00272 }
00273
00275 void destroy (pointer ) {
00276
00277
00278 }
00279
00281 void deallocate (pointer p, size_type num) {
00282 #ifdef DEBUG_ALLOC
00283 std::cerr << "deallocate " << num << " element(s)"
00284 << " of size " << sizeof(T)
00285 << " at: " << (void*)p << std::endl;
00286 #endif
00287
00288
00289 size_type alloc_bytes = num*sizeof(T);
00290 ldiv_t ldiff = ldiv( alloc_bytes, MemoryMap::OpenMS_getFileBlocksize() );
00291 size_type block_bytes = (ldiff.rem > 0 ? ldiff.quot + 1 : ldiff.quot) * MemoryMap::OpenMS_getFileBlocksize();
00292
00293
00294 shared_extalloc_->setTotalmappingsize(shared_extalloc_->getTotalmappingsize() - block_bytes);
00295
00296
00297
00298 int result = MemoryMap::OpenMS_unmap(p, alloc_bytes);
00299 if (result == OPENMS_MUNMAP_FAILURE)
00300 {
00301 std::cerr << "Severe WARNING: unable to unmap memory at " << p << std::endl;
00302 }
00303 }
00304
00306 Offset64Int getMappingSize()
00307 {
00308 return shared_extalloc_->getTotalmappingsize();
00309 }
00310
00311 };
00312
00314 template <class T1, class T2>
00315 bool operator== (const ExternalAllocator<T1>&,
00316 const ExternalAllocator<T2>&) throw() {
00317 return false;
00318 }
00319 template <class T1, class T2>
00320 bool operator!= (const ExternalAllocator<T1>&,
00321 const ExternalAllocator<T2>&) throw() {
00322 return true;
00323 }
00324
00325
00326
00327 }
00328 #endif //OPENMS_SYSTEM_EXTERNALALLOCATOR_H