Home  · Classes  · Annotated Classes  · Modules  · Members  · Namespaces  · Related Pages

ExternalAllocator.h (Maintainer: Chris Bielow)

Go to the documentation of this file.
00001 // -*- Mode: C++; tab-width: 2; -*-
00002 // vi: set ts=2:
00003 //
00004 // --------------------------------------------------------------------------
00005 //                   OpenMS Mass Spectrometry Framework
00006 // --------------------------------------------------------------------------
00007 //  Copyright (C) 2003-2008 -- Oliver Kohlbacher, Knut Reinert
00008 //
00009 //  This library is free software; you can redistribute it and/or
00010 //  modify it under the terms of the GNU Lesser General Public
00011 //  License as published by the Free Software Foundation; either
00012 //  version 2.1 of the License, or (at your option) any later version.
00013 //
00014 //  This library is distributed in the hope that it will be useful,
00015 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
00016 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00017 //  Lesser General Public License for more details.
00018 //
00019 //  You should have received a copy of the GNU Lesser General Public
00020 //  License along with this library; if not, write to the Free Software
00021 //  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00022 //
00023 // --------------------------------------------------------------------------
00024 // $Maintainer: Chris Bielow  $
00025 // --------------------------------------------------------------------------
00026 
00027 #ifndef OPENMS_SYSTEM_EXTERNALALLOCATOR_H
00028 #define OPENMS_SYSTEM_EXTERNALALLOCATOR_H
00029 
00030 #include <limits>
00031 #include <iostream>
00032 #include <stdio.h>
00033 #include <stdlib.h>
00034 #include <sys/stat.h>
00035 #include <sys/types.h>
00036 #include <fcntl.h> // for O_RDWR etc
00037 #include <math.h>  // for ldiv()
00038 
00039 
00040 
00041 #include <OpenMS/config.h>
00042 #include <OpenMS/CONCEPT/Exception.h>
00043 #include <OpenMS/DATASTRUCTURES/String.h>
00044 #include <OpenMS/SYSTEM/File.h>
00045 #include <OpenMS/SYSTEM/ExternalAllocatorUnique.h>
00046 #include <OpenMS/SYSTEM/MemoryMap.h>
00047 #include <boost/shared_ptr.hpp>
00048 
00049 
00050 namespace OpenMS
00051 {
00052 
00062   template <class T>
00063   class ExternalAllocator {
00064   
00065     protected:
00066 
00068       boost::shared_ptr<ExternalAllocatorUnique> shared_extalloc_;
00069     
00070     public:
00071       
00073       template < typename T_ > friend class ExternalAllocator;
00074     
00075       // type definitions
00076       typedef T        value_type;
00077       typedef T*       pointer;
00078       typedef const T* const_pointer;
00079       typedef T&       reference;
00080       typedef const T& const_reference;
00081       typedef std::size_t    size_type;
00082       typedef std::ptrdiff_t difference_type;
00083 
00085       template <class U>
00086       struct rebind 
00087       {
00088           typedef ExternalAllocator<U> other;
00089       };
00090 
00092       pointer address (reference value) const 
00093       {
00094         std::cout << "address of " << value << " is " << &value << "\n";
00095           return &value;
00096       }
00098       const_pointer address (const_reference value) const 
00099       {
00100         std::cout << "address of " << value << " is " << &value << " (c)\n";
00101           return &value;
00102       }
00103 
00104       /* constructors and destructor
00105        */
00106     
00108       ExternalAllocator(const String& filename = File::getUniqueName(), const Offset64Int &filesize = 1) 
00109       {
00110         #ifdef DEBUG_ALLOC      
00111         std::cout << "<<-->> 2-tuple Ctor called \n";
00112         #endif          
00113 
00114         // assign our only member
00115         ExternalAllocatorUnique * ea = new ExternalAllocatorUnique(filename, filesize);
00116         shared_extalloc_.reset( ea );
00117           
00118       }
00119       
00121       ExternalAllocator(const ExternalAllocator& rhs) throw() 
00122       :
00123         shared_extalloc_(rhs.shared_extalloc_)
00124       {
00125         #ifdef DEBUG_ALLOC      
00126         std::cerr << "<<-->> Copy Ctor called with nextfree_ " << shared_extalloc_->getNextfree() << "\n";
00127         #endif          
00128       }
00129       
00131       template <class U>
00132       ExternalAllocator (const ExternalAllocator<U>& rhs) throw()
00133       :
00134         shared_extalloc_(rhs.shared_extalloc_)
00135       {
00136         #ifdef DEBUG_ALLOC      
00137         std::cerr << "<<-->> !!strange!! Ctor called \n";
00138         #endif          
00139       }
00140       
00142       ~ExternalAllocator() throw() 
00143       {
00144          // we should be good. shared_ptr should call dtor of data-object and delete tmp-file      
00145       }
00146 
00148       size_type max_size () const throw() {
00149           return shared_extalloc_->getFilesize() / sizeof(T);
00150       }
00151 
00152       
00154       pointer allocate (size_type num, const void* = 0) 
00155       {
00156           pointer map = 0;
00157           
00158           // when the default c'tor for a std::vector is called, it will call allocate() for 0 elements
00159           if (num==0)
00160           {
00161             return map;
00162           }
00163           
00164           size_type alloc_bytes = num*sizeof(T);
00165           // round up to next free page (file location needs to be page-aligned)
00166           ldiv_t ldiff =  ldiv(  alloc_bytes,  MemoryMap::OpenMS_getFileBlocksize() );
00167 
00168           #ifdef DEBUG_ALLOC
00169           std::cout << "\n\n in:" << alloc_bytes << "/" << MemoryMap::OpenMS_getFileBlocksize() << "\n out:"<< ldiff.rem << " & " << ldiff.quot << "\n\n";
00170           #endif          
00171 
00172                               
00173           size_type block_bytes = (ldiff.rem > 0 ? ldiff.quot + 1 : ldiff.quot) * MemoryMap::OpenMS_getFileBlocksize();
00174           
00175           
00176           #ifdef DEBUG_ALLOC
00177           std::cout << "file space required: " << block_bytes << " blocksize: " << MemoryMap::OpenMS_getFileBlocksize() << std::endl;
00178           #endif    
00179       
00180                
00181           /* Now the file is ready to be mmapped */
00182 
00183           
00184           // print message and allocate memory
00185           #ifdef DEBUG_ALLOC
00186           std::cerr << "allocate " << num << " element(s)"
00187                     << " of size " << sizeof(T) << "(( " << shared_extalloc_->getMmapHandle() << " & " << shared_extalloc_->getNextfree() << "))"
00188                     << std::endl;
00189           #endif          
00190 
00191           // check if swap file is big enough
00192           if (!shared_extalloc_->hasFreeSwap(block_bytes)) 
00193           {
00194             //std::cerr << "extending swap in ExternalAllocator from " << shared_extalloc_->getFilesize() << " by " << block_bytes << std::endl;
00195             // extend swap file
00196             if (File::extendSparseFile(shared_extalloc_->getMmapHandle(), shared_extalloc_->getFilesize()+block_bytes))
00197             {
00198               shared_extalloc_->advanceFilesize(block_bytes);
00199             }
00200             else
00201             {
00202               std::cerr << "ExternalAllocator: Extending the swap file failed. Maybe you ran out of disk space!" << std::endl;
00203               return 0;
00204             }
00205             
00206             
00207           }
00208           
00209           map = static_cast<pointer> (MemoryMap::OpenMS_mmap(alloc_bytes, 
00210                                                              shared_extalloc_->getMmapHandle(), 
00211                                                              shared_extalloc_->getNextfree()
00212                                                             )
00213                                      );
00214           
00215           
00216                                    
00217           if (map == MAP_FAILED) {
00218             std::cerr << "MAPPING FAILED:  \n"
00219                       << " blocksize " << block_bytes << "\n"
00220                       << " nextfree: " << shared_extalloc_->getNextfree() << "( of allowed " << (shared_extalloc_->getFilesize() - block_bytes) << ")\n"
00221                       << " totally mapped: " << shared_extalloc_->getTotalmappingsize() << std::endl;
00222             #ifndef OPENMS_64BIT_ARCHITECTURE
00223             
00224             std::cerr << "The most common cause on 32-bit systems (like this one)"
00225                       << " is lack of virtual address space, which is usually 2-3 GB large. See the 'totally mapped' for information about your system."              
00226                       << "\nUpdate to a 64-bit OS to circumvent this restriction or use smaller datasets."
00227                       << std::endl;
00228                       //<< "Alternatively, you forgot to use large file pointers by configuring OpenMS with\n\n"
00229                       //<< "'./configure  --with-cxxflags=\"-D_FILE_OFFSET_BITS=64 -D_LARGE_FILE_SOURCE\"'\n\n"
00230                       //<< std::endl;
00231             #endif
00232             return 0;
00233           }
00234           
00235           #ifdef DEBUG_ALLOC
00236           std::cerr << " allocated at MEM: " << (void*)map << " <-> filepos: " << shared_extalloc_->getNextfree() << std::endl;
00237           #endif          
00238           
00239           
00240           // set the file offset where the next mapping will start
00241           shared_extalloc_->advanceNextfree(block_bytes);
00242           
00243           shared_extalloc_->setTotalmappingsize(shared_extalloc_->getTotalmappingsize() + block_bytes);
00244           
00245           
00246           #ifdef DEBUG_ALLOC
00247           FILE * pFile;
00248           String file = shared_extalloc_->getFilename() + ".log";
00249           pFile = fopen (file.c_str(),"a");
00250           if (pFile!=NULL)
00251           {
00252             String s((long long unsigned int)shared_extalloc_->getTotalmappingsize());
00253             String s2((long long unsigned int)shared_extalloc_->getNextfree());
00254             s = "totally mapped: " + s + " offset: " + s2 + "\n";
00255             //std::cerr << "@@@ printing:" << s << std::endl;
00256             fputs (s.c_str(),pFile);
00257             fclose (pFile);
00258           }
00259           
00260           std::cerr << " new filepos: " << shared_extalloc_->getNextfree() << std::endl;
00261           #endif          
00262           
00263           return map;
00264       } // end of allocate
00265 
00267       void construct (pointer p, const T& value) {
00268           // initialize memory with placement new
00269           //this function is essential because the vector will call this function 
00270           //whenever it needs to assign a value on an element (e.g. "push_back" or preinitialized constructor)
00271           new((void*)p)T(value);
00272       }
00273 
00275       void destroy (pointer /*p*/) {
00276           //TODO: is that really necessary?! maybe do two versions.. one fast&dangerous, one slow&secure (benchmark!)
00277           //p->~T();
00278       }
00279 
00281       void deallocate (pointer p, size_type num) {
00282           #ifdef DEBUG_ALLOC
00283           std::cerr << "deallocate " << num << " element(s)"
00284                     << " of size " << sizeof(T)
00285                     << " at: " << (void*)p << std::endl;
00286           #endif
00287           
00288           // round up to the next page size
00289           size_type alloc_bytes = num*sizeof(T);
00290           ldiv_t ldiff =  ldiv(  alloc_bytes,  MemoryMap::OpenMS_getFileBlocksize() );
00291           size_type block_bytes = (ldiff.rem > 0 ? ldiff.quot + 1 : ldiff.quot) * MemoryMap::OpenMS_getFileBlocksize();
00292           
00293           
00294           shared_extalloc_->setTotalmappingsize(shared_extalloc_->getTotalmappingsize() - block_bytes);
00295           
00296           //TODO: look at int madvise (void *addr, size_t length, int advice), especially 
00297           /* advice = POSIX_MADV_DONTNEED: The region is no longer needed. The kernel may free these pages, causing any changes to the pages to be lost, as well as swapped out pages to be discarded. */
00298           int result = MemoryMap::OpenMS_unmap(p, alloc_bytes);
00299           if (result == OPENMS_MUNMAP_FAILURE)
00300           { // this is not fatal, but still severe!
00301             std::cerr << "Severe WARNING: unable to unmap memory at " << p << std::endl;
00302           }
00303       }
00304       
00306       Offset64Int getMappingSize()
00307       {
00308         return shared_extalloc_->getTotalmappingsize();
00309       }
00310       
00311   }; //end class
00312 
00314   template <class T1, class T2>
00315   bool operator== (const ExternalAllocator<T1>&,
00316                     const ExternalAllocator<T2>&) throw() {
00317       return false;
00318   }
00319   template <class T1, class T2>
00320   bool operator!= (const ExternalAllocator<T1>&,
00321                     const ExternalAllocator<T2>&) throw() {
00322       return true;
00323   }
00324 
00325   
00326     
00327 }
00328 #endif //OPENMS_SYSTEM_EXTERNALALLOCATOR_H 

Generated Tue Apr 1 15:36:34 2008 -- using doxygen 1.5.4 OpenMS / TOPP 1.1