root/OpenSceneGraph/trunk/src/osgDB/DatabasePager.cpp @ 10520

Revision 10520, 77.4 kB (checked in by robert, 5 years ago)

Moved the updating and expiry of the Registry object cache from DatabasePager? into osgViewer::Viewer/CompositeViewer.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
RevLine 
[10415]1/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
[8633]2 *
3 * This library is open source and may be redistributed and/or modified under 
4 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
5 * (at your option) any later version.  The full license is in LICENSE file
6 * included with this distribution, and on the openscenegraph.org website.
7 *
8 * This library is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 * OpenSceneGraph Public License for more details.
12*/
13
[3422]14#include <osgDB/DatabasePager>
[8325]15#include <osgDB/WriteFile>
[8322]16#include <osgDB/FileNameUtils>
[8325]17#include <osgDB/FileUtils>
[8541]18#include <osgDB/Registry>
[3008]19
[2053]20#include <osg/Geode>
21#include <osg/Timer>
22#include <osg/Texture>
[2207]23#include <osg/Notify>
[8325]24#include <osg/ProxyNode>
[4307]25#include <osg/ApplicationUsage>
[2053]26
[2366]27#include <OpenThreads/ScopedLock>
28
[3999]29#include <algorithm>
[7662]30#include <functional>
31#include <set>
[3999]32
[9475]33#include <stdlib.h>
[9343]34#include <string.h>
35
[2053]36#ifdef WIN32
37#include <windows.h>
38#else
39#include <unistd.h>
40#endif
41
42using namespace osgDB;
[3419]43using namespace OpenThreads;
[2053]44
[4308]45static osg::ApplicationUsageProxy DatabasePager_e0(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DO_PRE_COMPILE <ON/OFF>","Switch on or off the pre compile of OpenGL object database pager.");
46static osg::ApplicationUsageProxy DatabasePager_e1(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_MINIMUM_COMPILE_TIME_PER_FRAME <float>","minimum compile time alloted to compiling OpenGL objects per frame in database pager.");
47static osg::ApplicationUsageProxy DatabasePager_e2(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_MAXIMUM_OBJECTS_TO_COMPILE_PER_FRAME <int>","maximum number of OpenGL objects to compile per frame in database pager.");
[4970]48static osg::ApplicationUsageProxy DatabasePager_e3(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DATABASE_PAGER_DRAWABLE <mode>","Set the drawable policy for setting of loaded drawable to specified type.  mode can be one of DoNotModify, DisplayList, VBO or VertexArrays>.");
[7287]49static osg::ApplicationUsageProxy DatabasePager_e4(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DATABASE_PAGER_PRIORITY <mode>", "Set the thread priority to DEFAULT, MIN, LOW, NOMINAL, HIGH or MAX.");
[7894]50static osg::ApplicationUsageProxy DatabasePager_e7(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_EXPIRY_DELAY <float> ","Set the length of time a PagedLOD child is kept in memory, without being used, before its tagged as expired, and ear marked to deletion.");
[8766]51static osg::ApplicationUsageProxy DatabasePager_e8(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_EXPIRY_FRAMES <int> ","Set number of frames a PagedLOD child is kept in memory, without being used, before its tagged as expired, and ear marked to deletion.");
[8994]52static osg::ApplicationUsageProxy DatabasePager_e9(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_RELEASE_DELAY <float> ","Set the length of time a PagedLOD child's OpenGL objects are kept in memory, without being used, before be released (setting to OFF disables this feature.)");
53static osg::ApplicationUsageProxy DatabasePager_e10(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_RELEASE_FRAMES <int> ","Set number of frames a PagedLOD child's OpenGL objects are kept in memory, without being used, before be released.");
[9068]54static osg::ApplicationUsageProxy DatabasePager_e11(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_MAX_PAGEDLOD <num>","Set the target maximum number of PagedLOD to maintain.");
55
[7662]56// Convert function objects that take pointer args into functions that a
57// reference to an osg::ref_ptr. This is quite useful for doing STL
58// operations on lists of ref_ptr. This code assumes that a function
59// with an argument const Foo* should be composed into a function of
60// argument type ref_ptr<Foo>&, not ref_ptr<const Foo>&. Some support
61// for that should be added to make this more general.
62
63namespace
64{
65template <typename U>
66struct PointerTraits
67{
68    typedef class NullType {} PointeeType;
69};
70
71template <typename U>
72struct PointerTraits<U*>
73{
74    typedef U PointeeType;
75};
76
77template <typename U>
78struct PointerTraits<const U*>
79{
80    typedef U PointeeType;
81};
82
83template <typename FuncObj>
84class RefPtrAdapter
85    : public std::unary_function<const osg::ref_ptr<typename PointerTraits<typename FuncObj::argument_type>::PointeeType>,
86                                 typename FuncObj::result_type>
87{
88public:
89    typedef typename PointerTraits<typename FuncObj::argument_type>::PointeeType PointeeType;
90    typedef osg::ref_ptr<PointeeType> RefPtrType;
91    explicit RefPtrAdapter(const FuncObj& funcObj) : _func(funcObj) {}
92    typename FuncObj::result_type operator()(const RefPtrType& refPtr) const
93    {
94        return _func(refPtr.get());
95    }
96protected:
97        FuncObj _func;
98};
99
100template <typename FuncObj>
101RefPtrAdapter<FuncObj> refPtrAdapt(const FuncObj& func)
102{
103    return RefPtrAdapter<FuncObj>(func);
104}
105}
106
[8325]107/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
108//
109//  FindCompileableGLObjectsVisitor
110//
111class DatabasePager::FindCompileableGLObjectsVisitor : public osg::NodeVisitor
112{
113public:
[10054]114    FindCompileableGLObjectsVisitor(DatabasePager::DataToCompile* dataToCompile,
[8325]115                               bool changeAutoUnRef, bool valueAutoUnRef,
116                               bool changeAnisotropy, float valueAnisotropy,
117                               DatabasePager::DrawablePolicy drawablePolicy,
118                               const DatabasePager* pager):
119            osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN),
120            _dataToCompile(dataToCompile),
121            _changeAutoUnRef(changeAutoUnRef), _valueAutoUnRef(valueAutoUnRef),
122            _changeAnisotropy(changeAnisotropy), _valueAnisotropy(valueAnisotropy),
123            _drawablePolicy(drawablePolicy), _pager(pager)
124    {
[10171]125        if (osgDB::Registry::instance()->getBuildKdTreesHint()==osgDB::Options::BUILD_KDTREES &&
[8543]126            osgDB::Registry::instance()->getKdTreeBuilder())
127        {
128            _kdTreeBuilder = osgDB::Registry::instance()->getKdTreeBuilder()->clone();
129        }
[8325]130    }
[10054]131
[9377]132    META_NodeVisitor("osgDB","FindCompileableGLObjectsVisitor")
[10054]133
[8325]134    virtual void apply(osg::Node& node)
135    {
136        apply(node.getStateSet());
137
138        traverse(node);
139    }
[10054]140
[8325]141    virtual void apply(osg::Geode& geode)
142    {
143        apply(geode.getStateSet());
[10054]144
[8325]145        for(unsigned int i=0;i<geode.getNumDrawables();++i)
146        {
147            apply(geode.getDrawable(i));
148        }
149
150        traverse(geode);
[10054]151
[8543]152        if (_kdTreeBuilder.valid())
153        {
154            geode.accept(*_kdTreeBuilder);
155        }
[8325]156    }
[10054]157
[8325]158    inline void apply(osg::StateSet* stateset)
159    {
160        if (stateset)
161        {
162            // search for the existance of any texture object
163            // attributes
164            // if texture object attributes exist and need to be
165            // compiled, add the state to the list for later
166            // compilation.
167            bool compileStateSet = false;
168            for(unsigned int i=0;i<stateset->getTextureAttributeList().size();++i)
169            {
170                osg::Texture* texture = dynamic_cast<osg::Texture*>(stateset->getTextureAttribute(i,osg::StateAttribute::TEXTURE));
171                // Has this texture already been encountered?
172                if (texture && !_textureSet.count(texture))
173                {
174                    _textureSet.insert(texture);
175                    if (_changeAutoUnRef) texture->setUnRefImageDataAfterApply(_valueAutoUnRef);
176                    if ((_changeAnisotropy
177                         && texture->getMaxAnisotropy() != _valueAnisotropy))
178                    {
179                        if (_changeAnisotropy)
180                            texture->setMaxAnisotropy(_valueAnisotropy);
181                    }
[10054]182
[8325]183                    if (!_pager->isCompiled(texture))
184                    {
185                        compileStateSet = true;
186                        if (osg::getNotifyLevel() >= osg::DEBUG_INFO)
187                        {
188                            osg::notify(osg::DEBUG_INFO)
189                                <<"Found compilable texture " << texture << " ";
190                            osg::Image* image = texture->getImage(0);
191                            if (image) osg::notify(osg::DEBUG_INFO) << image->getFileName();
192                            osg::notify(osg::DEBUG_INFO) << std:: endl;
193                        }
194                        break;
195                    }
196                }
197            }
[10054]198            if (compileStateSet && _dataToCompile)
[8325]199            {
[10054]200                _dataToCompile->first.insert(stateset);
[8325]201            }
202
203        }
204    }
[10054]205
[8325]206    inline void apply(osg::Drawable* drawable)
207    {
208        if (_drawableSet.count(drawable))
209            return;
[8944]210
211        _drawableSet.insert(drawable);
[10054]212
[8325]213        apply(drawable->getStateSet());
[10054]214
[8325]215        switch(_drawablePolicy)
216        {
217        case DatabasePager::DO_NOT_MODIFY_DRAWABLE_SETTINGS:
218             // do nothing, leave settings as they came in from loaded database.
219             // osg::notify(osg::NOTICE)<<"DO_NOT_MODIFY_DRAWABLE_SETTINGS"<<std::endl;
220             break;
221        case DatabasePager::USE_DISPLAY_LISTS:
222             drawable->setUseDisplayList(true);
223             drawable->setUseVertexBufferObjects(false);
224             break;
225        case DatabasePager::USE_VERTEX_BUFFER_OBJECTS:
226             drawable->setUseDisplayList(true);
227             drawable->setUseVertexBufferObjects(true);
228             // osg::notify(osg::NOTICE)<<"USE_VERTEX_BUFFER_OBJECTS"<<std::endl;
229             break;
230        case DatabasePager::USE_VERTEX_ARRAYS:
231             drawable->setUseDisplayList(false);
232             drawable->setUseVertexBufferObjects(false);
233             // osg::notify(osg::NOTICE)<<"USE_VERTEX_ARRAYS"<<std::endl;
234             break;
235        }
236        // Don't compile if already compiled. This can happen if the
237        // subgraph is shared with already-loaded nodes.
238        //
239        // XXX This "compiles" VBOs too, but compilation doesn't do
240        // anything for VBOs, does it?
[10054]241        if (_dataToCompile && drawable->getUseDisplayList() && !_pager->isCompiled(drawable))
[8325]242        {
243            // osg::notify(osg::NOTICE)<<"  Found compilable drawable"<<std::endl;
[10054]244            _dataToCompile->second.push_back(drawable);
[8325]245        }
246    }
247   
[10054]248    DatabasePager::DataToCompile*           _dataToCompile;
[8543]249    bool                                    _changeAutoUnRef;
250    bool                                    _valueAutoUnRef;
251    bool                                    _changeAnisotropy;
252    float                                   _valueAnisotropy;
253    DatabasePager::DrawablePolicy           _drawablePolicy;
254    const DatabasePager*                    _pager;
255    std::set<osg::ref_ptr<osg::Texture> >   _textureSet;
256    std::set<osg::ref_ptr<osg::Drawable> >  _drawableSet;
257    osg::ref_ptr<osg::KdTreeBuilder>        _kdTreeBuilder;
[9630]258   
259protected:
260
261    FindCompileableGLObjectsVisitor& operator = (const FindCompileableGLObjectsVisitor&) { return *this; }
[8325]262};
263
264
265/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
266//
[8510]267//  SortFileRequestFunctor
268//
269struct DatabasePager::SortFileRequestFunctor
270{
271    bool operator() (const osg::ref_ptr<DatabasePager::DatabaseRequest>& lhs,const osg::ref_ptr<DatabasePager::DatabaseRequest>& rhs) const
272    {
273        if (lhs->_timestampLastRequest>rhs->_timestampLastRequest) return true;
274        else if (lhs->_timestampLastRequest<rhs->_timestampLastRequest) return false;
275        else return (lhs->_priorityLastRequest>rhs->_priorityLastRequest);
276    }
277};
278
279
280/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
281//
[8434]282//  ReadQueue
[8325]283//
[8510]284void DatabasePager::RequestQueue::sort()
285{
[8511]286    std::sort(_requestList.begin(),_requestList.end(),SortFileRequestFunctor());
[8510]287}
288
289
290/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
291//
292//  ReadQueue
293//
[8434]294DatabasePager::ReadQueue::ReadQueue(DatabasePager* pager, const std::string& name):
[8325]295    _pager(pager),
296    _name(name)
297{
298    _block = new osg::RefBlock;
299}
300
[8434]301void DatabasePager::ReadQueue::clear()
[8325]302{
303    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
[8434]304
305    for(RequestList::iterator citr = _requestList.begin();
306        citr != _requestList.end();
307        ++citr)
308    {
309        (*citr)->_loadedModel = 0;
310        (*citr)->_requestQueue = 0;
311    }
312
[8325]313    _requestList.clear();
314
315    updateBlock();
316}
317
[8434]318void DatabasePager::ReadQueue::add(DatabasePager::DatabaseRequest* databaseRequest)
[8325]319{
320    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
321    _requestList.push_back(databaseRequest);
[8434]322    databaseRequest->_requestQueue = this;
[8325]323
324    updateBlock();
325}
326
[8434]327void DatabasePager::ReadQueue::takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest)
[8325]328{
329    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
330
331    if (!_requestList.empty())
332    {
[8510]333        sort();
334   
[8325]335        databaseRequest = _requestList.front();
[8434]336        databaseRequest->_requestQueue = 0;
[8325]337        _requestList.erase(_requestList.begin());
[8403]338
339        updateBlock();
[8325]340    }
341}
342
343/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
344//
345//  DatabaseThread
346//
347DatabasePager::DatabaseThread::DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name):
348    _done(false),
[9257]349    _active(false),
[8325]350    _pager(pager),
351    _mode(mode),
352    _name(name)
353{
354}
355
356DatabasePager::DatabaseThread::DatabaseThread(const DatabaseThread& dt, DatabasePager* pager):
357    _done(false),
[9257]358    _active(false),
[8325]359    _pager(pager),
360    _mode(dt._mode),
361    _name(dt._name)
362{
363   
364}
365
366DatabasePager::DatabaseThread::~DatabaseThread()
367{
368    cancel();
369}
370
371int DatabasePager::DatabaseThread::cancel()
372{
373    int result = 0;
374
375    if( isRunning() )
376    {
377   
378        _done = true;
[8328]379       
380        switch(_mode)
381        {
382            case(HANDLE_ALL_REQUESTS):
383                _pager->_fileRequestQueue->release();
384                break;
385            case(HANDLE_NON_HTTP):
386                _pager->_fileRequestQueue->release();
387                break;
388            case(HANDLE_ONLY_HTTP):
389                _pager->_httpRequestQueue->release();
390                break;
391        }
[8325]392
393        // release the frameBlock and _databasePagerThreadBlock in case its holding up thread cancellation.
394        // _databasePagerThreadBlock->release();
395
396        // then wait for the the thread to stop running.
397        while(isRunning())
398        {
399            // commenting out debug info as it was cashing crash on exit, presumable
400            // due to osg::notify or std::cout destructing earlier than this destructor.
401            // osg::notify(osg::DEBUG_INFO)<<"Waiting for DatabasePager to cancel"<<std::endl;
402            OpenThreads::Thread::YieldCurrentThread();
403        }
404       
405        // _startThreadCalled = false;
406    }
407    //std::cout<<"DatabasePager::~DatabasePager() stopped running"<<std::endl;
408    return result;
409
410}
411
412void DatabasePager::DatabaseThread::run()
413{
414    osg::notify(osg::INFO)<<_name<<": DatabasePager::DatabaseThread::run"<<std::endl;
415   
416#if 1
417    // need to set the texture object manager to be able to reuse textures
418    osg::Texture::setMinimumNumberOfTextureObjectsToRetainInCache(100);
419   
420    // need to set the display list manager to be able to reuse display lists
421    osg::Drawable::setMinimumNumberOfDisplayListsToRetainInCache(100);
422#else
423    // need to set the texture object manager to be able to reuse textures
424    osg::Texture::setMinimumNumberOfTextureObjectsToRetainInCache(0);
425   
426    // need to set the display list manager to be able to reuse display lists
427    osg::Drawable::setMinimumNumberOfDisplayListsToRetainInCache(0);
428#endif
429
430    bool firstTime = true;
431   
[8434]432    osg::ref_ptr<DatabasePager::ReadQueue> read_queue;
433    osg::ref_ptr<DatabasePager::ReadQueue> out_queue;
[8325]434   
435    switch(_mode)
436    {
437        case(HANDLE_ALL_REQUESTS):
[8328]438            read_queue = _pager->_fileRequestQueue;
[8325]439            break;
440        case(HANDLE_NON_HTTP):
[8328]441            read_queue = _pager->_fileRequestQueue;
442            out_queue = _pager->_httpRequestQueue;
[8325]443            break;
444        case(HANDLE_ONLY_HTTP):
[8328]445            read_queue = _pager->_httpRequestQueue;
[8325]446            break;
447    }
448
[10174]449
[8325]450    do
451    {
[9257]452        _active = false;
[8325]453
454        read_queue->block();
455
[9257]456        _active = true;
457
[8325]458        osg::notify(osg::INFO)<<_name<<": _pager->_requestList.size()= "<<read_queue->_requestList.size()<<" to delete = "<<read_queue->_childrenToDeleteList.size()<<std::endl;
459
460
461        //
462        // delete any children if required.
463        //
[8387]464        if (_pager->_deleteRemovedSubgraphsInDatabaseThread && !(read_queue->_childrenToDeleteList.empty()))
[8325]465        {
466            ObjectList deleteList;
467
468            {
469                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(read_queue->_childrenToDeleteListMutex);
470                deleteList.swap(read_queue->_childrenToDeleteList);
471                read_queue->updateBlock();
472            }
473        }
474
475        //
476        // load any subgraphs that are required.
477        //
478        osg::ref_ptr<DatabaseRequest> databaseRequest;
479        read_queue->takeFirst(databaseRequest);
[9038]480
481        bool readFromFileCache = false;
[10174]482
483        osg::ref_ptr<FileCache> fileCache = osgDB::Registry::instance()->getFileCache();
484        osg::ref_ptr<FileLocationCallback> fileLocationCallback = osgDB::Registry::instance()->getFileLocationCallback();
485
[8325]486        if (databaseRequest.valid())
487        {
[10174]488            if (databaseRequest->_loadOptions.valid())
489            {
490                if (databaseRequest->_loadOptions->getFileCache()) fileCache = databaseRequest->_loadOptions->getFileCache();
491                if (databaseRequest->_loadOptions->getFileLocationCallback()) fileLocationCallback = databaseRequest->_loadOptions->getFileLocationCallback();
492            }
493
494            // disable the FileCache if the fileLocationCallback tells us that it isn't required for this request.
495            if (fileLocationCallback.valid() && !fileLocationCallback->useFileCache()) fileCache = 0;
496
497
[8325]498            // check if databaseRequest is still relevant
499            if ((_pager->_frameNumber-databaseRequest->_frameNumberLastRequest)<=1)
500            {
501
502                // now check to see if this request is appropriate for this thread
503                switch(_mode)
504                {
505                    case(HANDLE_ALL_REQUESTS):
[10174]506                    {
[8325]507                        // do nothing as this thread can handle the load
[10174]508                        if (fileCache.valid() && fileCache->isFileAppropriateForFileCache(databaseRequest->_fileName))
[8325]509                        {
[10174]510                            if (fileCache->existsInCache(databaseRequest->_fileName))
[8325]511                            {
[9038]512                                readFromFileCache = true;
[8325]513                            }
514                        }
515                        break;
[10174]516                    }
[8325]517                    case(HANDLE_NON_HTTP):
[10174]518                    {
[8325]519                        // check the cache first
[10174]520                        bool isHighLatencyFileRequest = false;
521
522                        if (fileLocationCallback.valid())
[8325]523                        {
[10174]524                            isHighLatencyFileRequest = fileLocationCallback->fileLocation(databaseRequest->_fileName, databaseRequest->_loadOptions.get()) == FileLocationCallback::REMOTE_FILE;
525                        }
526                        else  if (fileCache.valid() && fileCache->isFileAppropriateForFileCache(databaseRequest->_fileName))
527                        {
528                            isHighLatencyFileRequest = true;
529                        }
530
531                        if (isHighLatencyFileRequest)
532                        {
[9038]533                            if (fileCache.valid() && fileCache->existsInCache(databaseRequest->_fileName))
[8325]534                            {
[9038]535                                readFromFileCache = true;
[8325]536                            }
537                            else
538                            {
[9038]539                                osg::notify(osg::INFO)<<_name<<": Passing http requests over "<<databaseRequest->_fileName<<std::endl;
[8325]540                                out_queue->add(databaseRequest.get());
541                                databaseRequest = 0;
542                            }
543                        }
544                        break;
[10174]545                    }
[8325]546                    case(HANDLE_ONLY_HTTP):
[10174]547                    {
548                        // accept all requests, as we'll assume only high latency requests will have got here.
[8325]549                        break;
[10174]550                    }
[8325]551                }
552            }
553            else
554            {               
555                databaseRequest = 0;
556            }
557        }
558       
559       
560        if (databaseRequest.valid())
561        {
562                       
563            // load the data, note safe to write to the databaseRequest since once
564            // it is created this thread is the only one to write to the _loadedModel pointer.
565            //osg::notify(osg::NOTICE)<<"In DatabasePager thread readNodeFile("<<databaseRequest->_fileName<<")"<<std::endl;
566            //osg::Timer_t before = osg::Timer::instance()->tick();
567
568
[9038]569            // assume that readNode is thread safe...
570            ReaderWriter::ReadResult rr = readFromFileCache ?
571                        fileCache->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false) :
572                        Registry::instance()->readNode(databaseRequest->_fileName, databaseRequest->_loadOptions.get(), false);
[8325]573
[9038]574            if (rr.validNode()) databaseRequest->_loadedModel = rr.getNode();
[9055]575            if (rr.error()) osg::notify(osg::WARN)<<"Error in reading file "<<databaseRequest->_fileName<<" : "<<rr.message() << std::endl;
[9038]576
[8325]577            if (databaseRequest->_loadedModel.valid() &&
[9038]578                fileCache.valid() &&
[10174]579                fileCache->isFileAppropriateForFileCache(databaseRequest->_fileName) &&
[9038]580                !readFromFileCache)
[8325]581            {
[9038]582                fileCache->writeNode(*(databaseRequest->_loadedModel), databaseRequest->_fileName, databaseRequest->_loadOptions.get());
[8325]583            }
584
585            if ((_pager->_frameNumber-databaseRequest->_frameNumberLastRequest)>1)
586            {
587                osg::notify(osg::INFO)<<_name<<": Warning DatabaseRquest no longer required."<<std::endl;
588                databaseRequest->_loadedModel = 0;
589            }
[8387]590           
591            osg::ref_ptr<osg::Group> groupForAddingLoadedSubgraph = databaseRequest->_groupForAddingLoadedSubgraph.get();
[8325]592
[8387]593            if (!groupForAddingLoadedSubgraph)
594            {
595                osg::notify(osg::INFO)<<_name<<": Warning parent of loaded subgraph, deleted."<<std::endl;
596                databaseRequest->_loadedModel = 0;
597            }
598
[8325]599            //osg::notify(osg::NOTICE)<<"     node read in "<<osg::Timer::instance()->delta_m(before,osg::Timer::instance()->tick())<<" ms"<<std::endl;
600
601            bool loadedObjectsNeedToBeCompiled = false;
602
[8543]603            if (databaseRequest->_loadedModel.valid())
[9055]604            {           
605                databaseRequest->_loadedModel->getBound();
606
[8543]607                osg::NodePath nodePath;
608                osg::NodePathList nodePathList = groupForAddingLoadedSubgraph->getParentalNodePaths();
609                if (!nodePathList.empty()) nodePath = nodePathList.front();
610                nodePath.push_back(groupForAddingLoadedSubgraph.get());
[8325]611
[9055]612#if 1
613                // force a compute of the loaded model's bounding volume, so that when the subgraph
614                // merged with the main scene graph and large computeBound() isn't incurred.
615                ActiveGraphicsContexts::iterator itr = _pager->_activeGraphicsContexts.begin();
616
[10054]617                DataToCompile* dtc = 0;
618                if (itr != _pager->_activeGraphicsContexts.end())
619                {
620                    dtc = &(databaseRequest->_dataToCompileMap[*itr]);
621                    ++itr;
622                }
[9055]623
624                // find all the compileable rendering objects
625                DatabasePager::FindCompileableGLObjectsVisitor frov(dtc,
626                                                     _pager->_changeAutoUnRef, _pager->_valueAutoUnRef,
627                                                     _pager->_changeAnisotropy, _pager->_valueAnisotropy,
628                                                     _pager->_drawablePolicy,
629                                                     _pager);
630
631                // push the soon to be parent on the nodepath of the NodeVisitor so that
632                // during traversal one can test for where it'll be in the overall scene graph               
633                for(osg::NodePath::iterator nitr = nodePath.begin();
634                    nitr != nodePath.end();
635                    ++nitr)
636                {
637                    frov.pushOntoNodePath(*nitr);
638                }
639
640                databaseRequest->_loadedModel->accept(frov);
641
[8543]642                if (_pager->_doPreCompile &&
643                    !_pager->_activeGraphicsContexts.empty())
644                {
[10054]645                    if (!dtc->first.empty() || !dtc->second.empty())
[9055]646                    {
647                        loadedObjectsNeedToBeCompiled = true;               
648
649                        // copy the objects from the compile list to the other graphics context list.
650                        for(;
651                            itr != _pager->_activeGraphicsContexts.end();
652                            ++itr)
653                        {
[10054]654                            databaseRequest->_dataToCompileMap[*itr] = *dtc;
[9055]655                        }
656                    }
657                }
658#else
659                if (_pager->_doPreCompile &&
660                    !_pager->_activeGraphicsContexts.empty())
661                {
[8543]662                    // force a compute of the loaded model's bounding volume, so that when the subgraph
663                    // merged with the main scene graph and large computeBound() isn't incurred.
664                    ActiveGraphicsContexts::iterator itr = _pager->_activeGraphicsContexts.begin();
[8325]665
[8543]666                    DataToCompile& dtc = databaseRequest->_dataToCompileMap[*itr];
667                    ++itr;               
[8325]668
[8543]669                    // find all the compileable rendering objects
[10054]670                    DatabasePager::FindCompileableGLObjectsVisitor frov(&dtc,
[8543]671                                                         _pager->_changeAutoUnRef, _pager->_valueAutoUnRef,
672                                                         _pager->_changeAnisotropy, _pager->_valueAnisotropy,
673                                                         _pager->_drawablePolicy,
674                                                         _pager);
675
676                    // push the soon to be parent on the nodepath of the NodeVisitor so that
677                    // during traversal one can test for where it'll be in the overall scene graph               
[8325]678                    for(osg::NodePath::iterator nitr = nodePath.begin();
679                        nitr != nodePath.end();
680                        ++nitr)
681                    {
682                        frov.pushOntoNodePath(*nitr);
683                    }
684
[8543]685                    databaseRequest->_loadedModel->accept(frov);
[8325]686
[8543]687                    if (!dtc.first.empty() || !dtc.second.empty())
688                    {
689                        loadedObjectsNeedToBeCompiled = true;               
[8325]690
[8543]691                        // copy the objects from the compile list to the other graphics context list.
692                        for(;
693                            itr != _pager->_activeGraphicsContexts.end();
694                            ++itr)
695                        {
696                            databaseRequest->_dataToCompileMap[*itr] = dtc;
697                        }
698                    }
699                }
700                else
[8325]701                {
[8543]702                    // check to see if we need to run the KdTreeBuilder
[10171]703                    if (osgDB::Registry::instance()->getBuildKdTreesHint()==osgDB::Options::BUILD_KDTREES &&
[8543]704                        osgDB::Registry::instance()->getKdTreeBuilder())
705                    {
706                        //osg::Timer_t before = osg::Timer::instance()->tick();
707                        //osg::notify(osg::NOTICE)<<"osgTerrain::GeometryTechnique::build kd tree"<<std::endl;
708                        osg::ref_ptr<osg::KdTreeBuilder> builder = osgDB::Registry::instance()->getKdTreeBuilder()->clone();
[8325]709
[8543]710                        for(osg::NodePath::iterator nitr = nodePath.begin();
711                            nitr != nodePath.end();
712                            ++nitr)
713                        {
714                            builder->pushOntoNodePath(*nitr);
715                        }
716
717                        databaseRequest->_loadedModel->accept(*builder);
718                        //osg::Timer_t after = osg::Timer::instance()->tick();
719                        //osg::notify(osg::NOTICE)<<"KdTree build time "<<osg::Timer::instance()->delta_m(before, after)<<std::endl;
[8325]720                    }
721                }
[9055]722#endif
[8325]723
[8543]724                // move the databaseRequest from the front of the fileRequest to the end of
725                // dataToCompile or dataToMerge lists.
[8325]726                if (loadedObjectsNeedToBeCompiled)
727                {
[8434]728                    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_pager->_dataToCompileList->_requestMutex);
729                    databaseRequest->_requestQueue = _pager->_dataToCompileList.get();
730                    _pager->_dataToCompileList->_requestList.push_back(databaseRequest);
[8325]731                }
732                else
733                {
[8434]734                    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_pager->_dataToMergeList->_requestMutex);
735                    databaseRequest->_requestQueue = _pager->_dataToMergeList.get();
736                    _pager->_dataToMergeList->_requestList.push_back(databaseRequest);
[8325]737                }
738            }       
739
740            // Prepare and prune the to-be-compiled list here in
741            // the pager thread rather than in the draw or
742            // graphics context thread(s).
743            if (loadedObjectsNeedToBeCompiled)
744            {
[8434]745                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_pager->_dataToCompileList->_requestMutex);
[8325]746
[8510]747                _pager->_dataToCompileList->sort();
748
[8325]749                // Prune all the old entries.
[8434]750                RequestQueue::RequestList::iterator tooOld
751                    = std::find_if(_pager->_dataToCompileList->_requestList.begin(),
752                                   _pager->_dataToCompileList->_requestList.end(),
[8325]753                                   refPtrAdapt(std::not1(std::bind2nd(std::mem_fun(&DatabaseRequest::isRequestCurrent),
754                                   _pager->_frameNumber))));
755
756                // This is the database thread, so just delete
[8434]757                for(RequestQueue::RequestList::iterator citr = tooOld;
758                    citr != _pager->_dataToCompileList->_requestList.end();
[8325]759                    ++citr)
760                {
761                    osg::notify(osg::INFO)<<_name<<": pruning from compile list"<<std::endl;
762                    (*citr)->_loadedModel = 0;
[8434]763                    (*citr)->_requestQueue = 0;
[8325]764                }
765
[8434]766                _pager->_dataToCompileList->_requestList.erase(tooOld, _pager->_dataToCompileList->_requestList.end());
[8325]767
[8434]768                loadedObjectsNeedToBeCompiled = !_pager->_dataToCompileList->_requestList.empty();
[8325]769            }
770
771            if (loadedObjectsNeedToBeCompiled && !_pager->_activeGraphicsContexts.empty())
772            {
773                for(ActiveGraphicsContexts::iterator itr = _pager->_activeGraphicsContexts.begin();
774                    itr != _pager->_activeGraphicsContexts.end();
775                    ++itr)
776                {
777                    osg::GraphicsContext* gc = osg::GraphicsContext::getCompileContext(*itr);
778                    if (gc)
779                    {   
780                        osg::GraphicsThread* gt = gc->getGraphicsThread();
781                        if (gt)
782                        {
783                            gt->add(new DatabasePager::CompileOperation(_pager));
784                        }
785                        else
786                        {
787                            gc->makeCurrent();
788
789                            _pager->compileAllGLObjects(*(gc->getState()));
790
791                            gc->releaseContext();
792                        }
793                    }
794                }
795
796                // osg::notify(osg::NOTICE)<<"Done compiling in paging thread"<<std::endl;                   
797            }
798        }
799        else
800        {
801            OpenThreads::Thread::YieldCurrentThread();
802        }
803       
804       
805        // go to sleep till our the next time our thread gets scheduled.
806
807        if (firstTime)
808        {
809            // do a yield to get round a peculiar thread hang when testCancel() is called
810            // in certain circumstances - of which there is no particular pattern.
811            YieldCurrentThread();
812            firstTime = false;
813        }
814
815    } while (!testCancel() && !_done);
816}
817
[9055]818
[8328]819DatabasePager::DatabasePager()
[8325]820{
[2365]821    //osg::notify(osg::INFO)<<"Constructing DatabasePager()"<<std::endl;
[2053]822   
[3423]823    _startThreadCalled = false;
824
[3419]825    _done = false;
826    _acceptNewRequests = true;
827    _databasePagerThreadPaused = false;
828   
[3439]829    _numFramesActive = 0;
[2376]830    _frameNumber = 0;
[7286]831   
[7287]832    const char* str = getenv("OSG_DATABASE_PAGER_PRIORITY");
833    if (str)
834    {
835        if (strcmp(str,"DEFAULT")==0)
836        {
837            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_DEFAULT);
838        }
839        else if (strcmp(str,"MIN")==0)
840        {
841            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_MIN);
842        }
843        else if (strcmp(str,"LOW")==0)
844        {
845            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_LOW);
846        }
847        else if (strcmp(str,"NOMINAL")==0)
848        {
849            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_NOMINAL);
850        }
851        else if (strcmp(str,"HIGH")==0)
852        {
853            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_HIGH);
854        }
855        else if (strcmp(str,"MAX")==0)
856        {
857            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_MAX);
858        }
859    }
[2376]860
[4824]861#if __APPLE__
862    // OSX really doesn't like compiling display lists, and performs poorly when they are used,
863    // so apply this hack to make up for its short comings.
864    _drawablePolicy = USE_VERTEX_ARRAYS;
865#else
866    _drawablePolicy = DO_NOT_MODIFY_DRAWABLE_SETTINGS;
867#endif   
868   
[7287]869    str = getenv("OSG_DATABASE_PAGER_GEOMETRY");
[4824]870    if (!str) str = getenv("OSG_DATABASE_PAGER_DRAWABLE");
871    if (str)
872    {
873        if (strcmp(str,"DoNotModify")==0)
874        {
875            _drawablePolicy = DO_NOT_MODIFY_DRAWABLE_SETTINGS;
876        }
877        else if (strcmp(str,"DisplayList")==0 || strcmp(str,"DL")==0)
878        {
879            _drawablePolicy = USE_DISPLAY_LISTS;
880        }
881        else if (strcmp(str,"VBO")==0)
882        {
883            _drawablePolicy = USE_VERTEX_BUFFER_OBJECTS;
884        }
885        else if (strcmp(str,"VertexArrays")==0 || strcmp(str,"VA")==0 )
886        {
887            _drawablePolicy = USE_VERTEX_ARRAYS;
888        }
889    }
890
[10407]891    _changeAutoUnRef = true;
892    _valueAutoUnRef = false;
893 
[3159]894    _changeAnisotropy = false;
895    _valueAnisotropy = 1.0f;
896
[7894]897    const char* ptr=0;
[2631]898
[8007]899    _deleteRemovedSubgraphsInDatabaseThread = true;
900    if( (ptr = getenv("OSG_DELETE_IN_DATABASE_THREAD")) != 0)
901    {
902        _deleteRemovedSubgraphsInDatabaseThread = strcmp(ptr,"yes")==0 || strcmp(ptr,"YES")==0 ||
903                        strcmp(ptr,"on")==0 || strcmp(ptr,"ON")==0;
904
905    }
906
[7894]907    _expiryDelay = 10.0;
908    if( (ptr = getenv("OSG_EXPIRY_DELAY")) != 0)
909    {
[10415]910        _expiryDelay = osg::asciiToDouble(ptr);
[10520]911        osg::notify(osg::NOTICE)<<"DatabasePager: Expiry delay = "<<_expiryDelay<<std::endl;
[7894]912    }
913
[8766]914    _expiryFrames = 1; // Last frame will not be expired
915    if( (ptr = getenv("OSG_EXPIRY_FRAMES")) != 0)
916    {
917        _expiryFrames = atoi(ptr);
[10520]918        osg::notify(osg::NOTICE)<<"DatabasePager: Expiry frames = "<<_expiryFrames<<std::endl;
[8766]919    }
920
[8994]921    if( (ptr = getenv("OSG_RELEASE_DELAY")) != 0)
922    {
923        if (strcmp(ptr,"OFF")==0 || strcmp(ptr,"Off")==0  || strcmp(ptr,"off")==0)
924        {
925            setReleaseDelay(DBL_MAX);
926        }
927        else
928        {
[10415]929            setReleaseDelay(osg::asciiToDouble(ptr));
[8994]930        }
931           
[10520]932        osg::notify(osg::NOTICE)<<"DatabasePager: Release delay = "<<_releaseDelay<<std::endl;
[8994]933    }
934    else
935    {
936        setReleaseDelay(DBL_MAX);
937    }
[9046]938   
[8994]939
940    _releaseFrames = 1; // Last frame will not be release
941    if( (ptr = getenv("OSG_RELEASE_FRAMES")) != 0)
942    {
943        _releaseFrames = atoi(ptr);
944        osg::notify(osg::NOTICE)<<"Release frames = "<<_releaseFrames<<std::endl;
945    }
946
[9046]947
[9352]948    _targetMaximumNumberOfPageLOD = 300;
[9046]949    if( (ptr = getenv("OSG_MAX_PAGEDLOD")) != 0)
950    {
[9257]951        _targetMaximumNumberOfPageLOD = atoi(ptr);
952        osg::notify(osg::NOTICE)<<"_targetMaximumNumberOfPageLOD = "<<_targetMaximumNumberOfPageLOD<<std::endl;
[9046]953    }
954
955
[9349]956    _doPreCompile = false;
[4308]957    if( (ptr = getenv("OSG_DO_PRE_COMPILE")) != 0)
958    {
959        _doPreCompile = strcmp(ptr,"yes")==0 || strcmp(ptr,"YES")==0 ||
[4476]960                        strcmp(ptr,"on")==0 || strcmp(ptr,"ON")==0;
[4308]961    }
962
[3915]963    _targetFrameRate = 100.0;
964    _minimumTimeAvailableForGLCompileAndDeletePerFrame = 0.001; // 1ms.
[4953]965    _maximumNumOfObjectsToCompilePerFrame = 4;
[4307]966    if( (ptr = getenv("OSG_MINIMUM_COMPILE_TIME_PER_FRAME")) != 0)
967    {
[10415]968        _minimumTimeAvailableForGLCompileAndDeletePerFrame = osg::asciiToDouble(ptr);
[4307]969    }
970
971    if( (ptr = getenv("OSG_MAXIMUM_OBJECTS_TO_COMPILE_PER_FRAME")) != 0)
972    {
973        _maximumNumOfObjectsToCompilePerFrame = atoi(ptr);
974    }
975
[7286]976    // initialize the stats variables
977    resetStats();
978
[2631]979    // make sure a SharedStateManager exists.
980    //osgDB::Registry::instance()->getOrCreateSharedStateManager();
981   
982    //if (osgDB::Registry::instance()->getSharedStateManager())
983        //osgDB::Registry::instance()->setUseObjectCacheHint(true);
[8325]984       
[8434]985    _fileRequestQueue = new ReadQueue(this,"fileRequestQueue");
986    _httpRequestQueue = new ReadQueue(this,"httpRequestQueue");
987   
988    _dataToCompileList = new RequestQueue;
989    _dataToMergeList = new RequestQueue;
990   
[9055]991    setUpThreads(
992        osg::DisplaySettings::instance()->getNumOfDatabaseThreadsHint(),
993        osg::DisplaySettings::instance()->getNumOfHttpDatabaseThreadsHint());
[8328]994
[2053]995}
996
[8328]997DatabasePager::DatabasePager(const DatabasePager& rhs)
[7205]998{
999    //osg::notify(osg::INFO)<<"Constructing DatabasePager(const DatabasePager& )"<<std::endl;
1000   
1001    _startThreadCalled = false;
1002
1003    _done = false;
1004    _acceptNewRequests = true;
1005    _databasePagerThreadPaused = false;
1006   
1007    _numFramesActive = 0;
1008    _frameNumber = 0;
1009
1010    _drawablePolicy = rhs._drawablePolicy;
1011
1012    _changeAutoUnRef = rhs._changeAutoUnRef;
1013    _valueAutoUnRef = rhs._valueAutoUnRef;
1014    _changeAnisotropy = rhs._changeAnisotropy;
1015    _valueAnisotropy = rhs._valueAnisotropy;
1016
1017
1018    _deleteRemovedSubgraphsInDatabaseThread = rhs._deleteRemovedSubgraphsInDatabaseThread;
1019   
1020    _expiryDelay = rhs._expiryDelay;
[8766]1021    _expiryFrames = rhs._expiryFrames;
[8994]1022
1023    _releaseDelay = rhs._releaseDelay;
1024    _releaseFrames = rhs._releaseFrames;
1025
[9257]1026    _targetMaximumNumberOfPageLOD = rhs._targetMaximumNumberOfPageLOD;
[9046]1027
[7205]1028    _doPreCompile = rhs._doPreCompile;
1029    _targetFrameRate = rhs._targetFrameRate;
1030    _minimumTimeAvailableForGLCompileAndDeletePerFrame = rhs._minimumTimeAvailableForGLCompileAndDeletePerFrame;
1031    _maximumNumOfObjectsToCompilePerFrame = rhs._maximumNumOfObjectsToCompilePerFrame;
[7286]1032
[8434]1033    _fileRequestQueue = new ReadQueue(this,"fileRequestQueue");
1034    _httpRequestQueue = new ReadQueue(this,"httpRequestQueue");
1035   
1036    _dataToCompileList = new RequestQueue;
1037    _dataToMergeList = new RequestQueue;
[8328]1038
[8325]1039    for(DatabaseThreadList::const_iterator dt_itr = rhs._databaseThreads.begin();
1040        dt_itr != rhs._databaseThreads.end();
1041        ++dt_itr)
1042    {
1043        _databaseThreads.push_back(new DatabaseThread(**dt_itr,this));
1044    }
1045
[7286]1046    // initialize the stats variables
1047    resetStats();
[7205]1048}
1049
1050
[2162]1051DatabasePager::~DatabasePager()
1052{
[3419]1053    cancel();
1054}
[2631]1055
[7205]1056osg::ref_ptr<DatabasePager>& DatabasePager::prototype()
1057{
1058    static osg::ref_ptr<DatabasePager> s_DatabasePager = new DatabasePager;
1059    return s_DatabasePager;
1060}
1061
1062DatabasePager* DatabasePager::create()
1063{
1064    return DatabasePager::prototype().valid() ?
1065           DatabasePager::prototype()->clone() :
1066           new DatabasePager;
1067}
1068
[9055]1069void DatabasePager::setUpThreads(unsigned int totalNumThreads, unsigned int numHttpThreads)
1070{
1071    _databaseThreads.clear();
1072   
1073    unsigned int numGeneralThreads = numHttpThreads < totalNumThreads ?
1074        totalNumThreads - numHttpThreads :
1075        1;
1076   
1077    if (numHttpThreads==0)
1078    {
1079        for(unsigned int i=0; i<numGeneralThreads; ++i)
1080        {
1081            addDatabaseThread(DatabaseThread::HANDLE_ALL_REQUESTS,"HANDLE_ALL_REQUESTS");
1082        }
1083    }
1084    else
1085    {
1086        for(unsigned int i=0; i<numGeneralThreads; ++i)
1087        {
1088            addDatabaseThread(DatabaseThread::HANDLE_NON_HTTP, "HANDLE_NON_HTTP");
1089        }
1090
1091        for(unsigned int i=0; i<numHttpThreads; ++i)
1092        {
1093            addDatabaseThread(DatabaseThread::HANDLE_ONLY_HTTP, "HANDLE_ONLY_HTTP");
1094        }
1095    }   
1096}
1097
1098unsigned int DatabasePager::addDatabaseThread(DatabaseThread::Mode mode, const std::string& name)
1099{
1100    osg::notify(osg::INFO)<<"DatabasePager::addDatabaseThread() "<<name<<std::endl;
1101
1102    unsigned int pos = _databaseThreads.size();
1103   
1104    DatabaseThread* thread = new DatabaseThread(this, mode,name);
1105    _databaseThreads.push_back(thread);
1106   
1107    if (_startThreadCalled)
1108    {
1109        osg::notify(osg::DEBUG_INFO)<<"DatabasePager::startThread()"<<std::endl;
1110        thread->startThread();
1111    }
1112   
1113    return pos;
1114}
1115
[8994]1116void DatabasePager::setReleaseDelay(double releaseDelay)
1117{
1118    _releaseDelay = releaseDelay;
1119
1120    if (_releaseDelay==DBL_MAX)
1121    {
1122        _changeAutoUnRef = true;
1123        _valueAutoUnRef = true;
1124    }
1125    else
1126    {
1127        // when GLObject release is used make sure Images aren't unref'd as they may be needed later.
1128        _changeAutoUnRef = true;
1129        _valueAutoUnRef = false;
1130    }
1131}
1132
[8325]1133int DatabasePager::setSchedulePriority(OpenThreads::Thread::ThreadPriority priority)
1134{
[8327]1135    int result = 0;
[8325]1136    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1137        dt_itr != _databaseThreads.end();
1138        ++dt_itr)
1139    {
[8327]1140        result = (*dt_itr)->setSchedulePriority(priority);
[8325]1141    }
[8327]1142    return result;
[8325]1143}
[7205]1144
[8325]1145bool DatabasePager::isRunning() const
1146{
1147    for(DatabaseThreadList::const_iterator dt_itr = _databaseThreads.begin();
1148        dt_itr != _databaseThreads.end();
1149        ++dt_itr)
1150    {
1151        if ((*dt_itr)->isRunning()) return true;
1152    }
1153   
1154    return false;
1155}
1156
[3419]1157int DatabasePager::cancel()
1158{
1159    int result = 0;
[8325]1160
1161    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1162        dt_itr != _databaseThreads.end();
1163        ++dt_itr)
[2162]1164    {
[8325]1165        (*dt_itr)->setDone(true);
1166    }
[2631]1167
[8325]1168    // release the frameBlock and _databasePagerThreadBlock in case its holding up thread cancellation.
[8328]1169    _fileRequestQueue->release();
1170    _httpRequestQueue->release();
[2162]1171
[8325]1172    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1173        dt_itr != _databaseThreads.end();
1174        ++dt_itr)
1175    {
1176        (*dt_itr)->cancel();
1177    }
[2812]1178
[8325]1179    _done = true;
1180    _startThreadCalled = false;
1181
[2618]1182    //std::cout<<"DatabasePager::~DatabasePager() stopped running"<<std::endl;
[3419]1183    return result;
[2162]1184}
1185
[3419]1186void DatabasePager::clear()
1187{
[8328]1188    _fileRequestQueue->clear();
1189    _httpRequestQueue->clear();
[8325]1190       
[3419]1191    {
[8434]1192        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToCompileList->_requestMutex);
1193        for(RequestQueue::RequestList::iterator citr = _dataToCompileList->_requestList.begin();
1194            citr != _dataToCompileList->_requestList.end();
1195            ++citr)
1196        {
1197            (*citr)->_loadedModel = 0;
1198            (*citr)->_requestQueue = 0;
1199        }
1200        _dataToCompileList->_requestList.clear();
[3419]1201    }
1202
1203    {
[8434]1204        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToMergeList->_requestMutex);
1205        for(RequestQueue::RequestList::iterator citr = _dataToMergeList->_requestList.begin();
1206            citr != _dataToMergeList->_requestList.end();
1207            ++citr)
1208        {
1209            (*citr)->_loadedModel = 0;
1210            (*citr)->_requestQueue = 0;
1211        }
1212        _dataToMergeList->_requestList.clear();
[3419]1213    }
1214
[8007]1215    // note, no need to use a mutex as the list is only accessed from the update thread.
[9046]1216    _activePagedLODList.clear();
1217    _inactivePagedLODList.clear();
[8007]1218
[3419]1219    // ??
1220    // _activeGraphicsContexts
1221}
1222
[7286]1223void DatabasePager::resetStats()
1224{
1225    // initialize the stats variables
1226    _minimumTimeToMergeTile = DBL_MAX;
1227    _maximumTimeToMergeTile = -DBL_MAX;
1228    _totalTimeToMergeTiles = 0.0;
1229    _numTilesMerges = 0;
1230}
1231
[9257]1232bool DatabasePager::getRequestsInProgress() const
1233{
1234    if (getFileRequestListSize()>0) return true;
1235
1236    if (getDataToCompileListSize()>0)
1237    {
1238        return true;
1239    }
1240
1241    if (getDataToMergeListSize()>0) return true;
1242
1243    for(DatabaseThreadList::const_iterator itr = _databaseThreads.begin();
[9736]1244        itr != _databaseThreads.end();
[9257]1245        ++itr)
1246    {
1247        if ((*itr)->getActive()) return true;
1248    }
1249    return false;
1250}
1251
1252
[5795]1253void DatabasePager::requestNodeFile(const std::string& fileName,osg::Group* group,
[8325]1254                                    float priority, const osg::FrameStamp* framestamp,
[10174]1255                                    osg::ref_ptr<osg::Referenced>& databaseRequestRef,
1256                                    const osg::Referenced* options)
[2053]1257{
[10174]1258    osgDB::Options* loadOptions = dynamic_cast<osgDB::Options*>(const_cast<osg::Referenced*>(options));
1259    if (!loadOptions)
1260    {
1261       loadOptions = Registry::instance()->getOptions();
[5795]1262
[10176]1263        // osg::notify(osg::NOTICE)<<"Using options from Registry "<<std::endl;
[10174]1264    }
1265    else
1266    {
[10176]1267        // osg::notify(osg::NOTICE)<<"options from requestNodeFile "<<std::endl;
[10174]1268    }
1269
1270
[3419]1271    if (!_acceptNewRequests) return;
[8325]1272   
1273
[2375]1274    double timestamp = framestamp?framestamp->getReferenceTime():0.0;
[2376]1275    int frameNumber = framestamp?framestamp->getFrameNumber():_frameNumber;
[10421]1276
1277// #define WITH_REQUESTNODEFILE_TIMING
[10407]1278#ifdef WITH_REQUESTNODEFILE_TIMING
[10421]1279    osg::Timer_t start_tick = osg::Timer::instance()->tick();
[8325]1280    static int previousFrame = -1;
1281    static double totalTime = 0.0;
1282   
1283    if (previousFrame!=frameNumber)
1284    {
[10407]1285        osg::notify(osg::NOTICE)<<"requestNodeFiles for "<<previousFrame<<" time = "<<totalTime<<std::endl;
[8325]1286
1287        previousFrame = frameNumber;
1288        totalTime = 0.0;
1289    }
[10407]1290#endif
[8325]1291   
[2053]1292    // search to see if filename already exist in the file loaded list.
1293    bool foundEntry = false;
1294
[8325]1295    if (databaseRequestRef.valid())
[3419]1296    {
[8325]1297        DatabaseRequest* databaseRequest = dynamic_cast<DatabaseRequest*>(databaseRequestRef.get());
1298        if (databaseRequest)
[2053]1299        {
[10407]1300            osg::notify(osg::INFO)<<"DatabasePager::requestNodeFile("<<fileName<<") updating already assigned."<<std::endl;
[8325]1301
[8434]1302            RequestQueue* requestQueue = databaseRequest->_requestQueue;
1303            if (requestQueue)
1304            {
1305                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(requestQueue->_requestMutex);
1306
1307                databaseRequest->_frameNumberLastRequest = frameNumber;
1308                databaseRequest->_timestampLastRequest = timestamp;
1309                databaseRequest->_priorityLastRequest = priority;
1310                ++(databaseRequest->_numOfRequests);
1311            }
1312            else
1313            {
1314                databaseRequest->_frameNumberLastRequest = frameNumber;
1315                databaseRequest->_timestampLastRequest = timestamp;
1316                databaseRequest->_priorityLastRequest = priority;
1317                ++(databaseRequest->_numOfRequests);
1318            }
[8325]1319           
1320            foundEntry = true;
1321
1322            if (databaseRequestRef->referenceCount()==1)
[2053]1323            {
[10407]1324                osg::notify(osg::INFO)<<"DatabasePager::requestNodeFile("<<fileName<<") orphaned, resubmitting."<<std::endl;
[2053]1325
[8325]1326                databaseRequest->_frameNumberFirstRequest = frameNumber;
1327                databaseRequest->_timestampFirstRequest = timestamp;
1328                databaseRequest->_priorityFirstRequest = priority;
1329                databaseRequest->_frameNumberLastRequest = frameNumber;
1330                databaseRequest->_timestampLastRequest = timestamp;
1331                databaseRequest->_priorityLastRequest = priority;
1332                databaseRequest->_groupForAddingLoadedSubgraph = group;
1333                databaseRequest->_loadOptions = loadOptions;
[8434]1334                databaseRequest->_requestQueue = _fileRequestQueue.get();
[2053]1335
[8328]1336                _fileRequestQueue->add(databaseRequest);
[3419]1337            }
[8325]1338           
1339        }
[2053]1340    }
[8325]1341
[2053]1342    if (!foundEntry)
1343    {
[10407]1344        osg::notify(osg::INFO)<<"In DatabasePager::requestNodeFile("<<fileName<<")"<<std::endl;
[8325]1345       
[8328]1346        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_requestMutex);
[8325]1347       
1348        if (!databaseRequestRef.valid() || databaseRequestRef->referenceCount()==1)
[3419]1349        {
[8325]1350            osg::ref_ptr<DatabaseRequest> databaseRequest = new DatabaseRequest;
[2053]1351
[8325]1352            databaseRequestRef = databaseRequest.get();
[2376]1353
[3419]1354            databaseRequest->_fileName = fileName;
1355            databaseRequest->_frameNumberFirstRequest = frameNumber;
1356            databaseRequest->_timestampFirstRequest = timestamp;
1357            databaseRequest->_priorityFirstRequest = priority;
1358            databaseRequest->_frameNumberLastRequest = frameNumber;
1359            databaseRequest->_timestampLastRequest = timestamp;
1360            databaseRequest->_priorityLastRequest = priority;
1361            databaseRequest->_groupForAddingLoadedSubgraph = group;
[8325]1362            databaseRequest->_loadOptions = loadOptions;
[8434]1363            databaseRequest->_requestQueue = _fileRequestQueue.get();
[2053]1364
[8328]1365            _fileRequestQueue->_requestList.push_back(databaseRequest.get());
[8322]1366
[8328]1367            _fileRequestQueue->updateBlock();
[3419]1368        }
[8325]1369       
[2053]1370    }
1371   
[8325]1372    if (!_startThreadCalled)
[2053]1373    {
[3423]1374        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_run_mutex);
[2366]1375       
[3423]1376        if (!_startThreadCalled)
[2366]1377        {
[3423]1378            _startThreadCalled = true;
[3419]1379            _done = false;
[2366]1380            osg::notify(osg::DEBUG_INFO)<<"DatabasePager::startThread()"<<std::endl;
[8325]1381           
[9055]1382            if (_databaseThreads.empty())
1383            {
1384                setUpThreads(
1385                    osg::DisplaySettings::instance()->getNumOfDatabaseThreadsHint(),
1386                    osg::DisplaySettings::instance()->getNumOfHttpDatabaseThreadsHint());
1387            }
1388
[8325]1389            for(DatabaseThreadList::const_iterator dt_itr = _databaseThreads.begin();
1390                dt_itr != _databaseThreads.end();
1391                ++dt_itr)
1392            {
1393                (*dt_itr)->startThread();
1394            }
[2366]1395        }
[2053]1396    }
[8325]1397
[10407]1398#ifdef WITH_REQUESTNODEFILE_TIMING
[8325]1399    totalTime += osg::Timer::instance()->delta_m(start_tick, osg::Timer::instance()->tick());
[10407]1400#endif
[2053]1401}
1402
[2376]1403void DatabasePager::signalBeginFrame(const osg::FrameStamp* framestamp)
1404{
1405    if (framestamp)
1406    {
[2400]1407        //osg::notify(osg::INFO) << "signalBeginFrame "<<framestamp->getFrameNumber()<<">>>>>>>>>>>>>>>>"<<std::endl;
[2376]1408        _frameNumber = framestamp->getFrameNumber();
[3425]1409       
[2400]1410    } //else osg::notify(osg::INFO) << "signalBeginFrame >>>>>>>>>>>>>>>>"<<std::endl;
[2376]1411}
1412
1413void DatabasePager::signalEndFrame()
1414{
[2400]1415    //osg::notify(osg::INFO) << "signalEndFrame <<<<<<<<<<<<<<<<<<<< "<<std::endl;
[2376]1416}
1417
[8325]1418void DatabasePager::setDatabasePagerThreadPause(bool pause)
[2053]1419{
[8325]1420    if (_databasePagerThreadPaused == pause) return;
[2053]1421   
[3419]1422    _databasePagerThreadPaused = pause;
[8328]1423    _fileRequestQueue->updateBlock();
1424    _httpRequestQueue->updateBlock();
[3419]1425}
1426
[2092]1427
[3427]1428bool DatabasePager::requiresUpdateSceneGraph() const
1429{
1430    {
[8434]1431        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToMergeList->_requestMutex);
1432        if (!_dataToMergeList->_requestList.empty()) return true;
[3427]1433    }
1434   
1435    return false;
1436   
1437}
1438 
1439
[8766]1440void DatabasePager::addLoadedDataToSceneGraph(const osg::FrameStamp &frameStamp)
[2053]1441{
[8766]1442    double timeStamp = frameStamp.getReferenceTime();
1443    int frameNumber = frameStamp.getFrameNumber();
[3422]1444
[9055]1445    osg::Timer_t before = osg::Timer::instance()->tick();
[3422]1446
[8434]1447    RequestQueue::RequestList localFileLoadedList;
[2053]1448
[9055]1449    // get the data for the _dataToCompileList, leaving it empty via a std::vector<>.swap.
[3419]1450    {
[8434]1451        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToMergeList->_requestMutex);
1452        localFileLoadedList.swap(_dataToMergeList->_requestList);
[3419]1453    }
1454       
[9055]1455    osg::Timer_t mid = osg::Timer::instance()->tick();
1456
[2053]1457    // add the loaded data into the scene graph.
[8434]1458    for(RequestQueue::RequestList::iterator itr=localFileLoadedList.begin();
[2053]1459        itr!=localFileLoadedList.end();
1460        ++itr)
1461    {
1462        DatabaseRequest* databaseRequest = itr->get();
[2631]1463
[3416]1464        // osg::notify(osg::NOTICE)<<"Merging "<<_frameNumber-(*itr)->_frameNumberLastRequest<<std::endl;
[2631]1465       
1466        if (osgDB::Registry::instance()->getSharedStateManager())
1467            osgDB::Registry::instance()->getSharedStateManager()->share(databaseRequest->_loadedModel.get());
1468
1469       
[9046]1470        registerPagedLODs(databaseRequest->_loadedModel.get(), frameStamp.getFrameNumber());
[2053]1471       
[8387]1472        osg::ref_ptr<osg::Group> group = databaseRequest->_groupForAddingLoadedSubgraph.get();
1473        if (group.valid())
[2053]1474        {
[8387]1475            osg::PagedLOD* plod = dynamic_cast<osg::PagedLOD*>(group.get());
1476            if (plod)
[8325]1477            {
[8766]1478                plod->setTimeStamp(plod->getNumChildren(), timeStamp);
1479                plod->setFrameNumber(plod->getNumChildren(), frameNumber);
[8387]1480                plod->getDatabaseRequest(plod->getNumChildren()) = 0;
1481            }
1482            else
1483            {
1484                osg::ProxyNode* proxyNode = dynamic_cast<osg::ProxyNode*>(group.get());
1485                if (proxyNode)
1486                {
1487                    proxyNode->getDatabaseRequest(proxyNode->getNumChildren()) = 0;
1488                }
1489            }
[2053]1490
[8387]1491            group->addChild(databaseRequest->_loadedModel.get());
[7286]1492
[9055]1493            // osg::notify(osg::NOTICE)<<"merged subgraph"<<databaseRequest->_fileName<<" after "<<databaseRequest->_numOfRequests<<" requests and time="<<(timeStamp-databaseRequest->_timestampFirstRequest)*1000.0<<std::endl;
[8387]1494
1495            double timeToMerge = timeStamp-databaseRequest->_timestampFirstRequest;
1496
1497            if (timeToMerge<_minimumTimeToMergeTile) _minimumTimeToMergeTile = timeToMerge;
1498            if (timeToMerge>_maximumTimeToMergeTile) _maximumTimeToMergeTile = timeToMerge;
1499
1500            _totalTimeToMergeTiles += timeToMerge;
1501            ++_numTilesMerges;
1502        }
1503               
[8325]1504        // reset the loadedModel pointer
1505        databaseRequest->_loadedModel = 0;
[7286]1506
1507        // osg::notify(osg::NOTICE)<<"curr = "<<timeToMerge<<" min "<<getMinimumTimeToMergeTile()*1000.0<<" max = "<<getMaximumTimeToMergeTile()*1000.0<<" average = "<<getAverageTimToMergeTiles()*1000.0<<std::endl;
[2053]1508    }
[3422]1509
[9055]1510    osg::Timer_t last = osg::Timer::instance()->tick();
1511
[10407]1512    if (!localFileLoadedList.empty())
1513    {
1514        osg::notify(osg::DEBUG_INFO)<<"Done DatabasePager::addLoadedDataToSceneGraph"<<
1515            osg::Timer::instance()->delta_m(before,mid)<<"ms,\t"<<
1516            osg::Timer::instance()->delta_m(mid,last)<<"ms"<<
1517            "  objects"<<localFileLoadedList.size()<<std::endl<<std::endl;
1518    }
[2053]1519}
1520
[8007]1521class DatabasePager::MarkPagedLODsVisitor : public osg::NodeVisitor
[2143]1522{
[8007]1523public:
1524    MarkPagedLODsVisitor(const std::string& marker):
1525        osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN),
[9046]1526        _marker(marker),
1527        _numPagedLODsMarked(0)
[2053]1528    {
[2677]1529    }
1530
[10054]1531    META_NodeVisitor("osgDB","MarkPagedLODsVisitor")
[9377]1532
[8007]1533    virtual void apply(osg::PagedLOD& plod)
[3422]1534    {
[9046]1535        if (plod.getName()!=_marker)
1536        {
1537            ++_numPagedLODsMarked;
1538            plod.setName(_marker);
[8007]1539   
[9046]1540            traverse(plod);
1541        }
[3422]1542    }
[8007]1543   
1544    std::string _marker;
[9062]1545    int _numPagedLODsMarked;
[8007]1546};
[2677]1547
[9046]1548void DatabasePager::removeExpiredSubgraphs(const osg::FrameStamp& frameStamp)
[8007]1549{
[9257]1550    if (_targetMaximumNumberOfPageLOD>0)
[9046]1551    {
1552        capped_removeExpiredSubgraphs(frameStamp);
1553    }
1554    else
1555    {
1556        expiry_removeExpiredSubgraphs(frameStamp);
1557    }
1558}
1559
1560void DatabasePager::capped_removeExpiredSubgraphs(const osg::FrameStamp& frameStamp)
1561{
1562    static double s_total_iter_stage_a = 0.0;
1563    static double s_total_time_stage_a = 0.0;
1564    static double s_total_max_stage_a = 0.0;
1565   
1566    static double s_total_iter_stage_b = 0.0;
1567    static double s_total_time_stage_b = 0.0;
1568    static double s_total_max_stage_b = 0.0;
1569
1570    static double s_total_iter_stage_c = 0.0;
1571    static double s_total_time_stage_c = 0.0;
1572    static double s_total_max_stage_c = 0.0;
1573
1574    osg::Timer_t startTick = osg::Timer::instance()->tick();
1575
1576
1577    unsigned int numPagedLODs = _activePagedLODList.size() + _inactivePagedLODList.size();
1578   
1579
1580    PagedLODList::iterator itr = _activePagedLODList.begin();
1581    for(PagedLODList::iterator itr = _activePagedLODList.begin();
1582        itr != _activePagedLODList.end();
1583        )
1584    {
1585        osg::PagedLOD* plod = itr->get();
1586       
1587        int delta = frameStamp.getFrameNumber() - plod->getFrameNumberOfLastTraversal();
1588        if (delta>1)
1589        {
1590            if (_releaseDelay!=DBL_MAX)
1591            {
1592                plod->releaseGLObjects();
[10473]1593                osg::notify(osg::INFO)<<"DatabasePager::removeExpiredSubgraphs(), releasing gl objects"<<std::endl;
[9046]1594            }
1595
1596            _inactivePagedLODList.push_back(plod);
1597           
1598            itr = _activePagedLODList.erase(itr);
1599        }
1600        else
1601        {
1602            ++itr;
1603        }
1604    }
1605
1606    for(PagedLODList::iterator itr = _inactivePagedLODList.begin();
1607        itr != _inactivePagedLODList.end();
1608        )
1609    {
1610        osg::PagedLOD* plod = itr->get();
1611       
1612        int delta = frameStamp.getFrameNumber() - plod->getFrameNumberOfLastTraversal();
1613        if (delta>1)
1614        {
1615            ++itr;
1616        }
1617        else
1618        {
1619            _activePagedLODList.push_back(plod);
1620           
1621            itr = _inactivePagedLODList.erase(itr);
1622        }
1623    }
1624
[9062]1625    int inactivePLOD = _inactivePagedLODList.size();
[9046]1626
1627   
1628    osg::Timer_t end_a_Tick = osg::Timer::instance()->tick();
1629    double time_a = osg::Timer::instance()->delta_m(startTick,end_a_Tick);
1630
1631    s_total_iter_stage_a += 1.0;
1632    s_total_time_stage_a += time_a;
1633    if (s_total_max_stage_a<time_a) s_total_max_stage_a = time_a;
1634   
1635
[9257]1636    if (numPagedLODs <= _targetMaximumNumberOfPageLOD)
[9046]1637    {
1638        // nothing to do
1639        return;
1640    }
1641   
[9257]1642    int numToPrune = numPagedLODs - _targetMaximumNumberOfPageLOD;
[9046]1643    if (numToPrune > inactivePLOD)
1644    {
1645        numToPrune = inactivePLOD;
1646    }
1647
[9387]1648
[9046]1649    osg::NodeList childrenRemoved;
1650   
1651    double expiryTime = frameStamp.getReferenceTime() - 0.1;
1652    int expiryFrame = frameStamp.getFrameNumber() - 1;
1653
1654    MarkPagedLODsVisitor markerVistor("NeedToRemove");
1655
[9387]1656    for(PagedLODList::iterator itr = _inactivePagedLODList.begin();
1657        itr!=_inactivePagedLODList.end() && markerVistor._numPagedLODsMarked<numToPrune;
1658        ++itr)
1659    {
1660        osg::PagedLOD* plod = itr->get();
[9046]1661
[9387]1662        osg::NodeList localChildrenRemoved;
1663        plod->removeExpiredChildren(expiryTime, expiryFrame, localChildrenRemoved);
1664        if (!localChildrenRemoved.empty())
1665        {
1666            for(osg::NodeList::iterator critr = localChildrenRemoved.begin();
1667                critr!=localChildrenRemoved.end();
1668                ++critr)
1669            {
1670                (*critr)->accept(markerVistor);
1671            }
1672           
1673            std::copy(localChildrenRemoved.begin(),localChildrenRemoved.end(),std::back_inserter(childrenRemoved));
1674        }
1675    }
1676   
1677    for(PagedLODList::iterator itr = _activePagedLODList.begin();
1678        itr!=_activePagedLODList.end() && markerVistor._numPagedLODsMarked<numToPrune;
[9046]1679        ++itr)
1680    {
1681        osg::PagedLOD* plod = itr->get();
1682
1683        osg::NodeList localChildrenRemoved;
1684        plod->removeExpiredChildren(expiryTime, expiryFrame, localChildrenRemoved);
1685        if (!localChildrenRemoved.empty())
1686        {
1687            for(osg::NodeList::iterator critr = localChildrenRemoved.begin();
1688                critr!=localChildrenRemoved.end();
1689                ++critr)
1690            {
1691                (*critr)->accept(markerVistor);
1692            }
1693           
1694            std::copy(localChildrenRemoved.begin(),localChildrenRemoved.end(),std::back_inserter(childrenRemoved));
1695        }
1696    }
1697   
1698    osg::Timer_t end_b_Tick = osg::Timer::instance()->tick();
1699    double time_b = osg::Timer::instance()->delta_m(end_a_Tick,end_b_Tick);
1700
1701    s_total_iter_stage_b += 1.0;
1702    s_total_time_stage_b += time_b;
1703    if (s_total_max_stage_b<time_b) s_total_max_stage_b = time_b;
1704
1705
1706
[9387]1707    //osg::notify(osg::NOTICE)<<"numToPrune "<<numToPrune<< " markerVistor._numPagedLODsMarked="<<markerVistor._numPagedLODsMarked<< " childrenRemoved.size()="<<childrenRemoved.size()<<std::endl;
1708
[9046]1709    if (!childrenRemoved.empty())
1710    {
[9055]1711        bool updateBlock = false;
1712
[9046]1713        // pass the objects across to the database pager delete list
1714        if (_deleteRemovedSubgraphsInDatabaseThread)
1715        {
1716            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_childrenToDeleteListMutex);
1717            for (osg::NodeList::iterator critr = childrenRemoved.begin();
1718                 critr!=childrenRemoved.end();
1719                 ++critr)
1720            {
1721                _fileRequestQueue->_childrenToDeleteList.push_back(critr->get());
1722            }
1723           
1724            updateBlock = true;
1725
1726        }
1727
[9062]1728        int numRemoved = 0;
1729        int numToRemove = markerVistor._numPagedLODsMarked;
[9046]1730       
1731       
[9387]1732        //osg::notify(osg::NOTICE)<<"Children to remove "<<childrenRemoved.size()<<" numToRemove="<<numToRemove<<std::endl;
[9046]1733
1734        // osg::notify(osg::NOTICE)<<"   time 2 "<<osg::Timer::instance()->delta_m(before,osg::Timer::instance()->tick())<<" ms "<<std::endl;
1735        int numSkipped = 0;
[9387]1736        for(PagedLODList::iterator itr = _inactivePagedLODList.begin();
1737            itr!=_inactivePagedLODList.end() && numRemoved<numToRemove;
[9046]1738            )
1739        {
1740            osg::PagedLOD* plod = itr->get();
1741            if (plod && plod->getName() != markerVistor._marker)
1742            {
1743                ++itr;
1744               
1745                ++numSkipped;
1746               
[9387]1747                // osg::notify(osg::NOTICE)<<"skipping"<<std::endl;
[9046]1748            }
1749            else
1750            {
[9387]1751                // osg::notify(osg::NOTICE)<<"removing"<<std::endl;
[9046]1752
1753                ++numRemoved;
1754
[9387]1755                itr = _inactivePagedLODList.erase(itr);
[9046]1756            }
1757        }
1758
[9387]1759        osg::notify(osg::INFO)<<"Number of PagedLODs skipped="<<numSkipped<<" removed "<<numRemoved<<std::endl;
[9046]1760
1761
1762        childrenRemoved.clear();
1763
1764
1765        if (updateBlock)
1766        {
1767            _fileRequestQueue->updateBlock();
1768        }
1769    }
1770   
1771    osg::Timer_t end_c_Tick = osg::Timer::instance()->tick();
1772    double time_c = osg::Timer::instance()->delta_m(end_b_Tick,end_c_Tick);
1773
1774    s_total_iter_stage_c += 1.0;
1775    s_total_time_stage_c += time_c;
1776    if (s_total_max_stage_c<time_c) s_total_max_stage_c = time_c;
1777
1778
[9387]1779
[9046]1780    osg::notify(osg::INFO)<<"active="<<_activePagedLODList.size()<<" inactive="<<_inactivePagedLODList.size()<<" overall = "<<osg::Timer::instance()->delta_m(startTick,end_c_Tick)<<
1781                              " A="<<time_a<<" avg="<<s_total_time_stage_a/s_total_iter_stage_a<<" max = "<<s_total_max_stage_a<<
1782                              " B="<<time_b<<" avg="<<s_total_time_stage_b/s_total_iter_stage_b<<" max = "<<s_total_max_stage_b<<
1783                              " C="<<time_c<<" avg="<<s_total_time_stage_c/s_total_iter_stage_c<<" max = "<<s_total_max_stage_c<<std::endl;
1784}
1785
1786void DatabasePager::expiry_removeExpiredSubgraphs(const osg::FrameStamp& frameStamp)
1787{
[8007]1788//    osg::notify(osg::NOTICE)<<"DatabasePager::new_removeExpiredSubgraphs()"<<std::endl;
[2677]1789
[9046]1790    static double s_total_iter = 0.0;
1791    static double s_total_time = 0.0;
1792    static double s_total_max = 0.0;
[8994]1793   
[9046]1794    osg::Timer_t startTick = osg::Timer::instance()->tick();
[8994]1795
[8766]1796    double expiryTime = frameStamp.getReferenceTime() - _expiryDelay;
1797    int expiryFrame = frameStamp.getFrameNumber() - _expiryFrames;
[3416]1798
[8994]1799    double releaseTime = frameStamp.getReferenceTime() - _releaseDelay;
1800    int releaseFrame = frameStamp.getFrameNumber() - _releaseFrames;
1801
[8007]1802    osg::NodeList childrenRemoved;
[7817]1803   
[9046]1804    for(PagedLODList::iterator itr = _activePagedLODList.begin();
1805        itr!=_activePagedLODList.end();
[8007]1806        ++itr)
[7817]1807    {
[8007]1808        osg::PagedLOD* plod = itr->get();
[8994]1809       
1810       
1811        if (_releaseDelay!=DBL_MAX && plod->releaseGLObjectsOnExpiredChildren(releaseTime, releaseFrame))
1812        {
1813            osg::notify(osg::INFO)<<"DatabasePager::removeExpiredSubgraphs(), releasing gl objects"<<std::endl;
1814        }
1815       
[8766]1816        plod->removeExpiredChildren(expiryTime, expiryFrame, childrenRemoved);
[7817]1817    }
[8007]1818   
[3416]1819    if (!childrenRemoved.empty())
1820    {
[8007]1821        MarkPagedLODsVisitor markerVistor("NeedToRemove");
1822        for(osg::NodeList::iterator critr = childrenRemoved.begin();
1823            critr!=childrenRemoved.end();
1824            ++critr)
1825        {
1826            (*critr)->accept(markerVistor);
1827        }   
1828   
1829        // osg::notify(osg::NOTICE)<<"Children to remove "<<childrenRemoved.size()<<std::endl;
1830   
[3416]1831        // pass the objects across to the database pager delete list
[5391]1832        if (_deleteRemovedSubgraphsInDatabaseThread)
[3416]1833        {
[8328]1834            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_childrenToDeleteListMutex);
[3416]1835            for (osg::NodeList::iterator critr = childrenRemoved.begin();
1836                 critr!=childrenRemoved.end();
1837                 ++critr)
1838            {
[8328]1839                _fileRequestQueue->_childrenToDeleteList.push_back(critr->get());
[3416]1840            }
[8007]1841
[8328]1842            _fileRequestQueue->updateBlock();
[3416]1843        }
[2672]1844
[8007]1845        // osg::notify(osg::NOTICE)<<"   time 2 "<<osg::Timer::instance()->delta_m(before,osg::Timer::instance()->tick())<<" ms "<<std::endl;
[9046]1846        for(PagedLODList::iterator itr = _activePagedLODList.begin();
1847            itr!=_activePagedLODList.end();
[8007]1848            )
1849        {
1850            osg::PagedLOD* plod = itr->get();
1851            if (plod && plod->getName() != markerVistor._marker)
1852            {
1853                ++itr;
1854            }
1855            else
1856            {
1857                PagedLODList::iterator itr_to_erase = itr;
1858                ++itr;
1859
[9046]1860                _activePagedLODList.erase(itr_to_erase);           
[8007]1861            }
1862        }
1863
[3416]1864        childrenRemoved.clear();
[8007]1865
[9046]1866
[3416]1867    }
[2677]1868
[9046]1869    osg::Timer_t endTick = osg::Timer::instance()->tick();
1870    double time = osg::Timer::instance()->delta_m(startTick,endTick);
1871
1872    s_total_iter += 1.0;
1873    s_total_time += time;
1874    if (s_total_max<time) s_total_max = time;
1875
[9048]1876    osg::notify(osg::INFO)<<"_activePagedLODList.size()="<<_activePagedLODList.size()<<" overall = "<<time<<
[9046]1877                              " avg="<<s_total_time/s_total_iter<<" max = "<<s_total_max<<std::endl;
[2053]1878}
1879
[3652]1880class DatabasePager::FindPagedLODsVisitor : public osg::NodeVisitor
[2053]1881{
1882public:
[8007]1883
[9046]1884    FindPagedLODsVisitor(DatabasePager::PagedLODList& pagedLODList, int frameNumber):
[2053]1885        osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN),
[9046]1886        _activePagedLODList(pagedLODList),
1887        _frameNumber(frameNumber)
[2053]1888    {
1889    }
1890   
[9377]1891    META_NodeVisitor("osgDB","FindPagedLODsVisitor")
1892
[2053]1893    virtual void apply(osg::PagedLOD& plod)
1894    {
[9046]1895        plod.setFrameNumberOfLastTraversal(_frameNumber);
1896       
1897        _activePagedLODList.push_back(&plod);
[2053]1898   
1899        traverse(plod);
1900    }
1901   
[9046]1902    DatabasePager::PagedLODList& _activePagedLODList;
1903    int _frameNumber;
[9460]1904       
1905protected:
1906   
1907    FindPagedLODsVisitor& operator = (const FindPagedLODsVisitor&) { return *this; }
[2053]1908};
1909
[9046]1910void DatabasePager::registerPagedLODs(osg::Node* subgraph, int frameNumber)
[2053]1911{
[8007]1912    if (!subgraph) return;
1913   
[9046]1914    FindPagedLODsVisitor fplv(_activePagedLODList, frameNumber);
[8007]1915    subgraph->accept(fplv);
[2053]1916}
1917
[3427]1918bool DatabasePager::requiresCompileGLObjects() const
1919{
[8434]1920    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToCompileList->_requestMutex);
1921    return !_dataToCompileList->_requestList.empty();
[3427]1922}
1923
[3423]1924void DatabasePager::setCompileGLObjectsForContextID(unsigned int contextID, bool on)
[2053]1925{
1926    if (on)
1927    {
1928        _activeGraphicsContexts.insert(contextID);
1929    }
1930    else
1931    {
1932        _activeGraphicsContexts.erase(contextID);
1933    }
1934}
1935
[3423]1936bool DatabasePager::getCompileGLObjectsForContextID(unsigned int contextID)
[2053]1937{
1938    return _activeGraphicsContexts.count(contextID)!=0;
1939}
1940
1941
[7017]1942DatabasePager::CompileOperation::CompileOperation(osgDB::DatabasePager* databasePager):
[7118]1943    osg::GraphicsOperation("DatabasePager::CompileOperation",false),
[7017]1944    _databasePager(databasePager)
1945{
1946}
1947
[7118]1948void DatabasePager::CompileOperation::operator () (osg::GraphicsContext* context)
[7017]1949{
1950    // osg::notify(osg::NOTICE)<<"Background thread compiling"<<std::endl;
[7118]1951
[7017]1952    if (_databasePager.valid()) _databasePager->compileAllGLObjects(*(context->getState()));
1953   
1954}
1955
1956bool DatabasePager::requiresExternalCompileGLObjects(unsigned int contextID) const
1957{
1958    if (_activeGraphicsContexts.count(contextID)==0) return false;
1959
[7074]1960    return osg::GraphicsContext::getCompileContext(contextID)==0;
[7017]1961}
1962
1963void DatabasePager::compileAllGLObjects(osg::State& state)
1964{
1965    double availableTime = DBL_MAX;
1966    compileGLObjects(state, availableTime);
1967}
1968
[2614]1969void DatabasePager::compileGLObjects(osg::State& state, double& availableTime)
[2053]1970{
[7017]1971    // osg::notify(osg::NOTICE)<<"DatabasePager::compileGLObjects "<<_frameNumber<<std::endl;
[2053]1972
[7017]1973    bool compileAll = (availableTime==DBL_MAX);
1974
[7662]1975    SharedStateManager *sharedManager
1976        = Registry::instance()->getSharedStateManager();
[5708]1977    osg::RenderInfo renderInfo;
1978    renderInfo.setState(&state);
1979
[3427]1980    if (availableTime>0.0)
[3419]1981    {
[3425]1982
1983        const osg::Timer& timer = *osg::Timer::instance();
1984        osg::Timer_t start_tick = timer.tick();
1985        double elapsedTime = 0.0;
[3427]1986        double estimatedTextureDuration = 0.0001;
1987        double estimatedDrawableDuration = 0.0001;
[3425]1988
1989        osg::ref_ptr<DatabaseRequest> databaseRequest;
1990
[7648]1991        // get the first compilable entry.
[2375]1992        {
[8434]1993            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToCompileList->_requestMutex);
[7662]1994       
1995            // advance to the next entry to compile if one is available.
[8995]1996            databaseRequest = _dataToCompileList->_requestList.empty() ? 0 : _dataToCompileList->_requestList.front().get();
[3425]1997        };
[2053]1998
[3425]1999        unsigned int numObjectsCompiled = 0;
[3422]2000
[3425]2001        // while there are valid databaseRequest's in the to compile list and there is
2002        // sufficient time left compile each databaseRequest's stateset and drawables.
[7017]2003        while (databaseRequest.valid() && (compileAll || (elapsedTime<availableTime && numObjectsCompiled<_maximumNumOfObjectsToCompilePerFrame)) )
[2053]2004        {
[3425]2005            DataToCompileMap& dcm = databaseRequest->_dataToCompileMap;
2006            DataToCompile& dtc = dcm[state.getContextID()];
[4953]2007            if (!dtc.first.empty() && (elapsedTime+estimatedTextureDuration)<availableTime && numObjectsCompiled<_maximumNumOfObjectsToCompilePerFrame)
[2053]2008            {
[3416]2009
[3425]2010                glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_PRIORITY, 1.0);
[3416]2011
[3425]2012                // we have StateSet's to compile
2013                StateSetList& sslist = dtc.first;
2014                //osg::notify(osg::INFO)<<"Compiling statesets"<<std::endl;
2015                StateSetList::iterator itr=sslist.begin();
[7662]2016                unsigned int objTemp = numObjectsCompiled;
[3425]2017                for(;
[9546]2018                    itr!=sslist.end() && (compileAll || ((elapsedTime+estimatedTextureDuration)<availableTime && numObjectsCompiled<_maximumNumOfObjectsToCompilePerFrame));
[3425]2019                    ++itr)
2020                {
2021                    //osg::notify(osg::INFO)<<"    Compiling stateset "<<(*itr).get()<<std::endl;
[7662]2022                    if (isCompiled(itr->get(), state.getContextID())
2023                        || (sharedManager && sharedManager->isShared(itr->get())))
2024                    {
2025                        elapsedTime = timer.delta_s(start_tick,timer.tick());
2026                        continue;
2027                    }
[3425]2028                    double startCompileTime = timer.delta_s(start_tick,timer.tick());
[3416]2029
[3425]2030                    (*itr)->compileGLObjects(state);
[3416]2031
[3425]2032                    GLint p;
2033                    glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_RESIDENT, &p);
2034
2035                    elapsedTime = timer.delta_s(start_tick,timer.tick());
2036
2037
2038                    // estimate the duration of the compile based on current compile duration.
2039                    estimatedTextureDuration = (elapsedTime-startCompileTime);
2040
2041                    ++numObjectsCompiled;
2042                }
[7662]2043                if (osg::getNotifyLevel() >= osg::DEBUG_INFO
2044                    && numObjectsCompiled > objTemp)
2045                    osg::notify(osg::DEBUG_INFO)<< _frameNumber << " compiled "
2046                                                << numObjectsCompiled - objTemp
2047                                                << " StateSets" << std::endl;
2048                // remove the compiled statesets from the list.
[3425]2049                sslist.erase(sslist.begin(),itr);
[2053]2050            }
[3425]2051
[7017]2052            if (!dtc.second.empty() && (compileAll || ((elapsedTime+estimatedDrawableDuration)<availableTime && numObjectsCompiled<_maximumNumOfObjectsToCompilePerFrame)))
[2053]2053            {
[3425]2054                // we have Drawable's to compile
2055                //osg::notify(osg::INFO)<<"Compiling drawables"<<std::endl;
2056                DrawableList& dwlist = dtc.second;
2057                DrawableList::iterator itr=dwlist.begin();
[7662]2058                unsigned int objTemp = numObjectsCompiled;
[3425]2059                for(;
[7017]2060                    itr!=dwlist.end() && (compileAll || ((elapsedTime+estimatedDrawableDuration)<availableTime && numObjectsCompiled<_maximumNumOfObjectsToCompilePerFrame));
[3425]2061                    ++itr)
2062                {
2063                    //osg::notify(osg::INFO)<<"    Compiling drawable "<<(*itr).get()<<std::endl;
[7662]2064                    if (isCompiled(itr->get(), state.getContextID()))
2065                    {
2066                        elapsedTime = timer.delta_s(start_tick,timer.tick());
2067                        continue;
2068                    }
[3425]2069                    double startCompileTime = timer.delta_s(start_tick,timer.tick());
[5708]2070                    (*itr)->compileGLObjects(renderInfo);
[3425]2071                    elapsedTime = timer.delta_s(start_tick,timer.tick());
[3416]2072
[3425]2073                    // estimate the duration of the compile based on current compile duration.
2074                    estimatedDrawableDuration = (elapsedTime-startCompileTime);
[3422]2075
[3425]2076                    ++numObjectsCompiled;
2077
2078                }
[7662]2079                if (osg::getNotifyLevel() >= osg::DEBUG_INFO
2080                    && numObjectsCompiled > objTemp)
2081                    osg::notify(osg::DEBUG_INFO)<< _frameNumber << " compiled "
2082                                                << numObjectsCompiled - objTemp
2083                                                << " Drawables" << std::endl;
[3425]2084                // remove the compiled drawables from the list.
2085                dwlist.erase(dwlist.begin(),itr);
[2053]2086            }
2087
[3425]2088            //osg::notify(osg::INFO)<<"Checking if compiled"<<std::endl;
[2053]2089
[3425]2090            // now check the to compile entries for all active graphics contexts
[7662]2091            // to make sure that all have been compiled. They won't be
2092            // if we ran out of time or if another thread is still
2093            // compiling for its graphics context.
[3425]2094            bool allCompiled = true;
2095            for(DataToCompileMap::iterator itr=dcm.begin();
2096                itr!=dcm.end() && allCompiled;
2097                ++itr)
[3419]2098            {
[3425]2099                if (!(itr->second.first.empty())) allCompiled=false;
2100                if (!(itr->second.second.empty())) allCompiled=false;
[3419]2101            }
[2053]2102
[7662]2103            //if (numObjectsCompiled > 0)
2104            //osg::notify(osg::NOTICE)<< _frameNumber << "compiled " << numObjectsCompiled << " objects" << std::endl;
2105           
[3425]2106            if (allCompiled)
[3419]2107            {
[7662]2108                // we've compiled all of the current databaseRequest so we can now pop it off the
2109                // to compile list and place it on the merge list.
[7017]2110                // osg::notify(osg::NOTICE)<<"All compiled"<<std::endl;
2111
[7662]2112
[8434]2113                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToCompileList->_requestMutex);
[3425]2114
[7662]2115                // The request might have been removed from the
2116                // _dataToCompile list by another graphics thread, in
2117                // which case it's already on the merge list, or by
2118                // the pager, which means that the request has
2119                // expired. Also, the compile list might have been
2120                // shuffled by the pager, so the current request is
2121                // not guaranteed to still be at the beginning of the
2122                // list.
[8434]2123               
2124                RequestQueue::RequestList::iterator requestIter
2125                    = std::find(_dataToCompileList->_requestList.begin(), _dataToCompileList->_requestList.end(),
[7662]2126                           databaseRequest);
[8434]2127                if (requestIter != _dataToCompileList->_requestList.end())
[3425]2128                {
[7662]2129                    {
[8434]2130                        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_dataToMergeList->_requestMutex);
2131                        databaseRequest->_requestQueue = _dataToMergeList.get();
2132                        _dataToMergeList->_requestList.push_back(databaseRequest);
[7662]2133                    }
[8434]2134                    _dataToCompileList->_requestList.erase(requestIter);
[3425]2135                }
[7662]2136               
[8434]2137                if (!_dataToCompileList->_requestList.empty()) databaseRequest = _dataToCompileList->_requestList.front();
[3425]2138                else databaseRequest = 0;
2139
[3419]2140            }
[3425]2141            else 
2142            {
[7017]2143                // osg::notify(osg::NOTICE)<<"Not all compiled"<<std::endl;
[3425]2144                databaseRequest = 0;
2145            }
[3419]2146
[3425]2147            elapsedTime = timer.delta_s(start_tick,timer.tick());
[2053]2148        }
[3425]2149
[3427]2150        availableTime -= elapsedTime;
[3425]2151
[3427]2152        //osg::notify(osg::NOTICE)<<"elapsedTime="<<elapsedTime<<"\ttime remaining ="<<availableTime<<"\tnumObjectsCompiled = "<<numObjectsCompiled<<std::endl;
[3425]2153        //osg::notify(osg::NOTICE)<<"estimatedTextureDuration="<<estimatedTextureDuration;
2154        //osg::notify(osg::NOTICE)<<"\testimatedDrawableDuration="<<estimatedDrawableDuration<<std::endl;
[2053]2155    }
[3425]2156    else
2157    {
2158        availableTime = 0.0f;
2159    }
[2053]2160}
Note: See TracBrowser for help on using the browser.