root/OpenSceneGraph/trunk/src/osgDB/DatabasePager.cpp @ 13041

Revision 13041, 59.6 kB (checked in by robert, 3 years ago)

Ran script to remove trailing spaces and tabs

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
Line 
1/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
2 *
3 * This library is open source and may be redistributed and/or modified under
4 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
5 * (at your option) any later version.  The full license is in LICENSE file
6 * included with this distribution, and on the openscenegraph.org website.
7 *
8 * This library is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 * OpenSceneGraph Public License for more details.
12*/
13
14#include <osgDB/DatabasePager>
15#include <osgDB/WriteFile>
16#include <osgDB/FileNameUtils>
17#include <osgDB/FileUtils>
18#include <osgDB/Registry>
19
20#include <osg/Geode>
21#include <osg/Timer>
22#include <osg/Texture>
23#include <osg/Notify>
24#include <osg/ProxyNode>
25#include <osg/ApplicationUsage>
26
27#include <OpenThreads/ScopedLock>
28
29#include <algorithm>
30#include <functional>
31#include <set>
32#include <iterator>
33
34#include <stdlib.h>
35#include <string.h>
36
37#ifdef WIN32
38#include <windows.h>
39#else
40#include <unistd.h>
41#endif
42
43using namespace osgDB;
44using namespace OpenThreads;
45
46static osg::ApplicationUsageProxy DatabasePager_e0(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DO_PRE_COMPILE <ON/OFF>","Switch on or off the pre compile of OpenGL object database pager.");
47static osg::ApplicationUsageProxy DatabasePager_e3(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DATABASE_PAGER_DRAWABLE <mode>","Set the drawable policy for setting of loaded drawable to specified type.  mode can be one of DoNotModify, DisplayList, VBO or VertexArrays>.");
48static osg::ApplicationUsageProxy DatabasePager_e4(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_DATABASE_PAGER_PRIORITY <mode>", "Set the thread priority to DEFAULT, MIN, LOW, NOMINAL, HIGH or MAX.");
49static osg::ApplicationUsageProxy DatabasePager_e11(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_MAX_PAGEDLOD <num>","Set the target maximum number of PagedLOD to maintain.");
50static osg::ApplicationUsageProxy DatabasePager_e12(osg::ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_ASSIGN_PBO_TO_IMAGES <ON/OFF>","Set whether PixelBufferObjects should be assigned to Images to aid download to the GPU.");
51
52// Convert function objects that take pointer args into functions that a
53// reference to an osg::ref_ptr. This is quite useful for doing STL
54// operations on lists of ref_ptr. This code assumes that a function
55// with an argument const Foo* should be composed into a function of
56// argument type ref_ptr<Foo>&, not ref_ptr<const Foo>&. Some support
57// for that should be added to make this more general.
58
59namespace
60{
61template <typename U>
62struct PointerTraits
63{
64    typedef class NullType {} PointeeType;
65};
66
67template <typename U>
68struct PointerTraits<U*>
69{
70    typedef U PointeeType;
71};
72
73template <typename U>
74struct PointerTraits<const U*>
75{
76    typedef U PointeeType;
77};
78
79template <typename FuncObj>
80class RefPtrAdapter
81    : public std::unary_function<const osg::ref_ptr<typename PointerTraits<typename FuncObj::argument_type>::PointeeType>,
82                                 typename FuncObj::result_type>
83{
84public:
85    typedef typename PointerTraits<typename FuncObj::argument_type>::PointeeType PointeeType;
86    typedef osg::ref_ptr<PointeeType> RefPtrType;
87    explicit RefPtrAdapter(const FuncObj& funcObj) : _func(funcObj) {}
88    typename FuncObj::result_type operator()(const RefPtrType& refPtr) const
89    {
90        return _func(refPtr.get());
91    }
92protected:
93        FuncObj _func;
94};
95
96template <typename FuncObj>
97RefPtrAdapter<FuncObj> refPtrAdapt(const FuncObj& func)
98{
99    return RefPtrAdapter<FuncObj>(func);
100}
101}
102
103
104/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
105//
106//  CountPagedLODList
107//
108struct DatabasePager::DatabasePagerCompileCompletedCallback : public osgUtil::IncrementalCompileOperation::CompileCompletedCallback
109{
110    DatabasePagerCompileCompletedCallback(osgDB::DatabasePager* pager, osgDB::DatabasePager::DatabaseRequest* databaseRequest):
111        _pager(pager),
112        _databaseRequest(databaseRequest) {}
113
114    virtual bool compileCompleted(osgUtil::IncrementalCompileOperation::CompileSet* compileSet)
115    {
116        _pager->compileCompleted(_databaseRequest.get());
117        return true;
118    }
119
120    osgDB::DatabasePager*                               _pager;
121    osg::ref_ptr<osgDB::DatabasePager::DatabaseRequest> _databaseRequest;
122};
123
124
125void DatabasePager::compileCompleted(DatabaseRequest* databaseRequest)
126{
127    //OSG_NOTICE<<"DatabasePager::compileCompleted("<<databaseRequest<<")"<<std::endl;
128    _dataToCompileList->remove(databaseRequest);
129    _dataToMergeList->add(databaseRequest);
130}
131
132// This class is a helper for the management of SetBasedPagedLODList.
133class DatabasePager::ExpirePagedLODsVisitor : public osg::NodeVisitor
134{
135public:
136    ExpirePagedLODsVisitor():
137        osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN)
138    {
139    }
140
141    META_NodeVisitor("osgDB","ExpirePagedLODsVisitor")
142
143    virtual void apply(osg::PagedLOD& plod)
144    {
145        _childPagedLODs.insert(&plod);
146        markRequestsExpired(&plod);
147        traverse(plod);
148    }
149
150    // Remove expired children from a PagedLOD. On return
151    // removedChildren contains the nodes removed by the call to
152    // PagedLOD::removeExpiredChildren, and the _childPagedLODs member
153    // contains all the PagedLOD objects found in those children's
154    // subgraphs.
155    bool removeExpiredChildrenAndFindPagedLODs(osg::PagedLOD* plod, double expiryTime, unsigned int expiryFrame, osg::NodeList& removedChildren)
156    {
157        size_t sizeBefore = removedChildren.size();
158
159        plod->removeExpiredChildren(expiryTime, expiryFrame, removedChildren);
160
161        for(size_t i = sizeBefore; i<removedChildren.size(); ++i)
162        {
163            removedChildren[i]->accept(*this);
164        }
165        return sizeBefore!=removedChildren.size();
166    }
167
168    typedef std::set<osg::ref_ptr<osg::PagedLOD> > PagedLODset;
169    PagedLODset         _childPagedLODs;
170private:
171    void markRequestsExpired(osg::PagedLOD* plod)
172    {
173        unsigned numFiles = plod->getNumFileNames();
174        for (unsigned i = 0; i < numFiles; ++i)
175        {
176            DatabasePager::DatabaseRequest* request
177                = dynamic_cast<DatabasePager::DatabaseRequest*>(plod->getDatabaseRequest(i).get());
178            if (request)
179                request->_groupExpired = true;
180        }
181    }
182};
183
184/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
185//
186//  SetBasedPagedLODList
187//
188class SetBasedPagedLODList : public DatabasePager::PagedLODList
189{
190public:
191
192    typedef std::set< osg::observer_ptr<osg::PagedLOD> > PagedLODs;
193    PagedLODs _pagedLODs;
194
195
196    virtual PagedLODList* clone() { return new SetBasedPagedLODList(); }
197    virtual void clear() { _pagedLODs.clear(); }
198    virtual unsigned int size() { return _pagedLODs.size(); }
199
200    virtual void removeExpiredChildren(
201        int numberChildrenToRemove, double expiryTime, unsigned int expiryFrame,
202        DatabasePager::ObjectList& childrenRemoved, bool visitActive)
203    {
204        int leftToRemove = numberChildrenToRemove;
205        for(PagedLODs::iterator itr = _pagedLODs.begin();
206            itr!=_pagedLODs.end() && leftToRemove > 0;
207            )
208        {
209            osg::ref_ptr<osg::PagedLOD> plod;
210            if (itr->lock(plod))
211            {
212                bool plodActive = expiryFrame < plod->getFrameNumberOfLastTraversal();
213                if (visitActive==plodActive) // true if (visitActive && plodActive) OR (!visitActive &&!plodActive)
214                {
215                    DatabasePager::ExpirePagedLODsVisitor expirePagedLODsVisitor;
216                    osg::NodeList expiredChildren; // expired PagedLODs
217                    expirePagedLODsVisitor.removeExpiredChildrenAndFindPagedLODs(
218                        plod.get(), expiryTime, expiryFrame, expiredChildren);
219                    // Clear any expired PagedLODs out of the set
220                    for (DatabasePager::ExpirePagedLODsVisitor::PagedLODset::iterator
221                             citr = expirePagedLODsVisitor._childPagedLODs.begin(),
222                             end = expirePagedLODsVisitor._childPagedLODs.end();
223                         citr != end;
224                        ++citr)
225                    {
226                        osg::observer_ptr<osg::PagedLOD> clod(*citr);
227                        // This child PagedLOD cannot be equal to the
228                        // PagedLOD pointed to by itr because it must be
229                        // in itr's subgraph. Therefore erasing it doesn't
230                        // invalidate itr.
231                        if (_pagedLODs.erase(clod) > 0)
232                            leftToRemove--;
233                    }
234                    std::copy(expiredChildren.begin(), expiredChildren.end(), std::back_inserter(childrenRemoved));
235                }
236
237                // advance the iterator to the next element
238                ++itr;
239            }
240            else
241            {
242                _pagedLODs.erase(itr++);
243                // numberChildrenToRemove includes possibly expired
244                // observer pointers.
245                leftToRemove--;
246                OSG_INFO<<"DatabasePager::removeExpiredSubgraphs() _inactivePagedLOD has been invalidated, but ignored"<<std::endl;
247            }
248        }
249    }
250
251    virtual void removeNodes(osg::NodeList& nodesToRemove)
252    {
253        for(osg::NodeList::iterator itr = nodesToRemove.begin();
254            itr != nodesToRemove.end();
255            ++itr)
256        {
257            osg::PagedLOD* plod = dynamic_cast<osg::PagedLOD*>(itr->get());
258            osg::observer_ptr<osg::PagedLOD> obs_ptr(plod);
259            PagedLODs::iterator plod_itr = _pagedLODs.find(obs_ptr);
260            if (plod_itr != _pagedLODs.end())
261            {
262                OSG_INFO<<"Removing node from PagedLOD list"<<std::endl;
263                _pagedLODs.erase(plod_itr);
264            }
265        }
266    }
267
268    virtual void insertPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod)
269    {
270        if (_pagedLODs.count(plod)!=0)
271        {
272            OSG_NOTICE<<"Warning: SetBasedPagedLODList::insertPagedLOD("<<plod.get()<<") already inserted"<<std::endl;
273            // abort();
274            return;
275        }
276
277        // OSG_NOTICE<<"OK: SetBasedPagedLODList::insertPagedLOD("<<plod<<") inserting"<<std::endl;
278
279        _pagedLODs.insert(plod);
280    }
281
282    virtual bool containsPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod) const
283    {
284        return (_pagedLODs.count(plod)!=0);
285    }
286
287};
288
289
290/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
291//
292//  FindCompileableGLObjectsVisitor
293//
294class DatabasePager::FindCompileableGLObjectsVisitor : public osgUtil::StateToCompile
295{
296public:
297    FindCompileableGLObjectsVisitor(const DatabasePager* pager):
298            osgUtil::StateToCompile(osgUtil::GLObjectsVisitor::COMPILE_DISPLAY_LISTS|osgUtil::GLObjectsVisitor::COMPILE_STATE_ATTRIBUTES),
299            _pager(pager),
300            _changeAutoUnRef(false), _valueAutoUnRef(false),
301            _changeAnisotropy(false), _valueAnisotropy(1.0)
302    {
303        _assignPBOToImages = _pager->_assignPBOToImages;
304
305        _changeAutoUnRef = _pager->_changeAutoUnRef;
306        _valueAutoUnRef = _pager->_valueAutoUnRef;
307        _changeAnisotropy = _pager->_changeAnisotropy;
308        _valueAnisotropy = _pager->_valueAnisotropy;
309
310        switch(_pager->_drawablePolicy)
311        {
312            case DatabasePager::DO_NOT_MODIFY_DRAWABLE_SETTINGS:
313                // do nothing, leave settings as they came in from loaded database.
314                // OSG_NOTICE<<"DO_NOT_MODIFY_DRAWABLE_SETTINGS"<<std::endl;
315                break;
316            case DatabasePager::USE_DISPLAY_LISTS:
317                _mode = _mode | osgUtil::GLObjectsVisitor::SWITCH_ON_DISPLAY_LISTS;
318                _mode = _mode | osgUtil::GLObjectsVisitor::SWITCH_OFF_VERTEX_BUFFER_OBJECTS;
319                _mode = _mode & ~osgUtil::GLObjectsVisitor::SWITCH_ON_VERTEX_BUFFER_OBJECTS;
320                break;
321            case DatabasePager::USE_VERTEX_BUFFER_OBJECTS:
322                _mode = _mode | osgUtil::GLObjectsVisitor::SWITCH_ON_VERTEX_BUFFER_OBJECTS;
323                break;
324            case DatabasePager::USE_VERTEX_ARRAYS:
325                _mode = _mode & ~osgUtil::GLObjectsVisitor::SWITCH_ON_DISPLAY_LISTS;
326                _mode = _mode & ~osgUtil::GLObjectsVisitor::SWITCH_ON_VERTEX_BUFFER_OBJECTS;
327                _mode = _mode | osgUtil::GLObjectsVisitor::SWITCH_OFF_DISPLAY_LISTS;
328                _mode = _mode | osgUtil::GLObjectsVisitor::SWITCH_OFF_VERTEX_BUFFER_OBJECTS;
329                break;
330        }
331
332        if (osgDB::Registry::instance()->getBuildKdTreesHint()==osgDB::Options::BUILD_KDTREES &&
333            osgDB::Registry::instance()->getKdTreeBuilder())
334        {
335            _kdTreeBuilder = osgDB::Registry::instance()->getKdTreeBuilder()->clone();
336        }
337    }
338
339    META_NodeVisitor("osgDB","FindCompileableGLObjectsVisitor")
340
341    bool requiresCompilation() const { return !empty(); }
342
343    virtual void apply(osg::Geode& geode)
344    {
345        StateToCompile::apply(geode);
346
347        if (_kdTreeBuilder.valid())
348        {
349            geode.accept(*_kdTreeBuilder);
350        }
351    }
352
353    void apply(osg::Texture& texture)
354    {
355        StateToCompile::apply(texture);
356
357        if (_changeAutoUnRef)
358        {
359            texture.setUnRefImageDataAfterApply(_valueAutoUnRef);
360        }
361
362        if ((_changeAnisotropy && texture.getMaxAnisotropy() != _valueAnisotropy))
363        {
364            texture.setMaxAnisotropy(_valueAnisotropy);
365        }
366    }
367
368    const DatabasePager*                    _pager;
369    bool                                    _changeAutoUnRef;
370    bool                                    _valueAutoUnRef;
371    bool                                    _changeAnisotropy;
372    float                                   _valueAnisotropy;
373    osg::ref_ptr<osg::KdTreeBuilder>        _kdTreeBuilder;
374
375protected:
376
377    FindCompileableGLObjectsVisitor& operator = (const FindCompileableGLObjectsVisitor&) { return *this; }
378};
379
380
381/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
382//
383//  SortFileRequestFunctor
384//
385struct DatabasePager::SortFileRequestFunctor
386{
387    bool operator() (const osg::ref_ptr<DatabasePager::DatabaseRequest>& lhs,const osg::ref_ptr<DatabasePager::DatabaseRequest>& rhs) const
388    {
389        if (lhs->_timestampLastRequest>rhs->_timestampLastRequest) return true;
390        else if (lhs->_timestampLastRequest<rhs->_timestampLastRequest) return false;
391        else return (lhs->_priorityLastRequest>rhs->_priorityLastRequest);
392    }
393};
394
395
396
397/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
398//
399//  DatabaseRequest
400//
401void DatabasePager::DatabaseRequest::invalidate()
402{
403    OSG_INFO<<"   DatabasePager::DatabaseRequest::invalidate()."<<std::endl;
404    _valid = false;
405    _loadedModel = 0;
406    _compileSet = 0;
407}
408
409/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
410//
411//  RequestQueue
412//
413DatabasePager::RequestQueue::RequestQueue(DatabasePager* pager):
414    _pager(pager),
415    _frameNumberLastPruned(osg::UNINITIALIZED_FRAME_NUMBER)
416{
417}
418
419DatabasePager::RequestQueue::~RequestQueue()
420{
421    OSG_INFO<<"DatabasePager::RequestQueue::~RequestQueue() Destructing queue."<<std::endl;
422    for(RequestList::iterator itr = _requestList.begin();
423        itr != _requestList.end();
424        ++itr)
425    {
426        invalidate(itr->get());
427    }
428}
429
430void DatabasePager::RequestQueue::invalidate(DatabaseRequest* dr)
431{
432    // OSG_NOTICE<<"DatabasePager::RequestQueue::invalidate(DatabaseRequest* dr) dr->_compileSet="<<dr->_compileSet.get()<<std::endl;
433    // XXX _dr_mutex?
434    osg::ref_ptr<osgUtil::IncrementalCompileOperation::CompileSet> compileSet;
435    if (dr->_compileSet.lock(compileSet) && _pager->getIncrementalCompileOperation())
436    {
437        _pager->getIncrementalCompileOperation()->remove(compileSet.get());
438    }
439
440
441    dr->invalidate();
442}
443
444
445bool DatabasePager::RequestQueue::pruneOldRequestsAndCheckIfEmpty()
446{
447    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
448
449    unsigned int frameNumber = _pager->_frameNumber;
450    if (_frameNumberLastPruned != frameNumber)
451    {
452        for(RequestQueue::RequestList::iterator citr = _requestList.begin();
453            citr != _requestList.end();
454            )
455        {
456            OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
457            if ((*citr)->isRequestCurrent(frameNumber))
458            {
459                ++citr;
460            }
461            else
462            {
463                invalidate(citr->get());
464
465                OSG_INFO<<"DatabasePager::RequestQueue::pruneOldRequestsAndCheckIfEmpty(): Pruning "<<(*citr)<<std::endl;
466                citr = _requestList.erase(citr);
467            }
468        }
469
470        _frameNumberLastPruned = frameNumber;
471
472        updateBlock();
473    }
474
475    return _requestList.empty();
476}
477
478bool DatabasePager::RequestQueue::empty()
479{
480    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
481    return _requestList.empty();
482}
483
484unsigned int DatabasePager::RequestQueue::size()
485{
486    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
487    return _requestList.size();
488}
489
490void DatabasePager::RequestQueue::clear()
491{
492    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
493
494    for(RequestList::iterator citr = _requestList.begin();
495        citr != _requestList.end();
496        ++citr)
497    {
498        OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
499        invalidate(citr->get());
500    }
501
502    _requestList.clear();
503
504    _frameNumberLastPruned = _pager->_frameNumber;
505
506    updateBlock();
507}
508
509
510void DatabasePager::RequestQueue::add(DatabasePager::DatabaseRequest* databaseRequest)
511{
512    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
513
514    addNoLock(databaseRequest);
515}
516
517void DatabasePager::RequestQueue::remove(DatabasePager::DatabaseRequest* databaseRequest)
518{
519    // OSG_NOTICE<<"DatabasePager::RequestQueue::remove(DatabaseRequest* databaseRequest)"<<std::endl;
520    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
521    for(RequestList::iterator citr = _requestList.begin();
522        citr != _requestList.end();
523        ++citr)
524    {
525        if (citr->get()==databaseRequest)
526        {
527            // OSG_NOTICE<<"  done remove(DatabaseRequest* databaseRequest)"<<std::endl;
528            _requestList.erase(citr);
529            return;
530        }
531    }
532}
533
534
535void DatabasePager::RequestQueue::addNoLock(DatabasePager::DatabaseRequest* databaseRequest)
536{
537    _requestList.push_back(databaseRequest);
538    updateBlock();
539}
540
541void DatabasePager::RequestQueue::swap(RequestList& requestList)
542{
543    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
544    _requestList.swap(requestList);
545}
546
547void DatabasePager::RequestQueue::takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest)
548{
549    OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_requestMutex);
550
551    if (!_requestList.empty())
552    {
553        DatabasePager::SortFileRequestFunctor highPriority;
554
555        RequestQueue::RequestList::iterator selected_itr = _requestList.end();
556
557        int frameNumber = _pager->_frameNumber;
558
559        for(RequestQueue::RequestList::iterator citr = _requestList.begin();
560            citr != _requestList.end();
561            )
562        {
563            OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
564            if ((*citr)->isRequestCurrent(frameNumber))
565            {
566                if (selected_itr==_requestList.end() || highPriority(*citr, *selected_itr))
567                {
568                    selected_itr = citr;
569                }
570
571                ++citr;
572            }
573            else
574            {
575                invalidate(citr->get());
576
577                OSG_INFO<<"DatabasePager::RequestQueue::takeFirst(): Pruning "<<(*citr)<<std::endl;
578                citr = _requestList.erase(citr);
579            }
580
581        }
582
583        _frameNumberLastPruned = frameNumber;
584
585        if (selected_itr != _requestList.end())
586        {
587            databaseRequest = *selected_itr;
588            _requestList.erase(selected_itr);
589            OSG_INFO<<" DatabasePager::RequestQueue::takeFirst() Found DatabaseRequest size()="<<_requestList.size()<<std::endl;
590        }
591        else
592        {
593            OSG_INFO<<" DatabasePager::RequestQueue::takeFirst() No suitable DatabaseRequest found size()="<<_requestList.size()<<std::endl;
594        }
595
596        updateBlock();
597    }
598}
599
600/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
601//
602//  ReadQueue
603//
604DatabasePager::ReadQueue::ReadQueue(DatabasePager* pager, const std::string& name):
605    RequestQueue(pager),
606    _name(name)
607{
608    _block = new osg::RefBlock;
609}
610
611void DatabasePager::ReadQueue::updateBlock()
612{
613    _block->set((!_requestList.empty() || !_childrenToDeleteList.empty()) &&
614                !_pager->_databasePagerThreadPaused);
615}
616
617/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
618//
619//  DatabaseThread
620//
621DatabasePager::DatabaseThread::DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name):
622    _done(false),
623    _active(false),
624    _pager(pager),
625    _mode(mode),
626    _name(name)
627{
628}
629
630DatabasePager::DatabaseThread::DatabaseThread(const DatabaseThread& dt, DatabasePager* pager):
631    _done(false),
632    _active(false),
633    _pager(pager),
634    _mode(dt._mode),
635    _name(dt._name)
636{
637}
638
639DatabasePager::DatabaseThread::~DatabaseThread()
640{
641    cancel();
642}
643
644int DatabasePager::DatabaseThread::cancel()
645{
646    int result = 0;
647
648    if( isRunning() )
649    {
650        setDone(true);
651
652        switch(_mode)
653        {
654            case(HANDLE_ALL_REQUESTS):
655                _pager->_fileRequestQueue->release();
656                break;
657            case(HANDLE_NON_HTTP):
658                _pager->_fileRequestQueue->release();
659                break;
660            case(HANDLE_ONLY_HTTP):
661                _pager->_httpRequestQueue->release();
662                break;
663        }
664
665        // then wait for the the thread to stop running.
666        while(isRunning())
667        {
668            // commenting out debug info as it was cashing crash on exit, presumable
669            // due to OSG_NOTICE or std::cout destructing earlier than this destructor.
670            // OSG_INFO<<"Waiting for DatabasePager::DatabaseThread to cancel"<<std::endl;
671            OpenThreads::Thread::YieldCurrentThread();
672        }
673
674        // _startThreadCalled = false;
675    }
676
677    //OSG_NOTICE<<"DatabasePager::DatabaseThread stopped running"<<std::endl;
678    return result;
679
680}
681
682void DatabasePager::DatabaseThread::run()
683{
684    OSG_INFO<<_name<<": DatabasePager::DatabaseThread::run"<<std::endl;
685
686
687    bool firstTime = true;
688
689    osg::ref_ptr<DatabasePager::ReadQueue> read_queue;
690    osg::ref_ptr<DatabasePager::ReadQueue> out_queue;
691
692    switch(_mode)
693    {
694        case(HANDLE_ALL_REQUESTS):
695            read_queue = _pager->_fileRequestQueue;
696            break;
697        case(HANDLE_NON_HTTP):
698            read_queue = _pager->_fileRequestQueue;
699            out_queue = _pager->_httpRequestQueue;
700            break;
701        case(HANDLE_ONLY_HTTP):
702            read_queue = _pager->_httpRequestQueue;
703            break;
704    }
705
706
707    do
708    {
709        _active = false;
710
711        read_queue->block();
712
713        if (_done)
714        {
715            break;
716        }
717
718        _active = true;
719
720        OSG_INFO<<_name<<": _pager->size()= "<<read_queue->size()<<" to delete = "<<read_queue->_childrenToDeleteList.size()<<std::endl;
721
722
723
724        //
725        // delete any children if required.
726        //
727        if (_pager->_deleteRemovedSubgraphsInDatabaseThread/* && !(read_queue->_childrenToDeleteList.empty())*/)
728        {
729            ObjectList deleteList;
730            {
731                // Don't hold lock during destruction of deleteList
732                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(read_queue->_requestMutex);
733                if (!read_queue->_childrenToDeleteList.empty())
734                {
735                    deleteList.swap(read_queue->_childrenToDeleteList);
736                    read_queue->updateBlock();
737                }
738            }
739        }
740
741        //
742        // load any subgraphs that are required.
743        //
744        osg::ref_ptr<DatabaseRequest> databaseRequest;
745        read_queue->takeFirst(databaseRequest);
746
747        bool readFromFileCache = false;
748
749        osg::ref_ptr<FileCache> fileCache = osgDB::Registry::instance()->getFileCache();
750        osg::ref_ptr<FileLocationCallback> fileLocationCallback = osgDB::Registry::instance()->getFileLocationCallback();
751        osg::ref_ptr<Options> dr_loadOptions;
752        std::string fileName;
753        int frameNumberLastRequest = 0;
754        if (databaseRequest.valid())
755        {
756            {
757                OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
758                dr_loadOptions = databaseRequest->_loadOptions;
759                fileName = databaseRequest->_fileName;
760                frameNumberLastRequest = databaseRequest->_frameNumberLastRequest;
761            }
762            if (dr_loadOptions.valid())
763            {
764                if (dr_loadOptions->getFileCache()) fileCache = dr_loadOptions->getFileCache();
765                if (dr_loadOptions->getFileLocationCallback()) fileLocationCallback = dr_loadOptions->getFileLocationCallback();
766
767                dr_loadOptions = dr_loadOptions->cloneOptions();
768            }
769            else
770            {
771                dr_loadOptions = new osgDB::Options;
772            }
773
774            dr_loadOptions->setTerrain(databaseRequest->_terrain);
775
776            // disable the FileCache if the fileLocationCallback tells us that it isn't required for this request.
777            if (fileLocationCallback.valid() && !fileLocationCallback->useFileCache()) fileCache = 0;
778
779
780            // check if databaseRequest is still relevant
781            if ((_pager->_frameNumber-frameNumberLastRequest)<=1)
782            {
783
784                // now check to see if this request is appropriate for this thread
785                switch(_mode)
786                {
787                    case(HANDLE_ALL_REQUESTS):
788                    {
789                        // do nothing as this thread can handle the load
790                        if (fileCache.valid() && fileCache->isFileAppropriateForFileCache(fileName))
791                        {
792                            if (fileCache->existsInCache(fileName))
793                            {
794                                readFromFileCache = true;
795                            }
796                        }
797                        break;
798                    }
799                    case(HANDLE_NON_HTTP):
800                    {
801                        // check the cache first
802                        bool isHighLatencyFileRequest = false;
803
804                        if (fileLocationCallback.valid())
805                        {
806                            isHighLatencyFileRequest = fileLocationCallback->fileLocation(fileName, dr_loadOptions.get()) == FileLocationCallback::REMOTE_FILE;
807                        }
808                        else  if (fileCache.valid() && fileCache->isFileAppropriateForFileCache(fileName))
809                        {
810                            isHighLatencyFileRequest = true;
811                        }
812
813                        if (isHighLatencyFileRequest)
814                        {
815                            if (fileCache.valid() && fileCache->existsInCache(fileName))
816                            {
817                                readFromFileCache = true;
818                            }
819                            else
820                            {
821                                OSG_INFO<<_name<<": Passing http requests over "<<fileName<<std::endl;
822                                out_queue->add(databaseRequest.get());
823                                databaseRequest = 0;
824                            }
825                        }
826                        break;
827                    }
828                    case(HANDLE_ONLY_HTTP):
829                    {
830                        // accept all requests, as we'll assume only high latency requests will have got here.
831                        break;
832                    }
833                }
834            }
835            else
836            {
837                databaseRequest = 0;
838            }
839        }
840
841
842        if (databaseRequest.valid())
843        {
844
845            // load the data, note safe to write to the databaseRequest since once
846            // it is created this thread is the only one to write to the _loadedModel pointer.
847            //OSG_NOTICE<<"In DatabasePager thread readNodeFile("<<databaseRequest->_fileName<<")"<<std::endl;
848            //osg::Timer_t before = osg::Timer::instance()->tick();
849
850
851            // assume that readNode is thread safe...
852            ReaderWriter::ReadResult rr = readFromFileCache ?
853                        fileCache->readNode(fileName, dr_loadOptions.get(), false) :
854                        Registry::instance()->readNode(fileName, dr_loadOptions.get(), false);
855
856            osg::ref_ptr<osg::Node> loadedModel;
857            if (rr.validNode()) loadedModel = rr.getNode();
858            if (rr.error()) OSG_WARN<<"Error in reading file "<<fileName<<" : "<<rr.message() << std::endl;
859            if (rr.notEnoughMemory()) OSG_INFO<<"Not enought memory to load file "<<fileName << std::endl;
860
861            if (loadedModel.valid() &&
862                fileCache.valid() &&
863                fileCache->isFileAppropriateForFileCache(fileName) &&
864                !readFromFileCache)
865            {
866                fileCache->writeNode(*(loadedModel), fileName, dr_loadOptions.get());
867            }
868
869            {
870                OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
871                if ((_pager->_frameNumber-databaseRequest->_frameNumberLastRequest)>1)
872                {
873                    OSG_INFO<<_name<<": Warning DatabaseRquest no longer required."<<std::endl;
874                    loadedModel = 0;
875                }
876            }
877
878            //OSG_NOTICE<<"     node read in "<<osg::Timer::instance()->delta_m(before,osg::Timer::instance()->tick())<<" ms"<<std::endl;
879
880            if (loadedModel.valid())
881            {
882                loadedModel->getBound();
883
884                // find all the compileable rendering objects
885                DatabasePager::FindCompileableGLObjectsVisitor stateToCompile(_pager);
886                loadedModel->accept(stateToCompile);
887
888                bool loadedObjectsNeedToBeCompiled = _pager->_doPreCompile &&
889                                                     _pager->_incrementalCompileOperation.valid() &&
890                                                     _pager->_incrementalCompileOperation->requiresCompile(stateToCompile);
891
892                // move the databaseRequest from the front of the fileRequest to the end of
893                // dataToCompile or dataToMerge lists.
894                osg::ref_ptr<osgUtil::IncrementalCompileOperation::CompileSet> compileSet = 0;
895                if (loadedObjectsNeedToBeCompiled)
896                {
897                    // OSG_NOTICE<<"Using IncrementalCompileOperation"<<std::endl;
898
899                    compileSet = new osgUtil::IncrementalCompileOperation::CompileSet(loadedModel.get());
900                    compileSet->buildCompileMap(_pager->_incrementalCompileOperation->getContextSet(), stateToCompile);
901                    compileSet->_compileCompletedCallback = new DatabasePagerCompileCompletedCallback(_pager, databaseRequest.get());
902                    _pager->_incrementalCompileOperation->add(compileSet.get(), false);
903                }
904                {
905                    OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_pager->_dr_mutex);
906                    databaseRequest->_loadedModel = loadedModel;
907                    databaseRequest->_compileSet = compileSet;
908                }
909                // Dereference the databaseRequest while the queue is
910                // locked. This prevents the request from being
911                // deleted at an unpredictable time within
912                // addLoadedDataToSceneGraph.
913                if (loadedObjectsNeedToBeCompiled)
914                {
915                    OpenThreads::ScopedLock<OpenThreads::Mutex> listLock(
916                        _pager->_dataToCompileList->_requestMutex);
917                    _pager->_dataToCompileList->addNoLock(databaseRequest.get());
918                    databaseRequest = 0;
919                }
920                else
921                {
922                    OpenThreads::ScopedLock<OpenThreads::Mutex> listLock(
923                        _pager->_dataToMergeList->_requestMutex);
924                    _pager->_dataToMergeList->addNoLock(databaseRequest.get());
925                    databaseRequest = 0;
926                }
927
928            }
929
930            // _pager->_dataToCompileList->pruneOldRequestsAndCheckIfEmpty();
931        }
932        else
933        {
934            OpenThreads::Thread::YieldCurrentThread();
935        }
936
937
938        // go to sleep till our the next time our thread gets scheduled.
939
940        if (firstTime)
941        {
942            // do a yield to get round a peculiar thread hang when testCancel() is called
943            // in certain circumstances - of which there is no particular pattern.
944            YieldCurrentThread();
945            firstTime = false;
946        }
947
948    } while (!testCancel() && !_done);
949}
950
951
952DatabasePager::DatabasePager()
953{
954    //OSG_INFO<<"Constructing DatabasePager()"<<std::endl;
955
956    _startThreadCalled = false;
957
958    _done = false;
959    _acceptNewRequests = true;
960    _databasePagerThreadPaused = false;
961
962    _numFramesActive = 0;
963    _frameNumber.exchange(0);
964
965
966#if __APPLE__
967    // OSX really doesn't like compiling display lists, and performs poorly when they are used,
968    // so apply this hack to make up for its short comings.
969    _drawablePolicy = USE_VERTEX_ARRAYS;
970#else
971    _drawablePolicy = DO_NOT_MODIFY_DRAWABLE_SETTINGS;
972#endif
973
974    const char* str = getenv("OSG_DATABASE_PAGER_GEOMETRY");
975    if (!str) str = getenv("OSG_DATABASE_PAGER_DRAWABLE");
976    if (str)
977    {
978        if (strcmp(str,"DoNotModify")==0)
979        {
980            _drawablePolicy = DO_NOT_MODIFY_DRAWABLE_SETTINGS;
981        }
982        else if (strcmp(str,"DisplayList")==0 || strcmp(str,"DL")==0)
983        {
984            _drawablePolicy = USE_DISPLAY_LISTS;
985        }
986        else if (strcmp(str,"VBO")==0)
987        {
988            _drawablePolicy = USE_VERTEX_BUFFER_OBJECTS;
989        }
990        else if (strcmp(str,"VertexArrays")==0 || strcmp(str,"VA")==0 )
991        {
992            _drawablePolicy = USE_VERTEX_ARRAYS;
993        }
994    }
995
996    _assignPBOToImages = false;
997    if( (str = getenv("OSG_ASSIGN_PBO_TO_IMAGES")) != 0)
998    {
999        _assignPBOToImages = strcmp(str,"yes")==0 || strcmp(str,"YES")==0 ||
1000                             strcmp(str,"on")==0 || strcmp(str,"ON")==0;
1001
1002        OSG_NOTICE<<"OSG_ASSIGN_PBO_TO_IMAGES set to "<<_assignPBOToImages<<std::endl;
1003    }
1004
1005    _changeAutoUnRef = true;
1006    _valueAutoUnRef = false;
1007
1008    _changeAnisotropy = false;
1009    _valueAnisotropy = 1.0f;
1010
1011
1012    _deleteRemovedSubgraphsInDatabaseThread = true;
1013    if( (str = getenv("OSG_DELETE_IN_DATABASE_THREAD")) != 0)
1014    {
1015        _deleteRemovedSubgraphsInDatabaseThread = strcmp(str,"yes")==0 || strcmp(str,"YES")==0 ||
1016                                                  strcmp(str,"on")==0 || strcmp(str,"ON")==0;
1017
1018    }
1019
1020    _targetMaximumNumberOfPageLOD = 300;
1021    if( (str = getenv("OSG_MAX_PAGEDLOD")) != 0)
1022    {
1023        _targetMaximumNumberOfPageLOD = atoi(str);
1024        OSG_NOTICE<<"_targetMaximumNumberOfPageLOD = "<<_targetMaximumNumberOfPageLOD<<std::endl;
1025    }
1026
1027
1028    _doPreCompile = true;
1029    if( (str = getenv("OSG_DO_PRE_COMPILE")) != 0)
1030    {
1031        _doPreCompile = strcmp(str,"yes")==0 || strcmp(str,"YES")==0 ||
1032                        strcmp(str,"on")==0 || strcmp(str,"ON")==0;
1033    }
1034
1035    // initialize the stats variables
1036    resetStats();
1037
1038    _fileRequestQueue = new ReadQueue(this,"fileRequestQueue");
1039    _httpRequestQueue = new ReadQueue(this,"httpRequestQueue");
1040
1041    _dataToCompileList = new RequestQueue(this);
1042    _dataToMergeList = new RequestQueue(this);
1043
1044    setUpThreads(
1045        osg::DisplaySettings::instance()->getNumOfDatabaseThreadsHint(),
1046        osg::DisplaySettings::instance()->getNumOfHttpDatabaseThreadsHint());
1047
1048    str = getenv("OSG_DATABASE_PAGER_PRIORITY");
1049    if (str)
1050    {
1051        if (strcmp(str,"DEFAULT")==0)
1052        {
1053            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_DEFAULT);
1054        }
1055        else if (strcmp(str,"MIN")==0)
1056        {
1057            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_MIN);
1058        }
1059        else if (strcmp(str,"LOW")==0)
1060        {
1061            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_LOW);
1062        }
1063        else if (strcmp(str,"NOMINAL")==0)
1064        {
1065            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_NOMINAL);
1066        }
1067        else if (strcmp(str,"HIGH")==0)
1068        {
1069            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_HIGH);
1070        }
1071        else if (strcmp(str,"MAX")==0)
1072        {
1073            setSchedulePriority(OpenThreads::Thread::THREAD_PRIORITY_MAX);
1074        }
1075    }
1076
1077    _activePagedLODList = new SetBasedPagedLODList;
1078}
1079
1080DatabasePager::DatabasePager(const DatabasePager& rhs)
1081{
1082    //OSG_INFO<<"Constructing DatabasePager(const DatabasePager& )"<<std::endl;
1083
1084    _startThreadCalled = false;
1085
1086    _done = false;
1087    _acceptNewRequests = true;
1088    _databasePagerThreadPaused = false;
1089
1090    _numFramesActive = 0;
1091    _frameNumber.exchange(0);
1092
1093    _drawablePolicy = rhs._drawablePolicy;
1094
1095    _assignPBOToImages = rhs._assignPBOToImages;
1096
1097    _changeAutoUnRef = rhs._changeAutoUnRef;
1098    _valueAutoUnRef = rhs._valueAutoUnRef;
1099    _changeAnisotropy = rhs._changeAnisotropy;
1100    _valueAnisotropy = rhs._valueAnisotropy;
1101
1102    _deleteRemovedSubgraphsInDatabaseThread = rhs._deleteRemovedSubgraphsInDatabaseThread;
1103
1104    _targetMaximumNumberOfPageLOD = rhs._targetMaximumNumberOfPageLOD;
1105
1106    _doPreCompile = rhs._doPreCompile;
1107
1108    _fileRequestQueue = new ReadQueue(this,"fileRequestQueue");
1109    _httpRequestQueue = new ReadQueue(this,"httpRequestQueue");
1110
1111    _dataToCompileList = new RequestQueue(this);
1112    _dataToMergeList = new RequestQueue(this);
1113
1114    for(DatabaseThreadList::const_iterator dt_itr = rhs._databaseThreads.begin();
1115        dt_itr != rhs._databaseThreads.end();
1116        ++dt_itr)
1117    {
1118        _databaseThreads.push_back(new DatabaseThread(**dt_itr,this));
1119    }
1120
1121    _activePagedLODList = rhs._activePagedLODList->clone();
1122
1123#if 1
1124    // need to set the display list manager to be able to reuse display lists
1125    osg::Drawable::setMinimumNumberOfDisplayListsToRetainInCache(100);
1126#else
1127    // need to set the display list manager to be able to reuse display lists
1128    osg::Drawable::setMinimumNumberOfDisplayListsToRetainInCache(0);
1129#endif
1130
1131    // initialize the stats variables
1132    resetStats();
1133}
1134
1135
1136void DatabasePager::setIncrementalCompileOperation(osgUtil::IncrementalCompileOperation* ico)
1137{
1138    _incrementalCompileOperation = ico;
1139}
1140
1141DatabasePager::~DatabasePager()
1142{
1143    // cancel the threads
1144    cancel();
1145
1146    // destruct all the threads
1147    _databaseThreads.clear();
1148
1149    // destruct all the queues
1150    _fileRequestQueue = 0;
1151    _httpRequestQueue = 0;
1152    _dataToCompileList = 0;
1153    _dataToMergeList = 0;
1154
1155    // remove reference to the ICO
1156    _incrementalCompileOperation = 0;
1157
1158    //_activePagedLODList;
1159    //_inactivePagedLODList;
1160}
1161
1162osg::ref_ptr<DatabasePager>& DatabasePager::prototype()
1163{
1164    static osg::ref_ptr<DatabasePager> s_DatabasePager = new DatabasePager;
1165    return s_DatabasePager;
1166}
1167
1168DatabasePager* DatabasePager::create()
1169{
1170    return DatabasePager::prototype().valid() ?
1171           DatabasePager::prototype()->clone() :
1172           new DatabasePager;
1173}
1174
1175void DatabasePager::setUpThreads(unsigned int totalNumThreads, unsigned int numHttpThreads)
1176{
1177    _databaseThreads.clear();
1178
1179    unsigned int numGeneralThreads = numHttpThreads < totalNumThreads ?
1180        totalNumThreads - numHttpThreads :
1181        1;
1182
1183    if (numHttpThreads==0)
1184    {
1185        for(unsigned int i=0; i<numGeneralThreads; ++i)
1186        {
1187            addDatabaseThread(DatabaseThread::HANDLE_ALL_REQUESTS,"HANDLE_ALL_REQUESTS");
1188        }
1189    }
1190    else
1191    {
1192        for(unsigned int i=0; i<numGeneralThreads; ++i)
1193        {
1194            addDatabaseThread(DatabaseThread::HANDLE_NON_HTTP, "HANDLE_NON_HTTP");
1195        }
1196
1197        for(unsigned int i=0; i<numHttpThreads; ++i)
1198        {
1199            addDatabaseThread(DatabaseThread::HANDLE_ONLY_HTTP, "HANDLE_ONLY_HTTP");
1200        }
1201    }
1202}
1203
1204unsigned int DatabasePager::addDatabaseThread(DatabaseThread::Mode mode, const std::string& name)
1205{
1206    OSG_INFO<<"DatabasePager::addDatabaseThread() "<<name<<std::endl;
1207
1208    unsigned int pos = _databaseThreads.size();
1209
1210    DatabaseThread* thread = new DatabaseThread(this, mode,name);
1211    _databaseThreads.push_back(thread);
1212
1213    if (_startThreadCalled)
1214    {
1215        OSG_INFO<<"DatabasePager::startThread()"<<std::endl;
1216        thread->startThread();
1217    }
1218
1219    return pos;
1220}
1221
1222int DatabasePager::setSchedulePriority(OpenThreads::Thread::ThreadPriority priority)
1223{
1224    int result = 0;
1225    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1226        dt_itr != _databaseThreads.end();
1227        ++dt_itr)
1228    {
1229        result = (*dt_itr)->setSchedulePriority(priority);
1230    }
1231    return result;
1232}
1233
1234bool DatabasePager::isRunning() const
1235{
1236    for(DatabaseThreadList::const_iterator dt_itr = _databaseThreads.begin();
1237        dt_itr != _databaseThreads.end();
1238        ++dt_itr)
1239    {
1240        if ((*dt_itr)->isRunning()) return true;
1241    }
1242
1243    return false;
1244}
1245
1246int DatabasePager::cancel()
1247{
1248    int result = 0;
1249
1250    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1251        dt_itr != _databaseThreads.end();
1252        ++dt_itr)
1253    {
1254        (*dt_itr)->setDone(true);
1255    }
1256
1257    // release the queue blocks in case they are holding up thread cancellation.
1258    _fileRequestQueue->release();
1259    _httpRequestQueue->release();
1260
1261    for(DatabaseThreadList::iterator dt_itr = _databaseThreads.begin();
1262        dt_itr != _databaseThreads.end();
1263        ++dt_itr)
1264    {
1265        (*dt_itr)->cancel();
1266    }
1267
1268    _done = true;
1269    _startThreadCalled = false;
1270
1271    return result;
1272}
1273
1274void DatabasePager::clear()
1275{
1276    _fileRequestQueue->clear();
1277    _httpRequestQueue->clear();
1278
1279    _dataToCompileList->clear();
1280    _dataToMergeList->clear();
1281
1282    // note, no need to use a mutex as the list is only accessed from the update thread.
1283    _activePagedLODList->clear();
1284
1285    // ??
1286    // _activeGraphicsContexts
1287}
1288
1289void DatabasePager::resetStats()
1290{
1291    // initialize the stats variables
1292    _minimumTimeToMergeTile = DBL_MAX;
1293    _maximumTimeToMergeTile = -DBL_MAX;
1294    _totalTimeToMergeTiles = 0.0;
1295    _numTilesMerges = 0;
1296}
1297
1298bool DatabasePager::getRequestsInProgress() const
1299{
1300    if (getFileRequestListSize()>0) return true;
1301
1302    if (getDataToCompileListSize()>0)
1303    {
1304        return true;
1305    }
1306
1307    if (getDataToMergeListSize()>0) return true;
1308
1309    for(DatabaseThreadList::const_iterator itr = _databaseThreads.begin();
1310        itr != _databaseThreads.end();
1311        ++itr)
1312    {
1313        if ((*itr)->getActive()) return true;
1314    }
1315    return false;
1316}
1317
1318
1319void DatabasePager::requestNodeFile(const std::string& fileName, osg::NodePath& nodePath,
1320                                    float priority, const osg::FrameStamp* framestamp,
1321                                    osg::ref_ptr<osg::Referenced>& databaseRequestRef,
1322                                    const osg::Referenced* options)
1323{
1324    osgDB::Options* loadOptions = dynamic_cast<osgDB::Options*>(const_cast<osg::Referenced*>(options));
1325    if (!loadOptions)
1326    {
1327       loadOptions = Registry::instance()->getOptions();
1328
1329        // OSG_NOTICE<<"Using options from Registry "<<std::endl;
1330    }
1331    else
1332    {
1333        // OSG_NOTICE<<"options from requestNodeFile "<<std::endl;
1334    }
1335
1336
1337    if (!_acceptNewRequests) return;
1338
1339
1340    if (nodePath.empty())
1341    {
1342        OSG_NOTICE<<"Warning: DatabasePager::requestNodeFile(..) passed empty NodePath, so nowhere to attach new subgraph to."<<std::endl;
1343        return;
1344    }
1345
1346    osg::Group* group = nodePath.back()->asGroup();
1347    if (!group)
1348    {
1349        OSG_NOTICE<<"Warning: DatabasePager::requestNodeFile(..) passed NodePath without group as last node in path, so nowhere to attach new subgraph to."<<std::endl;
1350        return;
1351    }
1352
1353    osg::Node* terrain = 0;
1354    for(osg::NodePath::reverse_iterator itr = nodePath.rbegin();
1355        itr != nodePath.rend();
1356        ++itr)
1357    {
1358        if ((*itr)->asTerrain()) terrain = *itr;
1359    }
1360
1361    double timestamp = framestamp?framestamp->getReferenceTime():0.0;
1362    unsigned int frameNumber = framestamp?framestamp->getFrameNumber():static_cast<unsigned int>(_frameNumber);
1363
1364// #define WITH_REQUESTNODEFILE_TIMING
1365#ifdef WITH_REQUESTNODEFILE_TIMING
1366    osg::Timer_t start_tick = osg::Timer::instance()->tick();
1367    static int previousFrame = -1;
1368    static double totalTime = 0.0;
1369
1370    if (previousFrame!=frameNumber)
1371    {
1372        OSG_NOTICE<<"requestNodeFiles for "<<previousFrame<<" time = "<<totalTime<<std::endl;
1373
1374        previousFrame = frameNumber;
1375        totalTime = 0.0;
1376    }
1377#endif
1378
1379    // search to see if filename already exist in the file loaded list.
1380    bool foundEntry = false;
1381
1382    if (databaseRequestRef.valid())
1383    {
1384        DatabaseRequest* databaseRequest = dynamic_cast<DatabaseRequest*>(databaseRequestRef.get());
1385        bool requeue = false;
1386        if (databaseRequest)
1387        {
1388            OpenThreads::ScopedLock<OpenThreads::Mutex> drLock(_dr_mutex);
1389            if (!(databaseRequest->valid()))
1390            {
1391                OSG_INFO<<"DatabaseRequest has been previously invalidated whilst still attached to scene graph."<<std::endl;
1392                databaseRequest = 0;
1393            }
1394            else
1395            {
1396                OSG_INFO<<"DatabasePager::requestNodeFile("<<fileName<<") updating already assigned."<<std::endl;
1397
1398
1399                databaseRequest->_valid = true;
1400                databaseRequest->_frameNumberLastRequest = frameNumber;
1401                databaseRequest->_timestampLastRequest = timestamp;
1402                databaseRequest->_priorityLastRequest = priority;
1403                ++(databaseRequest->_numOfRequests);
1404
1405                foundEntry = true;
1406
1407                if (databaseRequestRef->referenceCount()==1)
1408                {
1409                    OSG_INFO<<"DatabasePager::requestNodeFile("<<fileName<<") orphaned, resubmitting."<<std::endl;
1410
1411                    databaseRequest->_frameNumberLastRequest = frameNumber;
1412                    databaseRequest->_timestampLastRequest = timestamp;
1413                    databaseRequest->_priorityLastRequest = priority;
1414                    databaseRequest->_group = group;
1415                    databaseRequest->_terrain = terrain;
1416                    databaseRequest->_loadOptions = loadOptions;
1417                    requeue = true;
1418                }
1419
1420            }
1421        }
1422        if (requeue)
1423            _fileRequestQueue->add(databaseRequest);
1424    }
1425
1426    if (!foundEntry)
1427    {
1428        OSG_INFO<<"In DatabasePager::requestNodeFile("<<fileName<<")"<<std::endl;
1429
1430        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_requestMutex);
1431
1432        if (!databaseRequestRef.valid() || databaseRequestRef->referenceCount()==1)
1433        {
1434            osg::ref_ptr<DatabaseRequest> databaseRequest = new DatabaseRequest;
1435
1436            databaseRequestRef = databaseRequest.get();
1437
1438            databaseRequest->_valid = true;
1439            databaseRequest->_fileName = fileName;
1440            databaseRequest->_frameNumberFirstRequest = frameNumber;
1441            databaseRequest->_timestampFirstRequest = timestamp;
1442            databaseRequest->_priorityFirstRequest = priority;
1443            databaseRequest->_frameNumberLastRequest = frameNumber;
1444            databaseRequest->_timestampLastRequest = timestamp;
1445            databaseRequest->_priorityLastRequest = priority;
1446            databaseRequest->_group = group;
1447            databaseRequest->_terrain = terrain;
1448            databaseRequest->_loadOptions = loadOptions;
1449
1450            _fileRequestQueue->addNoLock(databaseRequest.get());
1451        }
1452    }
1453
1454    if (!_startThreadCalled)
1455    {
1456        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_run_mutex);
1457
1458        if (!_startThreadCalled)
1459        {
1460            _startThreadCalled = true;
1461            _done = false;
1462            OSG_INFO<<"DatabasePager::startThread()"<<std::endl;
1463
1464            if (_databaseThreads.empty())
1465            {
1466                setUpThreads(
1467                    osg::DisplaySettings::instance()->getNumOfDatabaseThreadsHint(),
1468                    osg::DisplaySettings::instance()->getNumOfHttpDatabaseThreadsHint());
1469            }
1470
1471            for(DatabaseThreadList::const_iterator dt_itr = _databaseThreads.begin();
1472                dt_itr != _databaseThreads.end();
1473                ++dt_itr)
1474            {
1475                (*dt_itr)->startThread();
1476            }
1477        }
1478    }
1479
1480#ifdef WITH_REQUESTNODEFILE_TIMING
1481    totalTime += osg::Timer::instance()->delta_m(start_tick, osg::Timer::instance()->tick());
1482#endif
1483}
1484
1485void DatabasePager::signalBeginFrame(const osg::FrameStamp* framestamp)
1486{
1487#if 0
1488    OSG_NOTICE<<"DatabasePager : _fileRequestQueue->size()="<<_fileRequestQueue->size()
1489               <<", _httpRequestQueue->size()= "<<_httpRequestQueue->size()
1490               <<", _dataToCompileList->size()= "<<_dataToCompileList->size()
1491               <<", _dataToMergeList->size()= "<<_dataToMergeList->size()<<std::endl;
1492#endif
1493    if (framestamp)
1494    {
1495        _dataToCompileList->pruneOldRequestsAndCheckIfEmpty();
1496
1497        //OSG_INFO << "signalBeginFrame "<<framestamp->getFrameNumber()<<">>>>>>>>>>>>>>>>"<<std::endl;
1498        _frameNumber.exchange(framestamp->getFrameNumber());
1499
1500    } //else OSG_INFO << "signalBeginFrame >>>>>>>>>>>>>>>>"<<std::endl;
1501}
1502
1503void DatabasePager::signalEndFrame()
1504{
1505    //OSG_INFO << "signalEndFrame <<<<<<<<<<<<<<<<<<<< "<<std::endl;
1506}
1507
1508void DatabasePager::setDatabasePagerThreadPause(bool pause)
1509{
1510    if (_databasePagerThreadPaused == pause) return;
1511
1512    _databasePagerThreadPaused = pause;
1513    {
1514        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_requestMutex);
1515        _fileRequestQueue->updateBlock();
1516    }
1517    {
1518        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_httpRequestQueue->_requestMutex);
1519        _httpRequestQueue->updateBlock();
1520    }
1521}
1522
1523
1524bool DatabasePager::requiresUpdateSceneGraph() const
1525{
1526    return !(_dataToMergeList->empty());
1527}
1528
1529void DatabasePager::updateSceneGraph(const osg::FrameStamp& frameStamp)
1530{
1531
1532#define UPDATE_TIMING 0
1533#if UPDATE_TIMING
1534    osg::ElapsedTime timer;
1535    double timeFor_removeExpiredSubgraphs, timeFor_addLoadedDataToSceneGraph;
1536#endif
1537
1538    {
1539        removeExpiredSubgraphs(frameStamp);
1540
1541#if UPDATE_TIMING
1542        timeFor_removeExpiredSubgraphs = timer.elapsedTime_m();
1543#endif
1544
1545        addLoadedDataToSceneGraph(frameStamp);
1546
1547#if UPDATE_TIMING
1548        timeFor_addLoadedDataToSceneGraph = timer.elapsedTime_m() - timeFor_removeExpiredSubgraphs;
1549#endif
1550
1551    }
1552
1553#if UPDATE_TIMING
1554    double elapsedTime = timer.elapsedTime_m();
1555    if (elapsedTime>0.4)
1556    {
1557        OSG_NOTICE<<"DatabasePager::updateSceneGraph() total time = "<<elapsedTime<<"ms"<<std::endl;
1558        OSG_NOTICE<<"   timeFor_removeExpiredSubgraphs    = "<<timeFor_removeExpiredSubgraphs<<"ms"<<std::endl;
1559        OSG_NOTICE<<"   timeFor_addLoadedDataToSceneGraph = "<<timeFor_addLoadedDataToSceneGraph<<"ms"<<std::endl;
1560        OSG_NOTICE<<"   _activePagedLODList.size()        = "<<_activePagedLODList->size()<<std::endl;
1561        OSG_NOTICE<<"   _inactivePagedLODList.size()      = "<<_inactivePagedLODList->size()<<std::endl;
1562        OSG_NOTICE<<"   total                             = "<<_activePagedLODList->size() + _inactivePagedLODList->size()<<std::endl;
1563    }
1564#endif
1565}
1566
1567
1568void DatabasePager::addLoadedDataToSceneGraph(const osg::FrameStamp &frameStamp)
1569{
1570    double timeStamp = frameStamp.getReferenceTime();
1571    unsigned int frameNumber = frameStamp.getFrameNumber();
1572
1573    osg::Timer_t before = osg::Timer::instance()->tick(), mid, last;
1574
1575    RequestQueue::RequestList localFileLoadedList;
1576
1577    // get the data from the _dataToMergeList, leaving it empty via a std::vector<>.swap.
1578    _dataToMergeList->swap(localFileLoadedList);
1579
1580    mid = osg::Timer::instance()->tick();
1581
1582    // add the loaded data into the scene graph.
1583    for(RequestQueue::RequestList::iterator itr=localFileLoadedList.begin();
1584        itr!=localFileLoadedList.end();
1585        ++itr)
1586    {
1587        DatabaseRequest* databaseRequest = itr->get();
1588
1589        // No need to take _dr_mutex. The pager threads are done with
1590        // the request; the cull traversal -- which might redispatch
1591        // the request -- can't run at the sametime as this update traversal.
1592        osg::ref_ptr<osg::Group> group;
1593        if (!databaseRequest->_groupExpired && databaseRequest->_group.lock(group))
1594        {
1595            if (osgDB::Registry::instance()->getSharedStateManager())
1596                osgDB::Registry::instance()->getSharedStateManager()->share(databaseRequest->_loadedModel.get());
1597
1598            osg::PagedLOD* plod = dynamic_cast<osg::PagedLOD*>(group.get());
1599            if (plod)
1600            {
1601                plod->setTimeStamp(plod->getNumChildren(), timeStamp);
1602                plod->setFrameNumber(plod->getNumChildren(), frameNumber);
1603                plod->getDatabaseRequest(plod->getNumChildren()) = 0;
1604            }
1605            else
1606            {
1607                osg::ProxyNode* proxyNode = dynamic_cast<osg::ProxyNode*>(group.get());
1608                if (proxyNode)
1609                {
1610                    proxyNode->getDatabaseRequest(proxyNode->getNumChildren()) = 0;
1611                }
1612            }
1613
1614            group->addChild(databaseRequest->_loadedModel.get());
1615
1616            // Check if parent plod was already registered if not start visitor from parent
1617            if( plod &&
1618                !_activePagedLODList->containsPagedLOD( plod ) )
1619            {
1620                registerPagedLODs(plod, frameNumber);
1621            }
1622            else
1623            {
1624                registerPagedLODs(databaseRequest->_loadedModel.get(), frameNumber);
1625            }
1626
1627            // OSG_NOTICE<<"merged subgraph"<<databaseRequest->_fileName<<" after "<<databaseRequest->_numOfRequests<<" requests and time="<<(timeStamp-databaseRequest->_timestampFirstRequest)*1000.0<<std::endl;
1628
1629            double timeToMerge = timeStamp-databaseRequest->_timestampFirstRequest;
1630
1631            if (timeToMerge<_minimumTimeToMergeTile) _minimumTimeToMergeTile = timeToMerge;
1632            if (timeToMerge>_maximumTimeToMergeTile) _maximumTimeToMergeTile = timeToMerge;
1633
1634            _totalTimeToMergeTiles += timeToMerge;
1635            ++_numTilesMerges;
1636        }
1637        else
1638        {
1639            OSG_INFO<<"DatabasePager::addLoadedDataToSceneGraph() node in parental chain deleted, discarding subgaph."<<std::endl;
1640        }
1641
1642        // reset the loadedModel pointer
1643        databaseRequest->_loadedModel = 0;
1644
1645        // OSG_NOTICE<<"curr = "<<timeToMerge<<" min "<<getMinimumTimeToMergeTile()*1000.0<<" max = "<<getMaximumTimeToMergeTile()*1000.0<<" average = "<<getAverageTimToMergeTiles()*1000.0<<std::endl;
1646    }
1647
1648    last = osg::Timer::instance()->tick();
1649
1650    if (!localFileLoadedList.empty())
1651    {
1652        OSG_INFO<<"Done DatabasePager::addLoadedDataToSceneGraph"<<
1653            osg::Timer::instance()->delta_m(before,mid)<<"ms,\t"<<
1654            osg::Timer::instance()->delta_m(mid,last)<<"ms"<<
1655            "  objects"<<localFileLoadedList.size()<<std::endl<<std::endl;
1656    }
1657
1658}
1659
1660
1661
1662void DatabasePager::removeExpiredSubgraphs(const osg::FrameStamp& frameStamp)
1663{
1664
1665    static double s_total_iter_stage_a = 0.0;
1666    static double s_total_time_stage_a = 0.0;
1667    static double s_total_max_stage_a = 0.0;
1668
1669    static double s_total_iter_stage_b = 0.0;
1670    static double s_total_time_stage_b = 0.0;
1671    static double s_total_max_stage_b = 0.0;
1672
1673    static double s_total_iter_stage_c = 0.0;
1674    static double s_total_time_stage_c = 0.0;
1675    static double s_total_max_stage_c = 0.0;
1676
1677    if (frameStamp.getFrameNumber()==0)
1678    {
1679        // No need to remove anything on first frame.
1680        return;
1681    }
1682
1683
1684    osg::Timer_t startTick = osg::Timer::instance()->tick();
1685
1686    // numPagedLODs >= actual number of PagedLODs. There can be
1687    // invalid observer pointers in _activePagedLODList.
1688    unsigned int numPagedLODs = _activePagedLODList->size();
1689
1690    osg::Timer_t end_a_Tick = osg::Timer::instance()->tick();
1691    double time_a = osg::Timer::instance()->delta_m(startTick,end_a_Tick);
1692
1693    s_total_iter_stage_a += 1.0;
1694    s_total_time_stage_a += time_a;
1695    if (s_total_max_stage_a<time_a) s_total_max_stage_a = time_a;
1696
1697
1698    if (numPagedLODs <= _targetMaximumNumberOfPageLOD)
1699    {
1700        // nothing to do
1701        return;
1702    }
1703
1704    int numToPrune = numPagedLODs - _targetMaximumNumberOfPageLOD;
1705
1706    ObjectList childrenRemoved;
1707
1708    double expiryTime = frameStamp.getReferenceTime() - 0.1;
1709    unsigned int expiryFrame = frameStamp.getFrameNumber() - 1;
1710
1711    // First traverse inactive PagedLODs, as their children will
1712    // certainly have expired. Then traverse active nodes if we still
1713    // need to prune.
1714    //OSG_NOTICE<<"numToPrune "<<numToPrune;
1715    if (numToPrune>0)
1716        _activePagedLODList->removeExpiredChildren(
1717            numToPrune, expiryTime, expiryFrame, childrenRemoved, false);
1718    numToPrune = _activePagedLODList->size() - _targetMaximumNumberOfPageLOD;
1719    if (numToPrune>0)
1720        _activePagedLODList->removeExpiredChildren(
1721            numToPrune, expiryTime, expiryFrame, childrenRemoved, true);
1722
1723    osg::Timer_t end_b_Tick = osg::Timer::instance()->tick();
1724    double time_b = osg::Timer::instance()->delta_m(end_a_Tick,end_b_Tick);
1725
1726    s_total_iter_stage_b += 1.0;
1727    s_total_time_stage_b += time_b;
1728    if (s_total_max_stage_b<time_b) s_total_max_stage_b = time_b;
1729
1730    //OSG_NOTICE<<" childrenRemoved.size()="<<childrenRemoved.size()<<std::endl;
1731
1732    if (!childrenRemoved.empty())
1733    {
1734        // pass the objects across to the database pager delete list
1735        if (_deleteRemovedSubgraphsInDatabaseThread)
1736        {
1737            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_fileRequestQueue->_requestMutex);
1738            // splice transfers the entire list in constant time.
1739            _fileRequestQueue->_childrenToDeleteList.splice(
1740                _fileRequestQueue->_childrenToDeleteList.end(),
1741                childrenRemoved);
1742            _fileRequestQueue->updateBlock();
1743        }
1744        else
1745            childrenRemoved.clear();
1746    }
1747
1748    osg::Timer_t end_c_Tick = osg::Timer::instance()->tick();
1749    double time_c = osg::Timer::instance()->delta_m(end_b_Tick,end_c_Tick);
1750
1751    s_total_iter_stage_c += 1.0;
1752    s_total_time_stage_c += time_c;
1753    if (s_total_max_stage_c<time_c) s_total_max_stage_c = time_c;
1754
1755    OSG_INFO<<"active="<<_activePagedLODList->size()<<" overall = "<<osg::Timer::instance()->delta_m(startTick,end_c_Tick)<<
1756                              " A="<<time_a<<" avg="<<s_total_time_stage_a/s_total_iter_stage_a<<" max = "<<s_total_max_stage_a<<
1757                              " B="<<time_b<<" avg="<<s_total_time_stage_b/s_total_iter_stage_b<<" max = "<<s_total_max_stage_b<<
1758                              " C="<<time_c<<" avg="<<s_total_time_stage_c/s_total_iter_stage_c<<" max = "<<s_total_max_stage_c<<std::endl;
1759}
1760
1761class DatabasePager::FindPagedLODsVisitor : public osg::NodeVisitor
1762{
1763public:
1764
1765    FindPagedLODsVisitor(DatabasePager::PagedLODList& pagedLODList, unsigned int frameNumber):
1766        osg::NodeVisitor(osg::NodeVisitor::TRAVERSE_ALL_CHILDREN),
1767        _activePagedLODList(pagedLODList),
1768        _frameNumber(frameNumber)
1769    {
1770    }
1771
1772    META_NodeVisitor("osgDB","FindPagedLODsVisitor")
1773
1774    virtual void apply(osg::PagedLOD& plod)
1775    {
1776        plod.setFrameNumberOfLastTraversal(_frameNumber);
1777
1778        osg::observer_ptr<osg::PagedLOD> obs_ptr(&plod);
1779        _activePagedLODList.insertPagedLOD(obs_ptr);
1780
1781        traverse(plod);
1782    }
1783
1784    DatabasePager::PagedLODList& _activePagedLODList;
1785    unsigned int _frameNumber;
1786
1787protected:
1788
1789    FindPagedLODsVisitor& operator = (const FindPagedLODsVisitor&) { return *this; }
1790};
1791
1792
1793void DatabasePager::registerPagedLODs(osg::Node* subgraph, unsigned int frameNumber)
1794{
1795    if (!subgraph) return;
1796
1797    FindPagedLODsVisitor fplv(*_activePagedLODList, frameNumber);
1798    subgraph->accept(fplv);
1799}
Note: See TracBrowser for help on using the browser.