index.cpp

/* This file is part of KMail
 * Copyright (C) 2005 Luís Pedro Coelho <luis@luispedro.org>
 *
 * KMail is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * KMail is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
 * In addition, as a special exception, the copyright holders give
 * permission to link the code of this program with any edition of
 * the TQt library by Trolltech AS, Norway (or with modified versions
 * of TQt that use the same license as TQt), and distribute linked
 * combinations including the two.  You must obey the GNU General
 * Public License in all respects for all of the code used other than
 * TQt.  If you modify this file, you may extend this exception to
 * your version of the file, but you are not obligated to do so.  If
 * you do not wish to do so, delete this exception statement from
 * your version.
 */
 
#include "index.h"
 
#include "kmkernel.h"
#include "kmfoldermgr.h"
#include "kmmsgdict.h"
#include "kmfolder.h"
#include "kmsearchpattern.h"
#include "kmfoldersearch.h"
 
#include <kdebug.h>
#include <tdeapplication.h>
#include <tqfile.h>
#include <tqtimer.h>
#include <tqvaluestack.h>
#include <tqptrlist.h>
#include <tqfileinfo.h>
#ifdef HAVE_INDEXLIB
#include <indexlib/create.h>
#endif
 
#include <sys/types.h>
#include <sys/stat.h>
 
#include <iostream>
#include <algorithm>
#include <cstdlib>
 
namespace {
const unsigned int MaintenanceLimit = 1000;
const char* const folderIndexDisabledKey = "fulltextIndexDisabled";
}
 
#ifdef HAVE_INDEXLIB
static
TQValueList<int> vectorToTQValueList( const std::vector<TQ_UINT32>& input ) {
    TQValueList<int> res;
    std::copy( input.begin(), input.end(), std::back_inserter( res ) );
    return res;
}
 
static
std::vector<TQ_UINT32> TQValueListToVector( const TQValueList<int>& input ) {
    std::vector<TQ_UINT32> res;
    // res.assign( input.begin(), input.end() ) doesn't work for some reason
    for ( TQValueList<int>::const_iterator first = input.begin(), past = input.end(); first != past; ++first ) {
        res.push_back( *first );
    }
    return res;
}
#endif
 
KMMsgIndex::KMMsgIndex( TQObject* parent ):
    TQObject( parent, "index" ),
    mState( s_idle ),
#ifdef HAVE_INDEXLIB
    mLockFile( std::string( static_cast<const char*>( TQFile::encodeName( defaultPath() ) + "/lock" ) ) ),
    mIndex( 0 ),
#endif
    mIndexPath( TQFile::encodeName( defaultPath() ) ),
    mTimer( new TQTimer( this, "mTimer" ) ),
    //mSyncState( ss_none ),
    //mSyncTimer( new TQTimer( this ) ),
    mSlowDown( false ) {
    kdDebug( 5006 ) << "KMMsgIndex::KMMsgIndex()" << endl;
 
    connect( kmkernel->folderMgr(), TQ_SIGNAL( msgRemoved( KMFolder*, TQ_UINT32 ) ), TQ_SLOT( slotRemoveMessage( KMFolder*, TQ_UINT32 ) ) );
    connect( kmkernel->folderMgr(), TQ_SIGNAL( msgAdded( KMFolder*, TQ_UINT32 ) ), TQ_SLOT( slotAddMessage( KMFolder*, TQ_UINT32 ) ) );
    connect( kmkernel->dimapFolderMgr(), TQ_SIGNAL( msgRemoved( KMFolder*, TQ_UINT32 ) ), TQ_SLOT( slotRemoveMessage( KMFolder*, TQ_UINT32 ) ) );
    connect( kmkernel->dimapFolderMgr(), TQ_SIGNAL( msgAdded( KMFolder*, TQ_UINT32 ) ), TQ_SLOT( slotAddMessage( KMFolder*, TQ_UINT32 ) ) );
 
    connect( mTimer, TQ_SIGNAL( timeout() ), TQ_SLOT( act() ) );
    //connect( mSyncTimer, TQ_SIGNAL( timeout() ), TQ_SLOT( syncIndex() ) );
 
#ifdef HAVE_INDEXLIB
    TDEConfigGroup cfg( KMKernel::config(), "text-index" );
    if ( !cfg.readBoolEntry( "enabled", false ) ) {
        indexlib::remove( mIndexPath );
        mLockFile.force_unlock();
        mState = s_disabled;
        return;
    }
    if ( !mLockFile.trylock() ) {
        indexlib::remove( mIndexPath );
 
        mLockFile.force_unlock();
        mLockFile.trylock();
    } else {
        mIndex = indexlib::open( mIndexPath, indexlib::open_flags::fail_if_nonexistant ).release();
    }
    if ( !mIndex ) {
        TQTimer::singleShot( 8000, this, TQ_SLOT( create() ) );
        mState = s_willcreate;
    } else {
        if ( cfg.readBoolEntry( "creating" ) ) {
            TQTimer::singleShot( 8000, this, TQ_SLOT( continueCreation() ) );
            mState = s_creating;
        } else {
            mPendingMsgs = TQValueListToVector( cfg.readIntListEntry( "pending" ) );
            mRemovedMsgs = TQValueListToVector( cfg.readIntListEntry( "removed" ) );
        }
    }
    mIndex = 0;
#else
    mState = s_error;
#endif
    //if ( mState == s_idle ) mSyncState = ss_synced;
}
 
 
KMMsgIndex::~KMMsgIndex() {
    kdDebug( 5006 ) << "KMMsgIndex::~KMMsgIndex()" << endl;
#ifdef HAVE_INDEXLIB
    TDEConfigGroup cfg( KMKernel::config(), "text-index" );
    cfg.writeEntry( "creating", mState == s_creating );
    TQValueList<int> pendingMsg;
    if ( mState == s_processing ) {
        Q_ASSERT( mAddedMsgs.empty() );
        pendingMsg = vectorToTQValueList( mPendingMsgs );
    }
    cfg.writeEntry( "pending", pendingMsg );
    cfg.writeEntry( "removed", vectorToTQValueList( mRemovedMsgs ) );
    delete mIndex;
#endif
}
 
bool KMMsgIndex::isIndexable( KMFolder* folder ) const {
    if ( !folder || !folder->parent() ) return false;
    const KMFolderMgr* manager = folder->parent()->manager();
    return manager == kmkernel->folderMgr() || manager == kmkernel->dimapFolderMgr();
}
 
bool KMMsgIndex::isIndexed( KMFolder* folder ) const {
    if ( !isIndexable( folder ) ) return false;
    TDEConfig* config = KMKernel::config();
    TDEConfigGroupSaver saver( config, "Folder-" + folder->idString() );
    return !config->readBoolEntry( folderIndexDisabledKey, false );
}
 
void KMMsgIndex::setEnabled( bool e ) {
    kdDebug( 5006 ) << "KMMsgIndex::setEnabled( " << e << " )" << endl;
    TDEConfig* config = KMKernel::config();
    TDEConfigGroupSaver saver( config, "text-index" );
    if ( config->readBoolEntry( "enabled", !e ) == e ) return;
    config->writeEntry( "enabled", e );
    if ( e ) {
        switch ( mState ) {
            case s_idle:
            case s_willcreate:
            case s_creating:
            case s_processing:
                // nothing to do
                return;
            case s_error:
                // nothing can be done, probably
                return;
            case s_disabled:
                TQTimer::singleShot( 8000, this, TQ_SLOT( create() ) );
                mState = s_willcreate;
        }
    } else {
        clear();
    }
}
 
void KMMsgIndex::setIndexingEnabled( KMFolder* folder, bool e ) {
    TDEConfig* config = KMKernel::config();
    TDEConfigGroupSaver saver( config, "Folder-" + folder->idString() );
    if ( config->readBoolEntry( folderIndexDisabledKey, e ) == e ) return; // nothing to do
    config->writeEntry( folderIndexDisabledKey, e );
 
    if ( e ) {
        switch ( mState ) {
            case s_idle:
            case s_creating:
            case s_processing:
                mPendingFolders.push_back( folder );
                scheduleAction();
                break;
            case s_willcreate:
                // do nothing, create() will handle this
                break;
            case s_error:
            case s_disabled:
                // nothing can be done
                break;
        }
 
    } else {
        switch ( mState ) {
            case s_willcreate:
                // create() will notice that folder is disabled
                break;
            case s_creating:
                if ( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) != mPendingFolders.end() ) {
                    // easy:
                    mPendingFolders.erase( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) );
                    break;
                }
                //else  fall-through
            case s_idle:
            case s_processing:
 
            case s_error:
            case s_disabled:
                // nothing can be done
                break;
        }
    }
}
 
void KMMsgIndex::clear() {
    kdDebug( 5006 ) << "KMMsgIndex::clear()" << endl;
#ifdef HAVE_INDEXLIB
    delete mIndex;
    mLockFile.force_unlock();
    mIndex = 0;
    indexlib::remove( mIndexPath );
    mPendingMsgs.clear();
    mPendingFolders.clear();
    mMaintenanceCount = 0;
    mAddedMsgs.clear();
    mRemovedMsgs.clear();
    mExisting.clear();
    mState = s_disabled;
    for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end(); first != past; ++first ) {
        ( *first )->close("msgindex");
    }
    mOpenedFolders.clear();
    for ( std::vector<Search*>::const_iterator first = mSearches.begin(), past = mSearches.end(); first != past; ++first ) {
        delete *first;
    }
    mSearches.clear();
    mTimer->stop();
#endif
}
 
void KMMsgIndex::maintenance() {
#ifdef HAVE_INDEXLIB
    if ( mState != s_idle || kapp->hasPendingEvents() ) {
        TQTimer::singleShot( 8000, this, TQ_SLOT( maintenance() ) );
        return;
    }
    mIndex->maintenance();
#endif
}
 
int KMMsgIndex::addMessage( TQ_UINT32 serNum ) {
    kdDebug( 5006 ) << "KMMsgIndex::addMessage( " << serNum << " )" << endl;
    if ( mState == s_error ) return 0;
#ifdef HAVE_INDEXLIB
    assert( mIndex );
    if ( !mExisting.empty() && std::binary_search( mExisting.begin(), mExisting.end(), serNum ) ) return 0;
 
    int idx = -1;
    KMFolder* folder = 0;
    KMMsgDict::instance()->getLocation( serNum, &folder, &idx );
    if ( !folder || idx == -1 ) return -1;
    if ( !mOpenedFolders.count( folder ) ) {
        mOpenedFolders.insert( folder );
        folder->open("msgindex");
    }
    KMMessage* msg = folder->getMsg( idx );
    /* I still don't know whether we should allow decryption or not.
     * Setting to false which makes more sense.
     * We keep signature to get the person's name
     */
    TQString body = msg->asPlainText( false, false );
    if ( !body.isEmpty() && static_cast<const char*>( body.latin1() ) ) {
        mIndex->add( body.latin1(), TQString::number( serNum ).latin1() );
    } else {
        kdDebug( 5006 ) << "Funny, no body" << endl;
    }
    folder->unGetMsg( idx );
#endif
    return 0;
}
 
void KMMsgIndex::act() {
    kdDebug( 5006 ) << "KMMsgIndex::act()" << endl;
    if ( kapp->hasPendingEvents() ) {
        //nah, some other time..
        mTimer->start( 500 );
        mSlowDown = true;
        return;
    }
    if ( mSlowDown ) {
        mSlowDown = false;
        mTimer->start( 0 );
    }
    if ( !mPendingMsgs.empty() ) {
        addMessage( mPendingMsgs.back() );
        mPendingMsgs.pop_back();
        return;
    }
    if ( !mPendingFolders.empty() ) {
        KMFolder *f = mPendingFolders.back();
        mPendingFolders.pop_back();
        if ( !mOpenedFolders.count( f ) ) {
            mOpenedFolders.insert( f );
            f->open("msgindex");
        }
        const KMMsgDict* dict = KMMsgDict::instance();
        TDEConfig* config = KMKernel::config();
        TDEConfigGroupSaver saver( config, "Folder-" + f->idString() );
        if ( config->readBoolEntry( folderIndexDisabledKey, true ) ) {
            for ( int i = 0; i < f->count(); ++i ) {
                mPendingMsgs.push_back( dict->getMsgSerNum( f, i ) );
            }
        }
        return;
    }
    if ( !mAddedMsgs.empty() ) {
        std::swap( mAddedMsgs, mPendingMsgs );
        mState = s_processing;
        return;
    }
    for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end();
            first != past;
            ++first ) {
        ( *first )->close("msgindex");
    }
    mOpenedFolders.clear();
    mState = s_idle;
    mTimer->stop();
}
 
void KMMsgIndex::continueCreation() {
    kdDebug( 5006 ) << "KMMsgIndex::continueCreation()" << endl;
#ifdef HAVE_INDEXLIB
    create();
    unsigned count = mIndex->ndocs();
    mExisting.clear();
    mExisting.reserve( count );
    for ( unsigned i = 0; i != count; ++i ) {
        mExisting.push_back( std::atoi( mIndex->lookup_docname( i ).c_str() ) );
    }
    std::sort( mExisting.begin(), mExisting.end() );
#endif
}
 
void KMMsgIndex::create() {
    kdDebug( 5006 ) << "KMMsgIndex::create()" << endl;
 
#ifdef HAVE_INDEXLIB
    if ( !TQFileInfo( mIndexPath ).exists() ) {
        ::mkdir( mIndexPath, S_IRWXU );
    }
    mState = s_creating;
    if ( !mIndex ) mIndex = indexlib::create( mIndexPath ).release();
    if ( !mIndex ) {
        kdDebug( 5006 ) << "Error creating index" << endl;
        mState = s_error;
        return;
    }
    TQValueStack<KMFolderDir*> folders;
    folders.push(&(kmkernel->folderMgr()->dir()));
    folders.push(&(kmkernel->dimapFolderMgr()->dir()));
    while ( !folders.empty() ) {
        KMFolderDir *dir = folders.pop();
        for(KMFolderNode *child = dir->first(); child; child = dir->next()) {
            if ( child->isDir() )
                folders.push((KMFolderDir*)child);
            else
                mPendingFolders.push_back( (KMFolder*)child );
        }
    }
    mTimer->start( 4000 ); // wait a couple of seconds before starting up...
    mSlowDown = true;
#endif
}
 
bool KMMsgIndex::startQuery( KMSearch* s ) {
    kdDebug( 5006 ) << "KMMsgIndex::startQuery( . )" << endl;
    if ( mState != s_idle ) return false;
    if ( !isIndexed( s->root() ) || !canHandleQuery( s->searchPattern() ) ) return false;
 
    kdDebug( 5006 ) << "KMMsgIndex::startQuery( . ) starting query" << endl;
    Search* search = new Search( s );
    connect( search, TQ_SIGNAL( finished( bool ) ), s, TQ_SIGNAL( finished( bool ) ) );
    connect( search, TQ_SIGNAL( finished( bool ) ), s, TQ_SLOT( indexFinished() ) );
    connect( search, TQ_SIGNAL( destroyed( TQObject* ) ), TQ_SLOT( removeSearch( TQObject* ) ) );
    connect( search, TQ_SIGNAL( found( TQ_UINT32 ) ), s, TQ_SIGNAL( found( TQ_UINT32 ) ) );
    mSearches.push_back( search );
    return true;
}
 
 
//void KMMsgIndex::startSync() {
//  switch ( mSyncState ) {
//      case ss_none:
//          mIndex->start_sync();
//          mSyncState = ss_started;
//          mSyncTimer.start( 4000, true );
//          break;
//      case ss_started:
//          mIndex->sync_now();
//          mSyncState = ss_synced;
//          mLockFile.unlock();
//          break;
//  }
//}
//
//void KMMsgIndex::finishSync() {
//
//}
 
void KMMsgIndex::removeSearch( TQObject* destroyed ) {
    mSearches.erase( std::find( mSearches.begin(), mSearches.end(), destroyed ) );
}
 
 
bool KMMsgIndex::stopQuery( KMSearch* s ) {
    kdDebug( 5006 ) << "KMMsgIndex::stopQuery( . )" << endl;
    for ( std::vector<Search*>::iterator iter = mSearches.begin(), past = mSearches.end(); iter != past; ++iter ) {
        if ( ( *iter )->search() == s ) {
            delete *iter;
            mSearches.erase( iter );
            return true;
        }
    }
    return false;
}
 
std::vector<TQ_UINT32> KMMsgIndex::simpleSearch( TQString s, bool* ok ) const {
    kdDebug( 5006 ) << "KMMsgIndex::simpleSearch( -" << s.latin1() << "- )" << endl;
    if ( mState == s_error || mState == s_disabled ) {
        if ( ok ) *ok = false;
        return std::vector<TQ_UINT32>();
    }
    std::vector<TQ_UINT32> res;
#ifdef HAVE_INDEXLIB
    assert( mIndex );
    std::vector<unsigned> residx = mIndex->search( s.latin1() )->list();
    res.reserve( residx.size() );
    for ( std::vector<unsigned>::const_iterator first = residx.begin(), past = residx.end();first != past; ++first ) {
        res.push_back( std::atoi( mIndex->lookup_docname( *first ).c_str() ) );
    }
    if ( ok ) *ok = true;
#endif
    return res;
}
 
bool KMMsgIndex::canHandleQuery( const KMSearchPattern* pat ) const {
    kdDebug( 5006 ) << "KMMsgIndex::canHandleQuery( . )" << endl;
    if ( !pat ) return false;
    TQPtrListIterator<KMSearchRule> it( *pat );
    KMSearchRule* rule;
    while ( (rule = it.current()) != 0 ) {
        ++it;
        if ( !rule->field().isEmpty() && !rule->contents().isEmpty() &&
                rule->function() == KMSearchRule::FuncContains &&
                rule->field() == "<body>" )  return true;
    }
    return false;
}
 
void KMMsgIndex::slotAddMessage( KMFolder*, TQ_UINT32 serNum ) {
    kdDebug( 5006 ) << "KMMsgIndex::slotAddMessage( . , " << serNum << " )" << endl;
    if ( mState == s_error || mState == s_disabled ) return;
 
    if ( mState == s_creating ) mAddedMsgs.push_back( serNum );
    else mPendingMsgs.push_back( serNum );
 
    if ( mState == s_idle ) mState = s_processing;
    scheduleAction();
}
 
void KMMsgIndex::slotRemoveMessage( KMFolder*, TQ_UINT32 serNum ) {
    kdDebug( 5006 ) << "KMMsgIndex::slotRemoveMessage( . , " << serNum << " )" << endl;
    if ( mState == s_error || mState == s_disabled ) return;
 
    if ( mState == s_idle ) mState = s_processing;
    mRemovedMsgs.push_back( serNum );
    scheduleAction();
}
 
void KMMsgIndex::scheduleAction() {
#ifdef HAVE_INDEXLIB
    if ( mState == s_willcreate || !mIndex ) return;
    if ( !mSlowDown ) mTimer->start( 0 );
#endif
}
 
void KMMsgIndex::removeMessage( TQ_UINT32 serNum ) {
    kdDebug( 5006 ) << "KMMsgIndex::removeMessage( " << serNum << " )" << endl;
    if ( mState == s_error || mState == s_disabled ) return;
 
#ifdef HAVE_INDEXLIB
    mIndex->remove_doc( TQString::number( serNum ).latin1() );
    ++mMaintenanceCount;
    if ( mMaintenanceCount > MaintenanceLimit && mRemovedMsgs.empty() ) {
        TQTimer::singleShot( 100, this, TQ_SLOT( maintenance() ) );
    }
#endif
}
 
TQString KMMsgIndex::defaultPath() {
    return KMKernel::localDataPath() + "text-index";
}
 
bool KMMsgIndex::creating() const {
    return !mPendingMsgs.empty() || !mPendingFolders.empty();
}
 
KMMsgIndex::Search::Search( KMSearch* s ):
    mSearch( s ),
    mTimer( new TQTimer( this, "mTimer" ) ),
    mResidual( new KMSearchPattern ),
    mState( s_starting ) {
    connect( mTimer, TQ_SIGNAL( timeout() ), TQ_SLOT( act() ) );
    mTimer->start( 0 );
}
 
KMMsgIndex::Search::~Search() {
    delete mTimer;
}
 
void KMMsgIndex::Search::act() {
    switch ( mState ) {
        case s_starting: {
            KMSearchPattern* pat = mSearch->searchPattern();
            TQString terms;
            for ( KMSearchRule* rule = pat->first(); rule; rule = pat->next() ) {
                Q_ASSERT( rule->function() == KMSearchRule::FuncContains );
                terms += TQString::fromLatin1( " %1 " ).arg( rule->contents() );
            }
 
            mValues = kmkernel->msgIndex()->simpleSearch( terms, 0  );
            break;
         }
        case s_emitstopped:
            mTimer->start( 0 );
            mState = s_emitting;
            // fall throu
        case s_emitting:
            if ( kapp->hasPendingEvents() ) {
                //nah, some other time..
                mTimer->start( 250 );
                mState = s_emitstopped;
                return;
            }
            for ( int i = 0; i != 16 && !mValues.empty(); ++i ) {
                KMFolder* folder;
                int index;
                KMMsgDict::instance()->getLocation( mValues.back(), &folder, &index );
                if ( folder &&
                    mSearch->inScope( folder ) &&
                    ( !mResidual || mResidual->matches( mValues.back() ) ) ) {
 
                    emit found( mValues.back() );
                }
                mValues.pop_back();
            }
            if ( mValues.empty() ) {
                emit finished( true );
                mState = s_done;
                mTimer->stop();
                delete this;
            }
            break;
        default:
        Q_ASSERT( 0 );
    }
}
#include "index.moc"