katebuffer.cpp

/* This file is part of the KDE libraries
   Copyright (c) 2000 Waldo Bastian <bastian@kde.org>
   Copyright (C) 2002-2004 Christoph Cullmann <cullmann@kde.org>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public
   License version 2 as published by the Free Software Foundation.
 
   This library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Library General Public License for more details.
 
   You should have received a copy of the GNU Library General Public License
   along with this library; see the file COPYING.LIB.  If not, write to
   the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
   Boston, MA 02110-1301, USA.
*/
 
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
 
#include "katebuffer.h"
#include "katebuffer.moc"
 
#include "katedocument.h"
#include "katehighlight.h"
#include "kateconfig.h"
#include "katefactory.h"
#include "kateautoindent.h"
 
#include <kdebug.h>
#include <tdeglobal.h>
#include <kcharsets.h>
 
#include <tqpopupmenu.h>
#include <tqfile.h>
#include <tqtextstream.h>
#include <tqtimer.h>
#include <tqtextcodec.h>
#include <tqcstring.h>
#include <tqdatetime.h>
 
static const TQ_ULONG KATE_FILE_LOADER_BS  = 256 * 1024;
 
static const TQ_ULONG KATE_AVG_BLOCK_SIZE  = 2048 * 80;
static const TQ_ULONG KATE_MAX_BLOCK_LINES = 2048;
 
static const uint KATE_HL_LOOKAHEAD = 64;
 
uint KateBuffer::m_maxLoadedBlocks = 16;
 
static const uint KATE_MAX_DYNAMIC_CONTEXTS = 512;
 
void KateBuffer::setMaxLoadedBlocks (uint count)
{
  m_maxLoadedBlocks = kMax (4U, count);
}
 
class KateFileLoader
{
  public:
    KateFileLoader (const TQString &filename, TQTextCodec *codec, bool removeTrailingSpaces)
      : m_file (filename)
      , m_buffer (kMin ((TQ_ULONG)m_file.size(), KATE_FILE_LOADER_BS))
      , m_codec (codec)
      , m_decoder (m_codec->makeDecoder())
      , m_position (0)
      , m_lastLineStart (0)
      , m_eof (false) // default to not eof
      , lastWasEndOfLine (true) // at start of file, we had a virtual newline
      , lastWasR (false) // we have not found a \r as last char
      , m_eol (-1) // no eol type detected atm
      , m_twoByteEncoding (TQString(codec->name()) == "ISO-10646-UCS-2")
      , m_binary (false)
      , m_removeTrailingSpaces (removeTrailingSpaces)
    {
      kdDebug (13020) << "OPEN USES ENCODING: " << m_codec->name() << endl;
    }
 
    ~KateFileLoader ()
    {
      delete m_decoder;
    }
 
    bool open ()
    {
      if (m_file.open (IO_ReadOnly))
      {
        int c = m_file.readBlock (m_buffer.data(), m_buffer.size());
 
        if (c > 0)
        {
          // fix utf16 LE, stolen from tdehtml ;)
          if ((c >= 2) && (m_codec->mibEnum() == 1000) && (m_buffer[1] == 0x00))
          {
            // utf16LE, we need to put the decoder in LE mode
            char reverseUtf16[3] = {'\xFF', '\xFE', '\x00'};
            m_decoder->toUnicode(reverseUtf16, 2);
          }
 
          processNull (c);
          m_text = m_decoder->toUnicode (m_buffer, c);
        }
 
        m_eof = (c == -1) || (c == 0) || (m_text.length() == 0) || m_file.atEnd();
 
        for (uint i=0; i < m_text.length(); i++)
        {
          if (m_text[i] == '\n')
          {
            m_eol = KateDocumentConfig::eolUnix;
            break;
          }
          else if ((m_text[i] == '\r'))
          {
            if (((i+1) < m_text.length()) && (m_text[i+1] == '\n'))
            {
              m_eol = KateDocumentConfig::eolDos;
              break;
            }
            else
            {
              m_eol = KateDocumentConfig::eolMac;
              break;
            }
          }
        }
 
        return true;
      }
 
      return false;
    }
 
    // no new lines around ?
    inline bool eof () const { return m_eof && !lastWasEndOfLine && (m_lastLineStart == m_text.length()); }
 
    // eol mode ? autodetected on open(), -1 for no eol found in the first block!
    inline int eol () const { return m_eol; }
 
    // binary ?
    inline bool binary () const { return m_binary; }
 
    // should spaces be ignored at end of line?
    inline bool removeTrailingSpaces () const { return m_removeTrailingSpaces; }
 
    // internal unicode data array
    inline const TQChar *unicode () const { return m_text.unicode(); }
 
    // read a line, return length + offset in unicode data
    void readLine (uint &offset, uint &length)
    {
      length = 0;
      offset = 0;
 
      while (m_position <= m_text.length())
      {
        if (m_position == m_text.length())
        {
          // try to load more text if something is around
          if (!m_eof)
          {
            int c = m_file.readBlock (m_buffer.data(), m_buffer.size());
 
            uint readString = 0;
            if (c > 0)
            {
              processNull (c);
 
              TQString str (m_decoder->toUnicode (m_buffer, c));
              readString = str.length();
 
              m_text = m_text.mid (m_lastLineStart, m_position-m_lastLineStart)
                       + str;
            }
            else
              m_text = m_text.mid (m_lastLineStart, m_position-m_lastLineStart);
 
            // is file completly read ?
            m_eof = (c == -1) || (c == 0) || (readString == 0) || m_file.atEnd();
 
            // recalc current pos and last pos
            m_position -= m_lastLineStart;
            m_lastLineStart = 0;
          }
 
          // oh oh, end of file, escape !
          if (m_eof && (m_position == m_text.length()))
          {
            lastWasEndOfLine = false;
 
            // line data
            offset = m_lastLineStart;
            length = m_position-m_lastLineStart;
 
            m_lastLineStart = m_position;
 
            return;
          }
        }
 
        if (m_text[m_position] == '\n')
        {
          lastWasEndOfLine = true;
 
          if (lastWasR)
          {
            m_lastLineStart++;
            lastWasR = false;
          }
          else
          {
            // line data
            offset = m_lastLineStart;
            length = m_position-m_lastLineStart;
 
            m_lastLineStart = m_position+1;
            m_position++;
 
            return;
          }
        }
        else if (m_text[m_position] == '\r')
        {
          lastWasEndOfLine = true;
          lastWasR = true;
 
          // line data
          offset = m_lastLineStart;
          length = m_position-m_lastLineStart;
 
          m_lastLineStart = m_position+1;
          m_position++;
 
          return;
        }
        else
        {
          lastWasEndOfLine = false;
          lastWasR = false;
        }
 
        m_position++;
      }
    }
 
    // this nice methode will kill all 0 bytes (or double bytes)
    // and remember if this was a binary or not ;)
    void processNull (uint length)
    {
      if (m_twoByteEncoding)
      {
        for (uint i=1; i < length; i+=2)
        {
          if ((m_buffer[i] == 0) && (m_buffer[i-1] == 0))
          {
            m_binary = true;
            m_buffer[i] = ' ';
          }
        }
      }
      else
      {
        for (uint i=0; i < length; i++)
        {
          if (m_buffer[i] == 0)
          {
            m_binary = true;
            m_buffer[i] = ' ';
          }
        }
      }
    }
 
  private:
    TQFile m_file;
    TQByteArray m_buffer;
    TQTextCodec *m_codec;
    TQTextDecoder *m_decoder;
    TQString m_text;
    uint m_position;
    uint m_lastLineStart;
    bool m_eof;
    bool lastWasEndOfLine;
    bool lastWasR;
    int m_eol;
    bool m_twoByteEncoding;
    bool m_binary;
    bool m_removeTrailingSpaces;
};
 
KateBuffer::KateBuffer(KateDocument *doc)
 : TQObject (doc),
   editSessionNumber (0),
   editIsRunning (false),
   editTagLineStart (0xffffffff),
   editTagLineEnd (0),
   editTagLineFrom (false),
   editChangesDone (false),
   m_doc (doc),
   m_lines (0),
   m_lastInSyncBlock (0),
   m_lastFoundBlock (0),
   m_cacheReadError(false),
   m_cacheWriteError(false),
   m_loadingBorked (false),
   m_binary (false),
   m_highlight (0),
   m_regionTree (this),
   m_tabWidth (8),
   m_lineHighlightedMax (0),
   m_lineHighlighted (0),
   m_maxDynamicContexts (KATE_MAX_DYNAMIC_CONTEXTS)
{
  clear();
}
 
KateBuffer::~KateBuffer()
{
  // DELETE ALL BLOCKS, will free mem
  for (uint i=0; i < m_blocks.size(); i++)
    delete m_blocks[i];
 
  // release HL
  if (m_highlight)
    m_highlight->release();
}
 
void KateBuffer::editStart ()
{
  editSessionNumber++;
 
  if (editSessionNumber > 1)
    return;
 
  editIsRunning = true;
 
  editTagLineStart = 0xffffffff;
  editTagLineEnd = 0;
  editTagLineFrom = false;
 
  editChangesDone = false;
}
 
void KateBuffer::editEnd ()
{
  if (editSessionNumber == 0)
    return;
 
  editSessionNumber--;
 
  if (editSessionNumber > 0)
    return;
 
  if (editChangesDone)
  {
    // hl update !!!
    if ( m_highlight && !m_highlight->noHighlighting()
        && (editTagLineStart <= editTagLineEnd)
        && (editTagLineEnd <= m_lineHighlighted))
    {
      // look one line too far, needed for linecontinue stuff
      editTagLineEnd++;
 
      // look one line before, needed nearly 100% only for indentation based folding !
      if (editTagLineStart > 0)
        editTagLineStart--;
 
      KateBufBlock *buf2 = 0;
      bool needContinue = false;
      while ((buf2 = findBlock(editTagLineStart)))
      {
        needContinue = doHighlight (buf2,
          (editTagLineStart > buf2->startLine()) ? editTagLineStart : buf2->startLine(),
          (editTagLineEnd > buf2->endLine()) ? buf2->endLine() : editTagLineEnd,
          true);
 
        editTagLineStart = (editTagLineEnd > buf2->endLine()) ? buf2->endLine() : editTagLineEnd;
 
        if ((editTagLineStart >= m_lines) || (editTagLineStart >= editTagLineEnd))
          break;
      }
 
      if (needContinue)
        m_lineHighlighted = editTagLineStart;
 
      if (editTagLineStart > m_lineHighlightedMax)
        m_lineHighlightedMax = editTagLineStart;
    }
    else if (editTagLineStart < m_lineHighlightedMax)
      m_lineHighlightedMax = editTagLineStart;
  }
 
  editIsRunning = false;
}
 
void KateBuffer::clear()
{
  m_regionTree.clear();
 
  // cleanup the blocks
  for (uint i=0; i < m_blocks.size(); i++)
    delete m_blocks[i];
 
  m_blocks.clear ();
 
  // create a bufblock with one line, we need that, only in openFile we won't have that
  KateBufBlock *block = new KateBufBlock(this, 0, 0);
  m_blocks.append (block);
 
  // reset the state
  m_lines = block->lines();
  m_lastInSyncBlock = 0;
  m_lastFoundBlock = 0;
  m_cacheWriteError = false;
  m_cacheReadError = false;
  m_loadingBorked = false;
  m_binary = false;
 
  m_lineHighlightedMax = 0;
  m_lineHighlighted = 0;
}
 
bool KateBuffer::openFile (const TQString &m_file)
{
  KateFileLoader file (m_file, m_doc->config()->codec(), m_doc->configFlags() & KateDocument::cfRemoveSpaces);
 
  bool ok = false;
  struct stat sbuf;
  if (stat(TQFile::encodeName(m_file), &sbuf) == 0)
  {
    if (S_ISREG(sbuf.st_mode) && file.open())
      ok = true;
  }
 
  if (!ok)
  {
    clear();
    return false; // Error
  }
 
  // set eol mode, if a eol char was found in the first 256kb block and we allow this at all!
  if (m_doc->config()->allowEolDetection() && (file.eol() != -1))
    m_doc->config()->setEol (file.eol());
 
  // flush current content
  clear ();
 
  // cleanup the blocks
  for (uint i=0; i < m_blocks.size(); i++)
    delete m_blocks[i];
 
  m_blocks.clear ();
 
  // do the real work
  KateBufBlock *block = 0;
  m_lines = 0;
  while (!file.eof() && !m_cacheWriteError)
  {
    block = new KateBufBlock (this, block, 0, &file);
 
    m_lines = block->endLine ();
 
    if (m_cacheWriteError || (block->lines() == 0))
    {
      delete block;
      break;
    }
    else
      m_blocks.append (block);
  }
 
  // we had a cache write error, this load is really borked !
  if (m_cacheWriteError)
    m_loadingBorked = true;
 
  if (m_blocks.isEmpty() || (m_lines == 0))
  {
    // file was really empty, clean the buffers + emit the line changed
    // loadingBorked will be false for such files, not matter what happened
    // before
    clear ();
  }
  else
  {
    // fix region tree
    m_regionTree.fixRoot (m_lines);
  }
 
  // if we have no hl or the "None" hl activated, whole file is correct highlighted
  // after loading, which wonder ;)
  if (!m_highlight || m_highlight->noHighlighting())
  {
    m_lineHighlighted = m_lines;
    m_lineHighlightedMax = m_lines;
  }
 
  // binary?
  m_binary = file.binary ();
 
  kdDebug (13020) << "LOADING DONE" << endl;
 
  return !m_loadingBorked;
}
 
bool KateBuffer::canEncode ()
{
  TQTextCodec *codec = m_doc->config()->codec();
 
  kdDebug(13020) << "ENC NAME: " << codec->name() << endl;
 
  // hardcode some unicode encodings which can encode all chars
  if ((TQString(codec->name()) == "UTF-8") || (TQString(codec->name()) == "ISO-10646-UCS-2"))
    return true;
 
  for (uint i=0; i < m_lines; i++)
  {
    if (!codec->canEncode (plainLine(i)->string()))
    {
      kdDebug(13020) << "STRING LINE: " << plainLine(i)->string() << endl;
      kdDebug(13020) << "ENC WORKING: FALSE" << endl;
 
      return false;
    }
  }
 
  return true;
}
 
bool KateBuffer::saveFile (const TQString &m_file)
{
  TQFile file (m_file);
  TQTextStream stream (&file);
 
  if ( !file.open( IO_WriteOnly ) )
  {
    return false; // Error
  }
 
  TQTextCodec *codec = m_doc->config()->codec();
 
  // disable Unicode headers
  stream.setEncoding(TQTextStream::RawUnicode);
 
  // this line sets the mapper to the correct codec
  stream.setCodec(codec);
 
  // our loved eol string ;)
  TQString eol = m_doc->config()->eolString ();
 
  // should we strip spaces?
  bool removeTrailingSpaces = m_doc->configFlags() & KateDocument::cfRemoveSpaces;
 
  // just dump the lines out ;)
  for (uint i=0; i < m_lines; i++)
  {
    KateTextLine::Ptr textline = plainLine(i);
 
    // strip spaces
    if (removeTrailingSpaces)
    {
      int lastChar = textline->lastChar();
 
      if (lastChar > -1)
      {
        stream << TQConstString (textline->text(), lastChar+1).string();
      }
    }
    else // simple, dump the line
      stream << textline->string();
 
    if ((i+1) < m_lines)
      stream << eol;
  }
 
  file.close ();
 
  m_loadingBorked = false;
 
  return (file.status() == IO_Ok);
}
 
KateTextLine::Ptr KateBuffer::line_internal (KateBufBlock *buf, uint i)
{
  // update hl until this line + max KATE_HL_LOOKAHEAD
  KateBufBlock *buf2 = 0;
  while ((i >= m_lineHighlighted) && (buf2 = findBlock(m_lineHighlighted)))
  {
    uint end = kMin(i + KATE_HL_LOOKAHEAD, buf2->endLine());
 
    doHighlight ( buf2,
                  kMax(m_lineHighlighted, buf2->startLine()),
                  end,
                  false );
 
    m_lineHighlighted = end;
  }
 
  // update hl max
  if (m_lineHighlighted > m_lineHighlightedMax)
    m_lineHighlightedMax = m_lineHighlighted;
 
  return buf->line (i - buf->startLine());
}
 
KateBufBlock *KateBuffer::findBlock_internal (uint i, uint *index)
{
  uint lastLine = m_blocks[m_lastInSyncBlock]->endLine ();
 
  if (lastLine > i) // we are in a allready known area !
  {
    while (true)
    {
      KateBufBlock *buf = m_blocks[m_lastFoundBlock];
 
      if ( (buf->startLine() <= i)
           && (buf->endLine() > i) )
      {
        if (index)
          (*index) = m_lastFoundBlock;
 
        return m_blocks[m_lastFoundBlock];
      }
 
      if (i < buf->startLine())
        m_lastFoundBlock--;
      else
        m_lastFoundBlock++;
    }
  }
  else // we need first to resync the startLines !
  {
    if ((m_lastInSyncBlock+1) < m_blocks.size())
      m_lastInSyncBlock++;
    else
      return 0;
 
    for (; m_lastInSyncBlock < m_blocks.size(); m_lastInSyncBlock++)
    {
      // get next block
      KateBufBlock *buf = m_blocks[m_lastInSyncBlock];
 
      // sync startLine !
      buf->setStartLine (lastLine);
 
      // is it allready the searched block ?
      if ((i >= lastLine) && (i < buf->endLine()))
      {
        // remember this block as last found !
        m_lastFoundBlock = m_lastInSyncBlock;
 
        if (index)
          (*index) = m_lastFoundBlock;
 
        return buf;
      }
 
      // increase lastLine with blocklinecount
      lastLine += buf->lines ();
    }
  }
 
  // no block found !
  // index will not be set to any useful value in this case !
  return 0;
}
 
void KateBuffer::changeLine(uint i)
{
  KateBufBlock *buf = findBlock(i);
 
  if (!buf)
    return;
 
  // mark this block dirty
  buf->markDirty ();
 
  // mark buffer changed
  editChangesDone = true;
 
  // tag this line as changed
  if (i < editTagLineStart)
    editTagLineStart = i;
 
  if (i > editTagLineEnd)
    editTagLineEnd = i;
}
 
void KateBuffer::insertLine(uint i, KateTextLine::Ptr line)
{
  uint index = 0;
  KateBufBlock *buf;
  if (i == m_lines)
    buf = findBlock(i-1, &index);
  else
    buf = findBlock(i, &index);
 
  if (!buf)
    return;
 
  buf->insertLine(i -  buf->startLine(), line);
 
  if (m_lineHighlightedMax > i)
    m_lineHighlightedMax++;
 
  if (m_lineHighlighted > i)
    m_lineHighlighted++;
 
  m_lines++;
 
  // last sync block adjust
  if (m_lastInSyncBlock > index)
    m_lastInSyncBlock = index;
 
  // last found
  if (m_lastInSyncBlock < m_lastFoundBlock)
    m_lastFoundBlock = m_lastInSyncBlock;
 
  // mark buffer changed
  editChangesDone = true;
 
  // tag this line as inserted
  if (i < editTagLineStart)
    editTagLineStart = i;
 
  if (i <= editTagLineEnd)
    editTagLineEnd++;
 
  if (i > editTagLineEnd)
    editTagLineEnd = i;
 
  // line inserted
  editTagLineFrom = true;
 
  m_regionTree.lineHasBeenInserted (i);
}
 
void KateBuffer::removeLine(uint i)
{
   uint index = 0;
   KateBufBlock *buf = findBlock(i, &index);
 
   if (!buf)
     return;
 
  buf->removeLine(i -  buf->startLine());
 
  if (m_lineHighlightedMax > i)
    m_lineHighlightedMax--;
 
  if (m_lineHighlighted > i)
    m_lineHighlighted--;
 
  m_lines--;
 
  // trash away a empty block
  if (buf->lines() == 0)
  {
    // we need to change which block is last in sync
    if (m_lastInSyncBlock >= index)
    {
      m_lastInSyncBlock = index;
 
      if (buf->next())
      {
        if (buf->prev())
          buf->next()->setStartLine (buf->prev()->endLine());
        else
          buf->next()->setStartLine (0);
      }
    }
 
    // cu block !
    delete buf;
    m_blocks.erase (m_blocks.begin()+index);
 
    // make sure we don't keep a pointer to the deleted block
    if( m_lastInSyncBlock >= index )
      m_lastInSyncBlock = index - 1;
  }
  else
  {
    // last sync block adjust
    if (m_lastInSyncBlock > index)
      m_lastInSyncBlock = index;
  }
 
  // last found
  if (m_lastInSyncBlock < m_lastFoundBlock)
    m_lastFoundBlock = m_lastInSyncBlock;
 
  // mark buffer changed
  editChangesDone = true;
 
  // tag this line as removed
   if (i < editTagLineStart)
    editTagLineStart = i;
 
  if (i < editTagLineEnd)
    editTagLineEnd--;
 
  if (i > editTagLineEnd)
    editTagLineEnd = i;
 
  // line removed
  editTagLineFrom = true;
 
  m_regionTree.lineHasBeenRemoved (i);
}
 
void KateBuffer::setTabWidth (uint w)
{
  if ((m_tabWidth != w) && (m_tabWidth > 0))
  {
    m_tabWidth = w;
 
    if (m_highlight && m_highlight->foldingIndentationSensitive())
      invalidateHighlighting();
  }
}
 
void KateBuffer::setHighlight(uint hlMode)
{
  KateHighlighting *h = KateHlManager::self()->getHl(hlMode);
 
   // aha, hl will change
  if (h != m_highlight)
  {
    bool invalidate = !h->noHighlighting();
 
    if (m_highlight)
    {
      m_highlight->release();
      invalidate = true;
    }
 
    h->use();
 
    // Clear code folding tree (see bug #124102)
    m_regionTree.clear();
    m_regionTree.fixRoot(m_lines);
 
    // try to set indentation
    if (!h->indentation().isEmpty())
      m_doc->config()->setIndentationMode (KateAutoIndent::modeNumber(h->indentation()));
 
    m_highlight = h;
 
    if (invalidate)
      invalidateHighlighting();
 
    // inform the document that the hl was really changed
    // needed to update attributes and more ;)
    m_doc->bufferHlChanged ();
  }
}
 
void KateBuffer::invalidateHighlighting()
{
  m_lineHighlightedMax = 0;
  m_lineHighlighted = 0;
}
 
 
void KateBuffer::updatePreviousNotEmptyLine(KateBufBlock *blk,uint current_line,bool addindent,uint deindent)
{
  KateTextLine::Ptr textLine;
  do {
    if (current_line>0) current_line--;
    else
    {
      uint line=blk->startLine()+current_line;
      if (line==0) return;
      line--;
      blk=findBlock(line);
      if (!blk) {
        kdDebug(13020)<<"updatePreviousNotEmptyLine: block not found, this must not happen"<<endl;
        return;
      }
      current_line=line-blk->startLine();
    }
    textLine = blk->line(current_line);
  } while (textLine->firstChar()==-1);
  kdDebug(13020)<<"updatePreviousNotEmptyLine: updating line:"<<(blk->startLine()+current_line)<<endl;
  TQMemArray<uint> foldingList=textLine->foldingListArray();
  while ( (foldingList.size()>0)  && ( labs(foldingList[foldingList.size()-2])==1)) {
    foldingList.resize(foldingList.size()-2,TQGArray::SpeedOptim);
  }
  addIndentBasedFoldingInformation(foldingList,addindent,deindent);
  textLine->setFoldingList(foldingList);
  bool retVal_folding = false;
  m_regionTree.updateLine (current_line + blk->startLine(), &foldingList, &retVal_folding, true,false);
  emit tagLines (blk->startLine()+current_line, blk->startLine()+current_line);
}
 
void KateBuffer::addIndentBasedFoldingInformation(TQMemArray<uint> &foldingList,bool addindent,uint deindent)
{
  if (addindent) {
    //kdDebug(13020)<<"adding indent for line :"<<current_line + buf->startLine()<<"  textLine->noIndentBasedFoldingAtStart"<<textLine->noIndentBasedFoldingAtStart()<<endl;
    kdDebug(13020)<<"adding ident"<<endl;
    foldingList.resize (foldingList.size() + 2, TQGArray::SpeedOptim);
    foldingList[foldingList.size()-2] = 1;
    foldingList[foldingList.size()-1] = 0;
  }
  kdDebug(13020)<<"DEINDENT: "<<deindent<<endl;
  if (deindent > 0)
  {
    foldingList.resize (foldingList.size() + (deindent*2), TQGArray::SpeedOptim);
 
    for (uint z= foldingList.size()-(deindent*2); z < foldingList.size(); z=z+2)
    {
      foldingList[z] = -1;
      foldingList[z+1] = 0;
    }
  }
}
 
bool KateBuffer::doHighlight (KateBufBlock *buf, uint startLine, uint endLine, bool invalidate)
{
  // no hl around, no stuff to do
  if (!m_highlight)
    return false;
 
  /*if (m_highlight->foldingIndentationSensitive())
  {
    startLine=0;
    endLine=50;
  }*/
 
  // we tried to start in a line behind this buf block !
  if (startLine >= (buf->startLine()+buf->lines()))
    return false;
 
  //TQTime t;
  //t.start();
  //kdDebug (13020) << "HIGHLIGHTED START --- NEED HL, LINESTART: " << startLine << " LINEEND: " << endLine << endl;
  //kdDebug (13020) << "HL UNTIL LINE: " << m_lineHighlighted << " MAX: " << m_lineHighlightedMax << endl;
  //kdDebug (13020) << "HL DYN COUNT: " << KateHlManager::self()->countDynamicCtxs() << " MAX: " << m_maxDynamicContexts << endl;
 
  // see if there are too many dynamic contexts; if yes, invalidate HL of all documents
  if (KateHlManager::self()->countDynamicCtxs() >= m_maxDynamicContexts)
  {
    {
      if (KateHlManager::self()->resetDynamicCtxs())
      {
        kdDebug (13020) << "HL invalidated - too many dynamic contexts ( >= " << m_maxDynamicContexts << ")" << endl;
 
        // avoid recursive invalidation
        KateHlManager::self()->setForceNoDCReset(true);
 
        for (KateDocument *doc = KateFactory::self()->documents()->first(); doc; doc = KateFactory::self()->documents()->next())
          doc->makeAttribs();
 
        // doHighlight *shall* do his work. After invalidation, some highlight has
        // been recalculated, but *maybe not* until endLine ! So we shall force it manually...
        KateBufBlock *buf = 0;
        while ((endLine > m_lineHighlighted) && (buf = findBlock(m_lineHighlighted)))
        {
          uint end = kMin(endLine, buf->endLine());
 
          doHighlight ( buf,
                        kMax(m_lineHighlighted, buf->startLine()),
                        end,
                        false );
 
          m_lineHighlighted = end;
        }
 
        KateHlManager::self()->setForceNoDCReset(false);
 
        return false;
      }
      else
      {
        m_maxDynamicContexts *= 2;
        kdDebug (13020) << "New dynamic contexts limit: " << m_maxDynamicContexts << endl;
      }
    }
  }
 
  // get the previous line, if we start at the beginning of this block
  // take the last line of the previous block
  KateTextLine::Ptr prevLine = 0;
 
  if ((startLine == buf->startLine()) && buf->prev() && (buf->prev()->lines() > 0))
    prevLine = buf->prev()->line (buf->prev()->lines() - 1);
  else if ((startLine > buf->startLine()) && (startLine <= buf->endLine()))
    prevLine = buf->line(startLine - buf->startLine() - 1);
  else
    prevLine = new KateTextLine ();
 
  // does we need to emit a signal for the folding changes ?
  bool codeFoldingUpdate = false;
 
  // here we are atm, start at start line in the block
  uint current_line = startLine - buf->startLine();
 
  // do we need to continue
  bool stillcontinue=false;
  bool indentContinueWhitespace=false;
  bool indentContinueNextWhitespace=false;
  // loop over the lines of the block, from startline to endline or end of block
  // if stillcontinue forces us to do so
  while ( (current_line < buf->lines())
          && (stillcontinue || ((current_line + buf->startLine()) <= endLine)) )
  {
    // current line
    KateTextLine::Ptr textLine = buf->line(current_line);
 
    TQMemArray<uint> foldingList;
    bool ctxChanged = false;
 
    m_highlight->doHighlight (prevLine, textLine, &foldingList, &ctxChanged);
 
    //
    // indentation sensitive folding
    //
    bool indentChanged = false;
    if (m_highlight->foldingIndentationSensitive())
    {
      // get the indentation array of the previous line to start with !
      TQMemArray<unsigned short> indentDepth;
      indentDepth.duplicate (prevLine->indentationDepthArray());
 
      // current indentation of this line
      uint iDepth = textLine->indentDepth(m_tabWidth);
      if ((current_line+buf->startLine())==0)
      {
          indentDepth.resize (1, TQGArray::SpeedOptim);
          indentDepth[0] = iDepth;
      }
 
      textLine->setNoIndentBasedFoldingAtStart(prevLine->noIndentBasedFolding());
      // this line is empty, beside spaces, or has indentaion based folding disabled, use indentation depth of the previous line !
      kdDebug(13020)<<"current_line:"<<current_line + buf->startLine()<<" textLine->noIndentBasedFoldingAtStart"<<textLine->noIndentBasedFoldingAtStart()<<endl;
      if ( (textLine->firstChar() == -1) || textLine->noIndentBasedFoldingAtStart())
      {
        // do this to get skipped empty lines indent right, which was given in the indenation array
        if (!prevLine->indentationDepthArray().isEmpty())
        {
          iDepth = (prevLine->indentationDepthArray())[prevLine->indentationDepthArray().size()-1];
          kdDebug(13020)<<"reusing old depth as current"<<endl;
        }
        else
        {
          iDepth = prevLine->indentDepth(m_tabWidth);
          kdDebug(13020)<<"creating indentdepth for previous line"<<endl;
        }
      }
 
      kdDebug(13020)<<"iDepth:"<<iDepth<<endl;
 
      // query the next line indentation, if we are at the end of the block
      // use the first line of the next buf block
      uint nextLineIndentation = 0;
      bool nextLineIndentationValid=true;
      indentContinueNextWhitespace=false;
      if ((current_line+1) < buf->lines())
      {
        if (buf->line(current_line+1)->firstChar() == -1)
        {
          nextLineIndentation = iDepth;
          indentContinueNextWhitespace=true;
        }
        else
          nextLineIndentation = buf->line(current_line+1)->indentDepth(m_tabWidth);
      }
      else
      {
        KateBufBlock *blk = buf->next();
 
        if (blk && (blk->lines() > 0))
        {
          if (blk->line (0)->firstChar() == -1)
          {
            nextLineIndentation = iDepth;
            indentContinueNextWhitespace=true;
          }
          else
            nextLineIndentation = blk->line (0)->indentDepth(m_tabWidth);
        }
        else nextLineIndentationValid=false;
      }
 
      if  (!textLine->noIndentBasedFoldingAtStart()) {
 
        if ((iDepth > 0) && (indentDepth.isEmpty() || (indentDepth[indentDepth.size()-1] < iDepth)))
        {
          kdDebug(13020)<<"adding depth to \"stack\":"<<iDepth<<endl;
          indentDepth.resize (indentDepth.size()+1, TQGArray::SpeedOptim);
          indentDepth[indentDepth.size()-1] = iDepth;
        } else {
          if (!indentDepth.isEmpty())
          {
            for (int z=indentDepth.size()-1; z > -1; z--)
              if (indentDepth[z]>iDepth)
                indentDepth.resize(z, TQGArray::SpeedOptim);
            if ((iDepth > 0) && (indentDepth.isEmpty() || (indentDepth[indentDepth.size()-1] < iDepth)))
            {
              kdDebug(13020)<<"adding depth to \"stack\":"<<iDepth<<endl;
              indentDepth.resize (indentDepth.size()+1, TQGArray::SpeedOptim);
              indentDepth[indentDepth.size()-1] = iDepth;
              if (prevLine->firstChar()==-1) {
 
              }
            }
          }
        }
      }
 
      if (!textLine->noIndentBasedFolding())
      {
        if (nextLineIndentationValid)
        {
          //if (textLine->firstChar()!=-1)
          {
            kdDebug(13020)<<"nextLineIndentation:"<<nextLineIndentation<<endl;
            bool addindent=false;
            uint deindent=0;
            if (!indentDepth.isEmpty())
              kdDebug()<<"indentDepth[indentDepth.size()-1]:"<<indentDepth[indentDepth.size()-1]<<endl;
            if ((nextLineIndentation>0) && ( indentDepth.isEmpty() || (indentDepth[indentDepth.size()-1]<nextLineIndentation)))
            {
              kdDebug(13020)<<"addindent==true"<<endl;
              addindent=true;
            } else {
            if ((!indentDepth.isEmpty()) && (indentDepth[indentDepth.size()-1]>nextLineIndentation))
              {
                kdDebug(13020)<<"...."<<endl;
                for (int z=indentDepth.size()-1; z > -1; z--)
                {
                  kdDebug(13020)<<indentDepth[z]<<"  "<<nextLineIndentation<<endl;
                  if (indentDepth[z]>nextLineIndentation)
                    deindent++;
                }
              }
            }
/*        }
        if (textLine->noIndentBasedFolding()) kdDebug(13020)<<"=============================indentation based folding disabled======================"<<endl;
        if (!textLine->noIndentBasedFolding()) {*/
            if ((textLine->firstChar()==-1)) {
              updatePreviousNotEmptyLine(buf,current_line,addindent,deindent);
              codeFoldingUpdate=true;
            }
            else
            {
              addIndentBasedFoldingInformation(foldingList,addindent,deindent);
            }
          }
        }
      }
      indentChanged = !(indentDepth == textLine->indentationDepthArray());
 
      // assign the new array to the textline !
      if (indentChanged)
        textLine->setIndentationDepth (indentDepth);
 
      indentContinueWhitespace=textLine->firstChar()==-1;
    }
    bool foldingColChanged=false;
    bool foldingChanged = false; 
    if (foldingList.size()!=textLine->foldingListArray().size()) {
      foldingChanged=true;
    } else {
      TQMemArray<uint>::ConstIterator it=foldingList.begin();
      TQMemArray<uint>::ConstIterator it1=textLine->foldingListArray();
      bool markerType=true;
      for(;it!=foldingList.end();++it,++it1) {
        if  (markerType) {
          if ( ((*it)!=(*it1))) {
            foldingChanged=true;
            foldingColChanged=false;
            break;
          }
        } else {
            if ((*it)!=(*it1)) {
              foldingColChanged=true;
            }
        }
        markerType=!markerType;
      }
    }
 
    if (foldingChanged || foldingColChanged) {
      textLine->setFoldingList(foldingList);
      if (foldingChanged==false){
        textLine->setFoldingColumnsOutdated(textLine->foldingColumnsOutdated() | foldingColChanged);
      } else textLine->setFoldingColumnsOutdated(false);
    }
    bool retVal_folding = false;
    //perhaps make en enums out of the change flags
    m_regionTree.updateLine (current_line + buf->startLine(), &foldingList, &retVal_folding, foldingChanged,foldingColChanged);
 
    codeFoldingUpdate = codeFoldingUpdate | retVal_folding;
 
    // need we to continue ?
    stillcontinue = ctxChanged || indentChanged || indentContinueWhitespace || indentContinueNextWhitespace;
 
    // move around the lines
    prevLine = textLine;
 
    // increment line
    current_line++;
  }
 
  buf->markDirty ();
 
  // tag the changed lines !
  if (invalidate)
    emit tagLines (startLine, current_line + buf->startLine());
 
  // emit that we have changed the folding
  if (codeFoldingUpdate)
    emit codeFoldingUpdated();
 
  //kdDebug (13020) << "HIGHLIGHTED END --- NEED HL, LINESTART: " << startLine << " LINEEND: " << endLine << endl;
  //kdDebug (13020) << "HL UNTIL LINE: " << m_lineHighlighted << " MAX: " << m_lineHighlightedMax << endl;
  //kdDebug (13020) << "HL DYN COUNT: " << KateHlManager::self()->countDynamicCtxs() << " MAX: " << m_maxDynamicContexts << endl;
  //kdDebug (13020) << "TIME TAKEN: " << t.elapsed() << endl;
 
  // if we are at the last line of the block + we still need to continue
  // return the need of that !
  return stillcontinue && ((current_line+1) == buf->lines());
}
 
void KateBuffer::codeFoldingColumnUpdate(unsigned int lineNr) {
  KateTextLine::Ptr line=plainLine(lineNr);
  if (!line) return;
  if (line->foldingColumnsOutdated()) {
    line->setFoldingColumnsOutdated(false);
    bool tmp;
    TQMemArray<uint> folding=line->foldingListArray();
    m_regionTree.updateLine(lineNr,&folding,&tmp,true,false);
  }
}
 
//BEGIN KateBufBlock
 
KateBufBlock::KateBufBlock ( KateBuffer *parent, KateBufBlock *prev, KateBufBlock *next,
                             KateFileLoader *stream )
: m_state (KateBufBlock::stateDirty),
  m_startLine (0),
  m_lines (0),
  m_vmblock (0),
  m_vmblockSize (0),
  m_parent (parent),
  m_prev (prev),
  m_next (next),
  list (0),
  listPrev (0),
  listNext (0)
{
  // init startline + the next pointers of the neighbour blocks
  if (m_prev)
  {
    m_startLine = m_prev->endLine ();
    m_prev->m_next = this;
  }
 
  if (m_next)
    m_next->m_prev = this;
 
  // we have a stream, use it to fill the block !
  // this can lead to 0 line blocks which are invalid !
  if (stream)
  {
    // this we lead to either dirty or swapped state
    fillBlock (stream);
  }
  else // init the block if no stream given !
  {
    // fill in one empty line !
    KateTextLine::Ptr textLine = new KateTextLine ();
    m_stringList.push_back (textLine);
    m_lines++;
 
    // if we have allready enough blocks around, swap one
    if (m_parent->m_loadedBlocks.count() >= KateBuffer::maxLoadedBlocks())
      m_parent->m_loadedBlocks.first()->swapOut();
 
    // we are a new nearly empty dirty block
    m_state = KateBufBlock::stateDirty;
    m_parent->m_loadedBlocks.append (this);
  }
}
 
KateBufBlock::~KateBufBlock ()
{
  // sync prev/next pointers
  if (m_prev)
    m_prev->m_next = m_next;
 
  if (m_next)
    m_next->m_prev = m_prev;
 
  // if we have some swapped data allocated, free it now or never
  if (m_vmblock)
    KateFactory::self()->vm()->free(m_vmblock);
 
  // remove me from the list I belong
  KateBufBlockList::remove (this);
}
 
void KateBufBlock::fillBlock (KateFileLoader *stream)
{
  // is allready too much stuff around in mem ?
  bool swap = m_parent->m_loadedBlocks.count() >= KateBuffer::maxLoadedBlocks();
 
  TQByteArray rawData;
 
  // calcs the approx size for KATE_AVG_BLOCK_SIZE chars !
  if (swap)
    rawData.resize ((KATE_AVG_BLOCK_SIZE * sizeof(TQChar)) + ((KATE_AVG_BLOCK_SIZE/80) * 8));
 
  char *buf = rawData.data ();
  uint size = 0;
  uint blockSize = 0;
  while (!stream->eof() && (blockSize < KATE_AVG_BLOCK_SIZE) && (m_lines < KATE_MAX_BLOCK_LINES))
  {
    uint offset = 0, length = 0;
    stream->readLine(offset, length);
    const TQChar *unicodeData = stream->unicode () + offset;
 
    // strip spaces at end of line
    if ( stream->removeTrailingSpaces() )
    {
      while (length > 0)
      {
        if (unicodeData[length-1].isSpace())
          --length;
        else
          break;
      }
    }
 
    blockSize += length;
 
    if (swap)
    {
      // create the swapped data on the fly, no need to waste time
      // via going over the textline classes and dump them !
      char attr = KateTextLine::flagNoOtherData;
      uint pos = size;
 
      // calc new size
      size = size + 1 + sizeof(uint) + (sizeof(TQChar)*length);
 
      if (size > rawData.size ())
      {
        rawData.resize (size);
        buf = rawData.data ();
      }
 
      memcpy(buf+pos, (char *) &attr, 1);
      pos += 1;
 
      memcpy(buf+pos, (char *) &length, sizeof(uint));
      pos += sizeof(uint);
 
      memcpy(buf+pos, (char *) unicodeData, sizeof(TQChar)*length);
      pos += sizeof(TQChar)*length;
    }
    else
    {
      KateTextLine::Ptr textLine = new KateTextLine ();
      textLine->insertText (0, length, unicodeData);
      m_stringList.push_back (textLine);
    }
 
    m_lines++;
  }
 
  if (swap)
  {
    m_vmblock = KateFactory::self()->vm()->allocate(size);
    m_vmblockSize = size;
 
    if (!rawData.isEmpty())
    {
      if (!KateFactory::self()->vm()->copyBlock(m_vmblock, rawData.data(), 0, size))
      {
        if (m_vmblock)
          KateFactory::self()->vm()->free(m_vmblock);
 
        m_vmblock = 0;
        m_vmblockSize = 0;
 
        m_parent->m_cacheWriteError = true;
      }
    }
 
    // fine, we are swapped !
    m_state = KateBufBlock::stateSwapped;
  }
  else
  {
    // we are a new dirty block without any swap data
    m_state = KateBufBlock::stateDirty;
    m_parent->m_loadedBlocks.append (this);
  }
 
  kdDebug (13020) << "A BLOCK LOADED WITH LINES: " << m_lines << endl;
}
 
KateTextLine::Ptr KateBufBlock::line(uint i)
{
  // take care that the string list is around !!!
  if (m_state == KateBufBlock::stateSwapped)
    swapIn ();
 
  // LRU
  if (!m_parent->m_loadedBlocks.isLast(this))
    m_parent->m_loadedBlocks.append (this);
 
  return m_stringList[i];
}
 
void KateBufBlock::insertLine(uint i, KateTextLine::Ptr line)
{
  // take care that the string list is around !!!
  if (m_state == KateBufBlock::stateSwapped)
    swapIn ();
 
  m_stringList.insert (m_stringList.begin()+i, line);
  m_lines++;
 
  markDirty ();
}
 
void KateBufBlock::removeLine(uint i)
{
  // take care that the string list is around !!!
  if (m_state == KateBufBlock::stateSwapped)
    swapIn ();
 
  m_stringList.erase (m_stringList.begin()+i);
  m_lines--;
 
  markDirty ();
}
 
void KateBufBlock::markDirty ()
{
  if (m_state != KateBufBlock::stateSwapped)
  {
    // LRU
    if (!m_parent->m_loadedBlocks.isLast(this))
      m_parent->m_loadedBlocks.append (this);
 
    if (m_state == KateBufBlock::stateClean)
    {
      // if we have some swapped data allocated which is dirty, free it now
      if (m_vmblock)
        KateFactory::self()->vm()->free(m_vmblock);
 
      m_vmblock = 0;
      m_vmblockSize = 0;
 
      // we are dirty
      m_state = KateBufBlock::stateDirty;
    }
  }
}
 
void KateBufBlock::swapIn ()
{
  if (m_state != KateBufBlock::stateSwapped)
    return;
 
  TQByteArray rawData (m_vmblockSize);
 
  // what to do if that fails ?
  if (!KateFactory::self()->vm()->copyBlock(rawData.data(), m_vmblock, 0, rawData.size()))
    m_parent->m_cacheReadError = true;
 
  // reserve mem, keep realloc away on push_back
  m_stringList.reserve (m_lines);
 
  char *buf = rawData.data();
  for (uint i=0; i < m_lines; i++)
  {
    KateTextLine::Ptr textLine = new KateTextLine ();
    buf = textLine->restore (buf);
    m_stringList.push_back (textLine);
  }
 
  // if we have allready enough blocks around, swap one
  if (m_parent->m_loadedBlocks.count() >= KateBuffer::maxLoadedBlocks())
    m_parent->m_loadedBlocks.first()->swapOut();
 
  // fine, we are now clean again, save state + append to clean list
  m_state = KateBufBlock::stateClean;
  m_parent->m_loadedBlocks.append (this);
}
 
void KateBufBlock::swapOut ()
{
  if (m_state == KateBufBlock::stateSwapped)
    return;
 
  if (m_state == KateBufBlock::stateDirty)
  {
    bool haveHl = m_parent->m_highlight && !m_parent->m_highlight->noHighlighting();
 
    // Calculate size.
    uint size = 0;
    for (uint i=0; i < m_lines; i++)
      size += m_stringList[i]->dumpSize (haveHl);
 
    TQByteArray rawData (size);
    char *buf = rawData.data();
 
    // Dump textlines
    for (uint i=0; i < m_lines; i++)
      buf = m_stringList[i]->dump (buf, haveHl);
 
    m_vmblock = KateFactory::self()->vm()->allocate(rawData.size());
    m_vmblockSize = rawData.size();
 
    if (!rawData.isEmpty())
    {
      if (!KateFactory::self()->vm()->copyBlock(m_vmblock, rawData.data(), 0, rawData.size()))
      {
        if (m_vmblock)
          KateFactory::self()->vm()->free(m_vmblock);
 
        m_vmblock = 0;
        m_vmblockSize = 0;
 
        m_parent->m_cacheWriteError = true;
 
        return;
      }
    }
  }
 
  m_stringList.clear();
 
  // we are now swapped out, set state + remove us out of the lists !
  m_state = KateBufBlock::stateSwapped;
  KateBufBlockList::remove (this);
}
 
//END KateBufBlock
 
//BEGIN KateBufBlockList
 
KateBufBlockList::KateBufBlockList ()
 : m_count (0),
   m_first (0),
   m_last (0)
{
}
 
void KateBufBlockList::append (KateBufBlock *buf)
{
  if (buf->list)
    buf->list->removeInternal (buf);
 
  m_count++;
 
  // append a element
  if (m_last)
  {
    m_last->listNext = buf;
 
    buf->listPrev = m_last;
    buf->listNext = 0;
 
    m_last = buf;
 
    buf->list = this;
 
    return;
  }
 
  // insert the first element
  m_last = buf;
  m_first = buf;
 
  buf->listPrev = 0;
  buf->listNext = 0;
 
  buf->list = this;
}
 
void KateBufBlockList::removeInternal (KateBufBlock *buf)
{
  if (buf->list != this)
    return;
 
  m_count--;
 
  if ((buf == m_first) && (buf == m_last))
  {
    // last element removed !
    m_first = 0;
    m_last = 0;
  }
  else if (buf == m_first)
  {
    // first element removed
    m_first = buf->listNext;
    m_first->listPrev = 0;
  }
  else if (buf == m_last)
  {
    // last element removed
    m_last = buf->listPrev;
    m_last->listNext = 0;
  }
  else
  {
    buf->listPrev->listNext = buf->listNext;
    buf->listNext->listPrev = buf->listPrev;
  }
 
  buf->listPrev = 0;
  buf->listNext = 0;
 
  buf->list = 0;
}
 
//END KateBufBlockList