void indri::file::Path::remove( const std::string& path ) {
  std::stack<indri::file::DirectoryIterator*> iterators;
  indri::utility::StackDeleter<indri::file::DirectoryIterator> sd( iterators );
  iterators.push( new indri::file::DirectoryIterator( path ) );

  while( iterators.size() ) {
    indri::file::DirectoryIterator* top = iterators.top();
    
    // all done, so go up a level
    if( (*top) == indri::file::DirectoryIterator::end() ) {
      // release any search handles that may point
      // to this directory
      top->close();

      int result = rmdir( top->base().c_str() );
      if( result != 0 )
        LEMUR_THROW( LEMUR_IO_ERROR, "indri::file::Path::remove couldn't remove directory '" + top->base() + "'." );

      delete top;
      iterators.pop();
      continue;
    }

    std::string path = **top;
    (*top)++;

    if( indri::file::Path::isFile( path ) ) {
      int result = lemur_compat::remove( path.c_str() );
      if( result != 0 )
        LEMUR_THROW( LEMUR_IO_ERROR, "indri::file::Path::remove couldn't remove file '" + path + "'." );
    } else {
      iterators.push( new indri::file::DirectoryIterator( path ) );
    }
  }
}

void indri::file::Path::rename( const std::string& oldName, const std::string& newName ) {
#ifndef WIN32
  int result = ::rename( oldName.c_str(), newName.c_str() );

  if( result != 0 ) {
    if( errno == EEXIST ) {
      LEMUR_THROW( LEMUR_IO_ERROR, "The destination file already exists: " + oldName );
    } else if( errno == EACCES || errno == EPERM ) {
      LEMUR_THROW( LEMUR_IO_ERROR, "Insufficient permissions to rename: '" + oldName + "' to '" + newName + "'." );
    } else {
      LEMUR_THROW( LEMUR_IO_ERROR, "Unable to rename: '" + oldName + "' to '" + newName + "'." );
    }
  }
#else
  BOOL result;

  if( Path::exists( newName ) ) {
    result = ReplaceFile( newName.c_str(), oldName.c_str(), NULL, REPLACEFILE_IGNORE_MERGE_ERRORS, NULL, NULL );
  } else {
    result = MoveFile( oldName.c_str(), newName.c_str() );
  }

  if( !result ) {
    LEMUR_THROW( LEMUR_IO_ERROR, "Unable to rename: '" + oldName + "' to '" + newName + "'." );
  }
#endif
}

      bool listen( unsigned int port ) {
        int result;
 
        lemur_compat::initializeNetwork();
        _socket = ::socket( AF_INET, SOCK_STREAM, 0 );

        sockaddr_in sa;
        sa.sin_addr.s_addr = INADDR_ANY;
        sa.sin_port = htons(port);
        sa.sin_family = AF_INET;
        memset( &sa.sin_zero, 0, sizeof sa.sin_zero );

        result = ::bind( _socket, (const sockaddr*) &sa, sizeof sa );
        if( result ) {
          close();
          LEMUR_THROW( LEMUR_IO_ERROR, "Wasn't able to bind port " + i64_to_string(port) );
        }
  
        result = ::listen( _socket, 8 );

        if( result ) {
          close();
          LEMUR_THROW( LEMUR_IO_ERROR, "Wasn't able to listen on port " + i64_to_string(port) );
        }

        return true;
      }

void indri::xml::XMLReader::_read( indri::xml::XMLNode** parent, const char* buffer, int start, int end ) {
  int tagType;

  for( int current = _tryFindBeginTag( buffer, start, end );
       current >= 0;
       current = _tryFindBeginTag( buffer, current, end ) ) {
    indri::xml::XMLNode* node;
    std::string tagName;
    std::map<std::string, std::string> attributes;
    bool tagsBetween;

    int endLevel;
    int endTag = _readTag( buffer, current, end, &tagName, &attributes, &tagType );

    if( tagType == TAG_CLOSE_TYPE )
      LEMUR_THROW( LEMUR_GENERIC_ERROR, "Found a close tag for '" + tagName + "' while looking for an open tag." );

    if( tagType == TAG_OPEN_TYPE ) {
      int closingTag = _findClosingTag( buffer, endTag, end, tagName, &tagsBetween );
      if( closingTag == -1 )
        LEMUR_THROW( LEMUR_GENERIC_ERROR, "Could not find a close tag for '" + tagName + "'");

      if( tagsBetween ) {
        node = new indri::xml::XMLNode( tagName, attributes );
        _read( &node, buffer, endTag, closingTag );
      } else {
        std::string nodeValue;
        nodeValue.assign( &buffer[endTag], &buffer[closingTag] ); 
        std::string::size_type dataStart = nodeValue.find("<!CDATA[");
        while (dataStart != std::string::npos) {          
          // munch any CDATA tags in the element's value.
          nodeValue.erase(dataStart, 8);
          std::string::size_type dataEnd = nodeValue.find("]]>");
          if (dataEnd != std::string::npos) 
            nodeValue.erase(dataEnd, 3);
          // else bad things here, should throw.
          dataStart = nodeValue.find("<!CDATA[");
        }
        node = new indri::xml::XMLNode( tagName, attributes, nodeValue );
      }

      endLevel = _findEndTag( buffer, closingTag, end )+1;
    } else {
      assert( tagType == TAG_OPEN_CLOSE_TYPE );
      node = new indri::xml::XMLNode( tagName, attributes );
      endLevel = endTag;
    }

    if( *parent ) {
      (*parent)->addChild( node );
    } else {
      *parent = node;
      break;
    }

    current = endLevel;
  }
}

int indri::xml::XMLReader::_findNotName( const char* buffer, int start, int finish ) {
  int i;

  for( i=start; i<finish; i++ ) {
    // this isn't unicode-safe, but it should be good for now
#ifndef WIN32
    if( !isalpha(buffer[i]) && 
        !isdigit(buffer[i]) &&
#else
    if( (buffer[i] >= 0 && !isalpha(buffer[i])) && 
        (buffer[i] >= 0 && !isdigit(buffer[i])) &&
#endif
        buffer[i] != '-' &&
        buffer[i] != '_' &&
        buffer[i] != ':' &&
        buffer[i] != '.' ) {
      break;
    }
  }

  if( i==finish )
    LEMUR_THROW( LEMUR_PARSE_ERROR, "Was looking for the end of a tag name, but couldn't find it." );

  return i;
}

int indri::xml::XMLReader::_findText( const char* buffer, int start, int finish ) {
  int result = _tryFindText( buffer, start, finish );
  if( result==finish )
    LEMUR_THROW( LEMUR_GENERIC_ERROR, "Was looking for text, but couldn't find any" );

  return result;
}

static void zlib_deflate( z_stream_s& stream, indri::file::SequentialWriteBuffer* outfile ) {
  if( stream.avail_in == 0 ) {
    // nothing to do...
    return;
  }
  
  if( stream.avail_out == 0 ) {
    stream.next_out = (Bytef*) outfile->write( OUTPUT_BUFFER_SIZE );
    stream.avail_out = OUTPUT_BUFFER_SIZE;
  }

  int result = deflate( &stream, 0 );

  // if we're fine, then just return (common case)
  while( result != Z_OK || stream.avail_in != 0 ) {
    // either we need more space, or an error happened
    if( result != Z_OK ) {
      LEMUR_THROW( LEMUR_IO_ERROR, "Tried to add a document to the collection, but zlib returned an error" );
    }

    // get more space
    stream.next_out = (Bytef*) outfile->write( OUTPUT_BUFFER_SIZE );
    stream.avail_out = OUTPUT_BUFFER_SIZE;
  
    result = deflate( &stream, 0 );
  }
}

lemur::api::DOCID_T indri::api::IndexEnvironment::addString( const std::string& documentString, const std::string& fileClass, const std::vector<indri::parse::MetadataPair>& metadata ) {
  indri::parse::UnparsedDocument document;
  indri::parse::Parser* parser;
  indri::parse::Tokenizer* tokenizer;
  indri::parse::DocumentIterator* iterator;
  indri::parse::Conflater* conflater;
  std::string nothing;

  _documentsSeen++;

  document.text = documentString.c_str();
  document.textLength = documentString.length() + 1; // for the null
  document.metadata = metadata;
  document.content = document.text;
  document.contentLength = document.textLength - 1;
  
  _getParsingContext( &parser, &tokenizer, &iterator, &conflater, fileClass );

  if( parser == 0 ) {
    LEMUR_THROW( LEMUR_RUNTIME_ERROR, "File class '" + fileClass + "' wasn't recognized." );
  }
  indri::parse::TokenizedDocument* tokenized = tokenizer->tokenize( &document );

  ParsedDocument* parsed = parser->parse( tokenized );
  lemur::api::DOCID_T documentID =_repository.addDocument( parsed );

  _documentsIndexed++;
  if( _callback ) (*_callback)( indri::api::IndexStatus::DocumentCount, nothing, _error, _documentsIndexed, _documentsSeen );

  return documentID;
}

void indri::api::Parameters::loadFile( const std::string& filename ) {
  std::ifstream input;
  indri::xml::XMLReader reader;
  
  input.open( filename.c_str(), std::ifstream::in );

  if( input.rdstate() & std::ios::failbit )
    LEMUR_THROW( LEMUR_IO_ERROR, "Couldn't open parameter file '" + filename + "' for reading." );

  input.seekg( 0, std::ios::end );
  size_t length = input.tellg();
  input.seekg( 0, std::ios::beg );
  // null terminate it to make a string in the XML reader for comment strip
  char* buffer = new char[length + 1];
  buffer[length] = '\0';

  try {
    input.read( buffer, length );
    std::auto_ptr<indri::xml::XMLNode> result( reader.read( buffer, length ) );

    _loadXML( result.get() );
  } catch( lemur::api::Exception& e ) {
    LEMUR_RETHROW( e, "Had trouble parsing parameter file '" + filename + "'" );
  }

  delete[] buffer;
  input.close();
}

int indri::collection::Repository::addDocument(indri::api::ParsedDocument* document, bool inCollection) {
  if (_readOnly)
    LEMUR_THROW(LEMUR_RUNTIME_ERROR, "addDocument: Cannot add documents to a repository that is opened for read-only access."); 

  while (_thrashing) {
    indri::thread::Thread::sleep(100);
  }

  indri::thread::ScopedLock lock(_addLock);

  for(size_t i=0; i<_transformations.size(); i++) {
    document = _transformations[i]->transform(document);
  }

  index_state state;

  { 
    // get a copy of current index state
    indri::thread::ScopedLock stateLock(_stateLock);
    state = _active;
  }

  int documentID = dynamic_cast<indri::index::MemoryIndex*>(state->back())->addDocument(*document);
  if (inCollection) _collection->addDocument(documentID, document);

  _countDocumentAdd();
  return documentID;
}

void indri::infnet::InferenceNetwork::_indexChanged( indri::index::Index& index ) {
  _closeIterators.clear();
  _closeIteratorBound = -1;

  // doc iterators
  for( size_t i=0; i<_termNames.size(); i++ ) {
    indri::index::DocListIterator* iterator = index.docListIterator( _termNames[i] );
    if( iterator )
      iterator->startIteration();

    _docIterators.push_back( iterator );
  }

  // field iterators
  for( size_t i=0; i<_fieldNames.size(); i++ ) {
    indri::index::DocExtentListIterator* iterator = index.fieldListIterator( _fieldNames[i] );
    if( iterator )
      iterator->startIteration();

    _fieldIterators.push_back( iterator );
  }
  
  // prior iterators
  for( size_t i=0; i<_priorNames.size(); i++ ) {
    // TODO: this is wasteful, since the prior is associated with the whole collection,
    // there's no need to fetch it for each index.  but, it's just easier to code it like this for now
    indri::collection::PriorListIterator* iterator = _repository.priorListIterator( _priorNames[i] );
    if( iterator )
      iterator->startIteration();
    else {
      // if the named prior doesn't exist in the Repository, throw an Exception
      LEMUR_THROW( LEMUR_RUNTIME_ERROR, "named prior: " + _priorNames[i] + " not found in Repository. Unable to process query." );
    }

    _priorIterators.push_back( iterator );
  }

  // extent iterator nodes
  std::vector<ListIteratorNode*>::iterator diter;
  for( diter = _listIteratorNodes.begin(); diter != _listIteratorNodes.end(); diter++ ) {
    (*diter)->indexChanged( index );
  }

  // belief nodes
  std::vector<BeliefNode*>::iterator biter;
  for( biter = _beliefNodes.begin(); biter != _beliefNodes.end(); biter++ ) {
    (*biter)->indexChanged( index );
  }

  // evaluator nodes
  std::vector<indri::infnet::EvaluatorNode*>::iterator eiter;
  for( eiter = _evaluators.begin(); eiter != _evaluators.end(); eiter++ ) {
    (*eiter)->indexChanged( index );
  }

  // document structure
  if (_documentStructureHolderNode != 0) {
    _documentStructureHolderNode->indexChanged( index );
  }
}

indri::parse::DocumentIterator* indri::parse::DocumentIteratorFactory::get( const std::string& type, const char* startDocTag, const char* endDocTag, const char* startMetadataTag ) {
  std::string preferred = preferredName( type );
  indri::parse::DocumentIterator* result = 0;

  if( preferred == TYPE_TAGGED ) {
    indri::parse::TaggedDocumentIterator* iter = new indri::parse::TaggedDocumentIterator();
    iter->setTags( startDocTag, endDocTag, startMetadataTag );
    result = iter;
  } else if( preferred == TYPE_WARC ) {
    result = new indri::parse::WARCDocumentIterator();
  } else if( preferred == TYPE_PDF ) {
    result = new indri::parse::PDFDocumentExtractor();
  } else if( preferred == TYPE_TEXT ) {
    result = new indri::parse::TextDocumentExtractor();
  } else if( preferred == TYPE_MBOX ) {
    result = new indri::parse::MboxDocumentIterator();
  }
#ifdef WIN32
  else if( preferred == TYPE_WORD ) {
    result = new indri::parse::WordDocumentExtractor();
  } else if( preferred == TYPE_PPT ) {
    result = new indri::parse::PowerPointDocumentExtractor();
  }
#endif

  if( !result )
    LEMUR_THROW( LEMUR_RUNTIME_ERROR, type + " is an unknown DocumentIterator type." );

  return result;
}

void indri::parse::TextDocumentExtractor::open( const std::string& filename ) {
  _in = gzopen( filename.c_str(), "rb" );
  _filename = filename;

  if( !_in )
    LEMUR_THROW( LEMUR_IO_ERROR, "Couldn't open file " + filename + "." );
}

static void zlib_read_document( z_stream_s& stream, indri::file::File& infile, UINT64 offset, indri::utility::Buffer& outputBuffer ) {
  // read in data from the file until the stream ends
  // split up the data as necessary
  // decompress positional info

  // read some data
  char inputBuffer[INPUT_BUFFER_SIZE];
  outputBuffer.grow( INPUT_BUFFER_SIZE );
  outputBuffer.write( sizeof(indri::api::ParsedDocument) );

  stream.avail_in = 0;
  stream.avail_out = 0;
  
  while(true) {
    if( !stream.avail_in ) {
      UINT64 readSize = infile.read( inputBuffer, offset, sizeof inputBuffer );
      offset += readSize; 
      
      stream.avail_in = readSize;
      stream.next_in = (Bytef*) inputBuffer;
    }

    stream.avail_out = outputBuffer.size() - outputBuffer.position();
    stream.next_out = (Bytef*) outputBuffer.write( outputBuffer.size() - outputBuffer.position() );

    int result = inflate( &stream, Z_NO_FLUSH );
    outputBuffer.unwrite( stream.avail_out );

    if( result == Z_STREAM_END ) {
      result = inflate( &stream, Z_FINISH );
      
      if( result < 0 )
        LEMUR_THROW( result, "Something bad happened while trying to finish decompressing a document." );

      inflateEnd( &stream );
      break;
    }

    if( result < 0 ) {
      LEMUR_THROW( result, "Something bad happened while trying to decompress a document." );
    }

    if( stream.avail_out == 0 ) {
      outputBuffer.grow();
    }
  }
}

void indri::parse::MboxDocumentIterator::open( const std::string& filename ) {
  _in.clear();
  _in.open( filename.c_str() );
  _filename = filename;

  if( !_in.good() )
    LEMUR_THROW( LEMUR_IO_ERROR, "Couldn't open file " + filename + "." );
}

void indri::net::XMLReplyReceiver::wait( indri::net::NetworkMessageStream* stream ) {
  while( !done() && stream->alive() && !_exception.size() ) {
    stream->read(*this);
  }
  
  if( _exception.size() )
    LEMUR_THROW( LEMUR_NETWORK_ERROR, _exception );
}

int indri::xml::XMLReader::_findChar( char ch, const char* buffer, int start, int finish ) {
  int result = _tryFindChar( ch, buffer, start, finish );

  if( result == -1 )
    LEMUR_THROW( LEMUR_PARSE_ERROR, "Was looking for '" + ch + "', but couldn't find it." );

  return result;
}

      indri::infnet::InferenceNetwork::MAllResults& getResults() {
        while( !_done && _stream->alive() && !_exception.length() )
          _stream->read(*this);

        if( _exception.length() )
          LEMUR_THROW( LEMUR_RUNTIME_ERROR, _exception );
    
        return _results;
      }

void lemur::file::Keyfile::create( const char* filename, int cacheSize ) {
  _buildHandle( cacheSize );
  
  int error = create_key( _handle, const_cast<char*>(filename), 
                          _handleSize );

  if( error )
    LEMUR_THROW(LEMUR_KEYFILE_IO_ERROR, "Unable to create '" + filename + "'");
}

void lemur::file::Keyfile::remove( const char* key ) {
  assert( key && "key cannot be null" );
  assert( _handle && "call open() or create() first" );
  int len = strlen(key); // fix for UTF-8
  int error = delete_rec( _handle, const_cast<char*>(key), len );

  if( error )
    LEMUR_THROW( LEMUR_KEYFILE_IO_ERROR, "Unable to delete record for key: " + key );
}