/* Closes all resouces and writes the entry table */
func (w *CompoundFileWriter) Close() (err error) {
	if w.closed {
		fmt.Println("CompoundFileWriter is already closed.")
		return nil
	}

	// TODO this code should clean up after itself (remove partial .cfs/.cfe)
	if err = func() (err error) {
		var success = false
		defer func() {
			if success {
				util.Close(w.dataOut)
			} else {
				util.CloseWhileSuppressingError(w.dataOut)
			}
		}()

		assert2(w.pendingEntries.Len() == 0 && !w.outputTaken.Get(),
			"CFS has pending open files")
		w.closed = true
		// open the compound stream; we can safely use IO_CONTEXT_DEFAULT
		// here because this will only open the output if no file was
		// added to the CFS
		_, err = w.output(IO_CONTEXT_DEFAULT)
		if err != nil {
			return
		}
		assert(w.dataOut != nil)
		err = codec.WriteFooter(w.dataOut)
		if err != nil {
			return
		}
		success = true
		return nil
	}(); err != nil {
		return
	}

	var entryTableOut IndexOutput
	var success = false
	defer func() {
		if success {
			util.Close(entryTableOut)
		} else {
			util.CloseWhileSuppressingError(entryTableOut)
		}
	}()
	entryTableOut, err = w.directory.CreateOutput(w.entryTableName, IO_CONTEXT_DEFAULT)
	if err != nil {
		return
	}
	err = w.writeEntryTable(w.entries, entryTableOut)
	if err != nil {
		return
	}
	success = true
	return
}

func (f *Lucene41PostingsFormat) FieldsProducer(state SegmentReadState) (FieldsProducer, error) {
	postingsReader, err := NewLucene41PostingsReader(state.dir,
		state.fieldInfos,
		state.segmentInfo,
		state.context,
		state.segmentSuffix)
	if err != nil {
		return nil, err
	}
	success := false
	defer func() {
		if !success {
			log.Printf("Failed to load FieldsProducer for %v.", f.Name())
			if err != nil {
				log.Print("DEBUG ", err)
			}
			util.CloseWhileSuppressingError(postingsReader)
		}
	}()

	fp, err := newBlockTreeTermsReader(state.dir,
		state.fieldInfos,
		state.segmentInfo,
		postingsReader,
		state.context,
		state.segmentSuffix,
		state.termsIndexDivisor)
	if err != nil {
		log.Print("DEBUG: ", err)
		return fp, err
	}
	success = true
	return fp, nil
}

func NewCompoundFileDirectory(directory Directory, fileName string, context IOContext, openForWrite bool) (d *CompoundFileDirectory, err error) {
	self := &CompoundFileDirectory{
		Locker:         &sync.Mutex{},
		directory:      directory,
		fileName:       fileName,
		readBufferSize: bufferSize(context),
		openForWrite:   openForWrite}
	self.DirectoryImpl = NewDirectoryImpl(self)

	if !openForWrite {
		log.Printf("Open for read.")
		success := false
		defer func() {
			if !success {
				util.CloseWhileSuppressingError(self.handle)
			}
		}()
		self.handle, err = directory.CreateSlicer(fileName, context)
		if err != nil {
			return self, err
		}
		self.entries, err = readEntries(self.handle, directory, fileName)
		if err != nil {
			return self, err
		}
		success = true
		self.DirectoryImpl.IsOpen = true
		return self, err
	} else {
		panic("not supported yet")
	}
}

func (nc *NormsConsumer) Close() (err error) {
	var success = false
	defer func() {
		if success {
			err = util.Close(nc.data, nc.meta)
		} else {
			util.CloseWhileSuppressingError(nc.data, nc.meta)
		}
	}()

	if nc.meta != nil {
		if err = nc.meta.WriteVInt(-1); err != nil { // write EOF marker
			return
		}
		if err = codec.WriteFooter(nc.meta); err != nil { // write checksum
			return
		}
	}
	if nc.data != nil {
		if err = codec.WriteFooter(nc.data); err != nil { // write checksum
			return
		}
	}
	success = true
	return nil
}

/*
Copies the file src to 'to' under the new file name dest.

If you want to copy the entire source directory to the destination
one, you can do so like this:

		var to Directory // the directory to copy to
		for _, file := range dir.ListAll() {
			dir.Copy(to, file, newFile, IO_CONTEXT_DEFAULT)
			// newFile can be either file, or a new name
		}

NOTE: this method does not check whether dest exists and will
overwrite it if it does.
*/
func (d *DirectoryImpl) Copy(to Directory, src, dest string, ctx IOContext) (err error) {
	var os IndexOutput
	var is IndexInput
	var success = false
	defer func() {
		if success {
			err = util.Close(os, is)
		} else {
			util.CloseWhileSuppressingError(os, is)
		}
		defer func() {
			recover() // ignore panic
		}()
		to.DeleteFile(dest) // ignore error
	}()

	os, err = to.CreateOutput(dest, ctx)
	if err != nil {
		return err
	}
	is, err = d.spi.OpenInput(src, ctx)
	if err != nil {
		return err
	}
	err = os.CopyBytes(is, is.Length())
	if err != nil {
		return err
	}
	success = true
	return nil
}

func openStandardDirectoryReader(directory store.Directory,
	commit IndexCommit, termInfosIndexDivisor int) (r DirectoryReader, err error) {
	// log.Print("Initializing SegmentsFile...")
	obj, err := NewFindSegmentsFile(directory, func(segmentFileName string) (interface{}, error) {
		sis := &SegmentInfos{}
		err := sis.Read(directory, segmentFileName)
		if err != nil {
			return nil, err
		}
		// log.Printf("Found %v segments...", len(sis.Segments))
		readers := make([]AtomicReader, len(sis.Segments))
		for i := len(sis.Segments) - 1; i >= 0; i-- {
			sr, err := NewSegmentReader(sis.Segments[i], termInfosIndexDivisor, store.IO_CONTEXT_READ)
			if err != nil {
				for _, r := range readers {
					if r != nil {
						util.CloseWhileSuppressingError(r)
					}
				}
				return nil, err
			}
			readers[i] = sr
		}
		// log.Printf("Obtained %v SegmentReaders.", len(readers))
		return newStandardDirectoryReader(directory, readers, sis, termInfosIndexDivisor, false), nil
	}).run(commit)
	if err != nil {
		return nil, err
	}
	return obj.(*StandardDirectoryReader), err
}

func (w *CompressingStoredFieldsWriter) Abort() {
	assert(w != nil)
	util.CloseWhileSuppressingError(w)
	util.DeleteFilesIgnoringErrors(w.directory,
		util.SegmentFileName(w.segment, w.segmentSuffix, lucene40.FIELDS_EXTENSION),
		util.SegmentFileName(w.segment, w.segmentSuffix, lucene40.FIELDS_INDEX_EXTENSION))
}

func (f *Lucene41PostingsFormat) FieldsProducer(state SegmentReadState) (FieldsProducer, error) {
	postingsReader, err := NewLucene41PostingsReader(state.Dir,
		state.FieldInfos,
		state.SegmentInfo,
		state.Context,
		state.SegmentSuffix)
	if err != nil {
		return nil, err
	}
	success := false
	defer func() {
		if !success {
			fmt.Printf("Failed to load FieldsProducer for %v.\n", f.Name())
			util.CloseWhileSuppressingError(postingsReader)
		}
	}()

	fp, err := blocktree.NewBlockTreeTermsReader(state.Dir,
		state.FieldInfos,
		state.SegmentInfo,
		postingsReader,
		state.Context,
		state.SegmentSuffix,
		state.TermsIndexDivisor)
	if err != nil {
		return fp, err
	}
	success = true
	return fp, nil
}

func (w *Lucene41PostingsWriter) Close() (err error) {
	var success = false
	defer func() {
		if success {
			err = util.Close(w.docOut, w.posOut, w.payOut)
		} else {
			util.CloseWhileSuppressingError(w.docOut, w.posOut, w.payOut)
		}
		w.docOut = nil
		w.posOut = nil
		w.payOut = nil
	}()

	if err == nil && w.docOut != nil {
		err = codec.WriteFooter(w.docOut)
	}
	if err == nil && w.posOut != nil {
		err = codec.WriteFooter(w.posOut)
	}
	if err == nil && w.payOut != nil {
		err = codec.WriteFooter(w.payOut)
	}
	if err != nil {
		return
	}
	success = true
	return nil
}

func (p *StoredFieldsProcessor) flush(state SegmentWriteState) (err error) {
	numDocs := state.segmentInfo.DocCount()
	if numDocs > 0 {
		// It's possible that all documents seen in this segment hit
		// non-aborting errors, in which case we will not have yet init'd
		// the FieldsWriter:
		err = p.initFieldsWriter(state.context)
		if err == nil {
			err = p.fill(numDocs)
		}
	}
	if w := p.fieldsWriter; w != nil {
		var success = false
		defer func() {
			if success {
				err = util.CloseWhileHandlingError(err, w)
			} else {
				util.CloseWhileSuppressingError(w)
			}
		}()

		err = w.Finish(state.fieldInfos, numDocs)
		if err != nil {
			return err
		}
		success = true
	}
	return
}

func newLucene49NormsConsumer(state *SegmentWriteState,
	dataCodec, dataExtension, metaCodec, metaExtension string) (nc *NormsConsumer, err error) {

	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(1))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(2))
	assert(packed.PackedFormat(packed.PACKED_SINGLE_BLOCK).IsSupported(4))

	nc = &NormsConsumer{maxDoc: state.SegmentInfo.DocCount()}
	var success = false
	defer func() {
		if !success {
			util.CloseWhileSuppressingError(nc)
		}
	}()

	dataName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension)
	if nc.data, err = state.Directory.CreateOutput(dataName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.data, dataCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	metaName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension)
	if nc.meta, err = state.Directory.CreateOutput(metaName, state.Context); err != nil {
		return nil, err
	}
	if err = codec.WriteHeader(nc.meta, metaCodec, VERSION_CURRENT); err != nil {
		return nil, err
	}
	success = true
	return nc, nil
}

func (w *CompressingStoredFieldsWriter) Abort() {
	if w == nil { // tolerate early released pointer
		return
	}
	util.CloseWhileSuppressingError(w)
	util.DeleteFilesIgnoringErrors(w.directory,
		util.SegmentFileName(w.segment, w.segmentSuffix, lucene40.FIELDS_EXTENSION),
		util.SegmentFileName(w.segment, w.segmentSuffix, lucene40.FIELDS_INDEX_EXTENSION))
}

func (w *BlockTreeTermsWriter) Close() (err error) {
	var success = false
	defer func() {
		if success {
			util.Close(w.out, w.indexOut, w.postingsWriter)
		} else {
			util.CloseWhileSuppressingError(w.out, w.indexOut, w.postingsWriter)
		}
	}()

	dirStart := w.out.FilePointer()
	indexDirStart := w.indexOut.FilePointer()

	if err = w.out.WriteVInt(int32(len(w.fields))); err != nil {
		return
	}

	for _, field := range w.fields {
		// fmt.Printf("  field %v %v terms\n", field.fieldInfo.Name, field.numTerms)
		if err = w.out.WriteVInt(field.fieldInfo.Number); err == nil {
			assert(field.numTerms > 0)
			if err = w.out.WriteVLong(field.numTerms); err == nil {
				if err = w.out.WriteVInt(int32(len(field.rootCode))); err == nil {
					err = w.out.WriteBytes(field.rootCode)
					if err == nil && field.fieldInfo.IndexOptions() != INDEX_OPT_DOCS_ONLY {
						err = w.out.WriteVLong(field.sumTotalTermFreq)
					}
					if err == nil {
						if err = w.out.WriteVLong(field.sumDocFreq); err == nil {
							if err = w.out.WriteVInt(int32(field.docCount)); err == nil {
								if err = w.out.WriteVInt(int32(field.longsSize)); err == nil {
									if err = w.indexOut.WriteVLong(field.indexStartFP); err == nil {
										if err = writeBytesRef(w.out, field.minTerm); err == nil {
											err = writeBytesRef(w.out, field.maxTerm)
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
	if err == nil {
		if err = w.writeTrailer(w.out, dirStart); err == nil {
			if err = codec.WriteFooter(w.out); err == nil {
				if err = w.writeIndexTrailer(w.indexOut, indexDirStart); err == nil {
					if err = codec.WriteFooter(w.indexOut); err == nil {
						success = true
					}
				}
			}
		}
	}
	return
}

func newPerFieldPostingsReader(state SegmentReadState) (fp FieldsProducer, err error) {
	ans := PerFieldPostingsReader{
		make(map[string]FieldsProducer),
		make(map[string]FieldsProducer),
	}
	// Read _X.per and init each format:
	success := false
	defer func() {
		if !success {
			// log.Printf("Failed to initialize PerFieldPostingsReader.")
			// if err != nil {
			// 	log.Print("DEBUG ", err)
			// }
			fps := make([]FieldsProducer, 0)
			for _, v := range ans.formats {
				fps = append(fps, v)
			}
			items := make([]io.Closer, len(fps))
			for i, v := range fps {
				items[i] = v
			}
			util.CloseWhileSuppressingError(items...)
		}
	}()
	// Read field name -> format name
	for _, fi := range state.FieldInfos.Values {
		// log.Printf("Processing %v...", fi)
		if fi.IsIndexed() {
			fieldName := fi.Name
			// log.Printf("Name: %v", fieldName)
			if formatName := fi.Attribute(PER_FIELD_FORMAT_KEY); formatName != "" {
				// log.Printf("Format: %v", formatName)
				// null formatName means the field is in fieldInfos, but has no postings!
				suffix := fi.Attribute(PER_FIELD_SUFFIX_KEY)
				// log.Printf("Suffix: %v", suffix)
				assert(suffix != "")
				format := LoadPostingsFormat(formatName)
				segmentSuffix := formatName + "_" + suffix
				// log.Printf("Segment suffix: %v", segmentSuffix)
				if _, ok := ans.formats[segmentSuffix]; !ok {
					// log.Printf("Loading fields producer: %v", segmentSuffix)
					newReadState := state // clone
					newReadState.SegmentSuffix = formatName + "_" + suffix
					fp, err = format.FieldsProducer(newReadState)
					if err != nil {
						return fp, err
					}
					ans.formats[segmentSuffix] = fp
				}
				ans.fields[fieldName] = ans.formats[segmentSuffix]
			}
		}
	}
	success = true
	return &ans, nil
}

func NewLucene41PostingsReader(dir store.Directory,
	fis model.FieldInfos, si *model.SegmentInfo,
	ctx store.IOContext, segmentSuffix string) (r PostingsReaderBase, err error) {

	log.Print("Initializing Lucene41PostingsReader...")
	success := false
	var docIn, posIn, payIn store.IndexInput = nil, nil, nil
	defer func() {
		if !success {
			log.Print("Failed to initialize Lucene41PostingsReader.")
			if err != nil {
				log.Print("DEBUG ", err)
			}
			util.CloseWhileSuppressingError(docIn, posIn, payIn)
		}
	}()

	docIn, err = dir.OpenInput(util.SegmentFileName(si.Name, segmentSuffix, LUCENE41_DOC_EXTENSION), ctx)
	if err != nil {
		return r, err
	}
	_, err = codec.CheckHeader(docIn, LUCENE41_DOC_CODEC, LUCENE41_VERSION_CURRENT, LUCENE41_VERSION_CURRENT)
	if err != nil {
		return r, err
	}
	forUtil, err := NewForUtil(docIn)
	if err != nil {
		return r, err
	}

	if fis.HasProx {
		posIn, err = dir.OpenInput(util.SegmentFileName(si.Name, segmentSuffix, LUCENE41_POS_EXTENSION), ctx)
		if err != nil {
			return r, err
		}
		_, err = codec.CheckHeader(posIn, LUCENE41_POS_CODEC, LUCENE41_VERSION_CURRENT, LUCENE41_VERSION_CURRENT)
		if err != nil {
			return r, err
		}

		if fis.HasPayloads || fis.HasOffsets {
			payIn, err = dir.OpenInput(util.SegmentFileName(si.Name, segmentSuffix, LUCENE41_PAY_EXTENSION), ctx)
			if err != nil {
				return r, err
			}
			_, err = codec.CheckHeader(payIn, LUCENE41_PAY_CODEC, LUCENE41_VERSION_CURRENT, LUCENE41_VERSION_CURRENT)
			if err != nil {
				return r, err
			}
		}
	}

	success = true
	return &Lucene41PostingsReader{docIn, posIn, payIn, forUtil}, nil
}

func NewCompoundFileDirectory(directory Directory, fileName string, context IOContext, openForWrite bool) (d *CompoundFileDirectory, err error) {
	self := &CompoundFileDirectory{
		Locker:         &sync.Mutex{},
		directory:      directory,
		fileName:       fileName,
		readBufferSize: bufferSize(context),
		openForWrite:   openForWrite}
	self.DirectoryImpl = NewDirectoryImpl(self)
	self.BaseDirectory = NewBaseDirectory(self)

	if !openForWrite {
		// log.Printf("Open for read.")
		success := false
		defer func() {
			if !success {
				util.CloseWhileSuppressingError(self.handle)
			}
		}()
		self.handle, err = directory.OpenInput(fileName, context)
		if err != nil {
			return nil, err
		}
		self.entries, err = self.readEntries(self.handle, directory, fileName)
		if err != nil {
			return nil, err
		}
		if self.version >= CFD_VERSION_CHECKSUM {
			if _, err = codec.CheckHeader(self.handle, CFD_DATA_CODEC,
				int32(self.version), int32(self.version)); err != nil {
				return nil, err
			}
			// NOTE: data file is too costly to verify checksum against all the
			// bytes on open, but for now we at least verify proper structure
			// of the checksum footer: which looks for FOOTER_MAGIC +
			// algorithmID. This is cheap and can detect some forms of
			// corruption such as file trucation.
			if _, err = codec.RetrieveChecksum(self.handle); err != nil {
				return nil, err
			}
		}
		success = true
		self.BaseDirectory.IsOpen = true
		return self, nil
	} else {
		assert2(reflect.TypeOf(directory).Name() != "CompoundFileDirectory",
			"compound file inside of compound file: %v", fileName)
		self.entries = SENTINEL
		self.IsOpen = true
		self.writer = newCompoundFileWriter(directory, fileName)
		self.handle = nil
		return self, nil
	}
}

func (sis *SegmentInfos) rollbackCommit(dir store.Directory) {
	if sis.pendingSegnOutput != nil {
		// Suppress so we keep throwing the original error in our caller
		util.CloseWhileSuppressingError(sis.pendingSegnOutput)
		sis.pendingSegnOutput = nil

		// Must carefully compute filename from "generation" since
		// lastGeneration isn't incremented:
		segmentFilename := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)

		// Suppress so we keep throwing the original error in our caller
		util.DeleteFilesIgnoringErrors(dir, segmentFilename)
	}
}

func newLucene42DocValuesProducer(state SegmentReadState,
	dataCodec, dataExtension, metaCodec, metaExtension string) (dvp *Lucene42DocValuesProducer, err error) {
	dvp = &Lucene42DocValuesProducer{
		numericInstances: make(map[int]NumericDocValues),
	}
	dvp.maxDoc = state.segmentInfo.DocCount()
	metaName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, metaExtension)
	// read in the entries from the metadata file.
	in, err := state.dir.OpenInput(metaName, state.context)
	if err != nil {
		return dvp, err
	}
	success := false
	defer func() {
		if success {
			err = util.Close(in)
		} else {
			util.CloseWhileSuppressingError(in)
		}
	}()

	version, err := codec.CheckHeader(in, metaCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT)
	if err != nil {
		return dvp, err
	}
	dvp.numerics = make(map[int]NumericEntry)
	dvp.binaries = make(map[int]BinaryEntry)
	dvp.fsts = make(map[int]FSTEntry)
	err = dvp.readFields(in)
	if err != nil {
		return dvp, err
	}
	success = true

	success = false
	dataName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, dataExtension)
	dvp.data, err = state.dir.OpenInput(dataName, state.context)
	if err != nil {
		return dvp, err
	}
	version2, err := codec.CheckHeader(dvp.data, dataCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT)
	if err != nil {
		return dvp, err
	}

	if version != version2 {
		return dvp, errors.New("Format versions mismatch")
	}
	return dvp, nil
}

func (f *Lucene41PostingsFormat) FieldsConsumer(state *SegmentWriteState) (FieldsConsumer, error) {
	postingsWriter, err := newLucene41PostingsWriterCompact(state)
	if err != nil {
		return nil, err
	}
	var success = false
	defer func() {
		if !success {
			util.CloseWhileSuppressingError(postingsWriter)
		}
	}()
	ret, err := blocktree.NewBlockTreeTermsWriter(state, postingsWriter, f.minTermBlockSize, f.maxTermBlockSize)
	if err != nil {
		return nil, err
	}
	success = true
	return ret, nil
}

func (w *Lucene40SegmentInfoWriter) Write(dir store.Directory,
	si *SegmentInfo, fis FieldInfos, ctx store.IOContext) (err error) {

	filename := util.SegmentFileName(si.Name, "", LUCENE40_SI_EXTENSION)
	si.AddFile(filename)

	var output store.IndexOutput
	output, err = dir.CreateOutput(filename, ctx)
	if err != nil {
		return err
	}

	var success = false
	defer func() {
		if !success {
			util.CloseWhileSuppressingError(output)
			si.Dir.DeleteFile(filename) // ignore error
		} else {
			err = mergeError(err, output.Close())
		}
	}()

	err = codec.WriteHeader(output, LUCENE40_CODEC_NAME, LUCENE40_VERSION_CURRENT)
	if err != nil {
		return err
	}
	// Write the Lucene version that created this segment, since 3.1
	err = store.Stream(output).WriteString(si.Version().String()).
		WriteInt(int32(si.DocCount())).
		WriteByte(func() byte {
			if si.IsCompoundFile() {
				return SEGMENT_INFO_YES
			}
			return byte((SEGMENT_INFO_NO + 256) % 256) // Go byte is non-negative, unlike Java
		}()).WriteStringStringMap(si.Diagnostics()).
		WriteStringStringMap(map[string]string{}).
		WriteStringSet(si.Files()).Close()
	if err != nil {
		return err
	}

	success = true
	return nil
}