import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Returns an array containing the non-zero indices of this sparse vector.
 * 
 * @return an integer array containing the non-zero indices of this sparse vector
 */
public int[] getNonZeroIndices() {
   if (isChanged) {
      List<Integer> indicesWithNonZeroEntry = new ArrayList<>(length());
      List<Double> nonZeroEntries = new ArrayList<>(length());
      for (int i = 0; i < length(); i++) {
         double value = internalVector.get(i);
         if (Double.compare(value, 0.0) != 0) {
            indicesWithNonZeroEntry.add(i);
            nonZeroEntries.add(value);
         }
      }
      // do we need to recopy?
      if (indicesWithNonZeroEntry.size() != internalVector.getIndex().length) {
         this.internalVector = new SparseVector(indicesWithNonZeroEntry.size(),
               CollectionsUtils.convertIntegerListToArray(indicesWithNonZeroEntry),
               CollectionsUtils.convertDoubleListToArray(nonZeroEntries));
      }
      setUnchanged();
   }
   return internalVector.getIndex();
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Saving sparse word embeddings.
 * 
 * @param fieldIndex
 *          in-memory forward index
 * @param fileName
 *          output file name
 * @param dict
 *          the model itself in the form of a mapping wordId => embedding
 * @param maxDigit
 *          the maximum # of digits to print
 * @throws IOException
 */
public static void saveDict(InMemForwardIndex           fieldIndex,
                            String                      fileName, 
                            HashIntObjMap<SparseVector> dict,
                            int                         maxDigit) throws IOException {
  BufferedWriter out = new BufferedWriter(
                            new OutputStreamWriter(CompressUtils.createOutputStream(fileName)));
  
  String outFormat= " %d:%." + maxDigit + "e";
  
  for (Entry<Integer, SparseVector> e: dict.entrySet()) {
    out.write(fieldIndex.getWord(e.getKey()));
    for (VectorEntry ve : e.getValue()) {
      Formatter f = new Formatter();
      f.format(outFormat, ve.index(), ve.get());
      out.write(f.toString());
      f.close();
    }
    out.newLine();
  }
  
  out.close();    
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
void initHighorderModels(int fieldId, InMemIndexFeatureExtractor ... donorExtractors) throws Exception {
  // First try to reuse donor's models
  for (int donorId = 0; donorId < donorExtractors.length; donorId++) {
    InMemIndexFeatureExtractor donnor = donorExtractors[donorId];
    if (null == donnor) continue;
    if (null == mHighOrderModels.get(fieldId)) 
      mHighOrderModels.set(fieldId, donnor.mHighOrderModels.get(fieldId));
  }
  // If a donor doesn't have models, create new models from scratch
  if (mHighOrderModels.get(fieldId) == null) {
    initFieldIndex(fieldId, donorExtractors);            
    
    mHighOrderModels.set(fieldId, new ArrayList<HashIntObjMap<SparseVector>>());          
    
    for (int k = 0; k < mHighOrderModelFiles.length; ++k) {
      String fileName = mHighOrderModelFiles[k];
      mHighOrderModels.get(fieldId).add(SparseEmbeddingReaderAndRecorder.readDict(mFieldIndex[fieldId], mEmbedDir + "/" + fileName));
      logger.info("Read ebmedding file: " + fileName);
    }
  }

}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
@Override
public IVector solve(IVector b) {
   Vector bInternal = b.toSparseVector().internalVector;
   SparseVector result = (SparseVector) bInternal.copy();
   internalMatrix.solve(bInternal, result);
   return new SparseDoubleVector(result);
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Creates a new SparseDoubleVector which contains the given values.
 * 
 * @param asArray the vector in the form on a double array
 */
public SparseDoubleVector(double[] asArray) {
   List<Integer> indicesWithNonZeroEntry = new ArrayList<>();
   List<Double> nonZeroEntries = new ArrayList<>();
   for (int i = 0; i < asArray.length; i++) {
      if (Double.compare(asArray[i], 0.0) != 0) {
         indicesWithNonZeroEntry.add(i);
         nonZeroEntries.add(asArray[i]);
      }
   }

   internalVector = new SparseVector(asArray.length, CollectionsUtils.convertIntegerListToArray(indicesWithNonZeroEntry),
         CollectionsUtils.convertDoubleListToArray(nonZeroEntries));
   setIsChanged();
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
@Override
public void multiplyByVectorPairwise(double[] vectorAsArray) {
   setIsChanged();
   SparseVector vector = internalVector;
   int[] indexes = vector.getIndex();
   for (int i = 0; i < indexes.length; i++) {
      vector.set(indexes[i], vector.get(indexes[i]) * vectorAsArray[indexes[i]]);
   }
   internalVector = vector;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
@Override
public void divideByVectorPairwise(double[] vectorAsArray) {
   setIsChanged();
   SparseVector vector = internalVector;
   int[] indexes = vector.getIndex();
   for (int i = 0; i < indexes.length; i++) {
      vector.set(indexes[i], vector.get(indexes[i]) / vectorAsArray[indexes[i]]);
   }
   internalVector = vector;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public TranBasedWordEbmeddings(SparseVector embedVector, 
                               SparseVector embedVectorTFxIDF,
                               SparseVector embedVectorTFProb) {
  mEmbedVector        = embedVector;
  mEmbedVectorTFxIDF  = embedVectorTFxIDF;
  mEmbedVectorTFProb  = embedVectorTFProb;
  mEmbedVectorNorm       = Math.max(mEmbedVector.norm(Norm.Two), Float.MIN_NORMAL);
  mEmbedVectorL1Norm     = Math.max(mEmbedVector.norm(Norm.One), Float.MIN_NORMAL);
  mEmbedVectorTFxIDFNorm = Math.max(mEmbedVectorTFxIDF.norm(Norm.Two), Float.MIN_NORMAL);
  mEmbedVectorTFProbNorm = Math.max(mEmbedVectorTFProb.norm(Norm.Two), Float.MIN_NORMAL);
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public static HashIntObjMap<SparseVector> readDict(InMemForwardIndex    fieldIndex,
                                                   String               fileName) throws Exception {
  BufferedReader in = new BufferedReader(
                              new InputStreamReader(CompressUtils.createInputStream(fileName)));
  
  HashIntObjMap<SparseVector> res = HashIntObjMaps.<SparseVector>newMutableMap(fieldIndex.getMaxWordId() + 1);
  
  String line;
  int lineNum = 0;
  while ((line = in.readLine()) != null) {
    ++lineNum;
    line = line.trim();
    if (line.isEmpty()) continue;
    String parts[] = line.split("\\s+");
    if (parts.length < 1) {
      throw new Exception("Bug: not enough elements in line: " + lineNum + " file: '" + fileName + "'");
    }
    String word = parts[0];
    WordEntry e = fieldIndex.getWordEntry(word);
    if (e == null) {
      throw new Exception("The word embedding was computed for a different collection, we encountered an unknown word '" + word + 
                          "' :" + lineNum + " file: '" + fileName + "'");
    }
    int     index[] = new int[parts.length-1];
    double  data [] = new double[parts.length-1];
    for (int i = 1; i < parts.length; ++i) {
      String tmp[] = parts[i].split(":");
      if (tmp.length != 2) {
        throw new Exception("Wrong format of the part '" + parts[i] + "' in line: " + lineNum + " file: '" + fileName + "'");
      }
      index[i-1]=Integer.parseInt(tmp[0]);
      data [i-1]=Float.parseFloat(tmp[1]);
    }
    res.put(e.mWordId, new SparseVector(fieldIndex.getMaxWordId() + 1, index, data, false));
  }
  
  return res;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public static HashIntObjMap<SparseVector> createTranVecDict(InMemForwardIndex                 fieldIndex,
                                                            FrequentIndexWordFilterAndRecoder filter, 
                                                            float                             minProb, 
                                                            GizaTranTableReaderAndRecoder     answToQuestTran) throws Exception {
  HashIntObjMap<SparseVector> res = HashIntObjMaps.<SparseVector>newMutableMap(fieldIndex.getMaxWordId() + 1);
  
  for (int srcWordId : fieldIndex.getAllWordIds()) 
  if (filter.checkWordId(srcWordId)) {
    res.put(srcWordId, createOneTranVector(fieldIndex, minProb, answToQuestTran, srcWordId));
  }
  
  return res;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public static HashIntObjMap<SparseVector> nextOrderDict(HashIntObjMap<SparseVector>   dict, 
                                                        InMemForwardIndex             fieldIndex,
                                                        float                         minProb, 
                                                        GizaTranTableReaderAndRecoder answToQuestTran) throws Exception {
  HashIntObjMap<SparseVector> res = HashIntObjMaps.<SparseVector>newMutableMap(fieldIndex.getMaxWordId() + 1);
  
  for (int srcWordId : dict.keySet()) {
    SparseVector    newVal = new SparseVector(fieldIndex.getMaxWordId() + 1); 
    GizaOneWordTranRecs tranRecs = answToQuestTran.getTranProbs(srcWordId);                 
    if (null != tranRecs) {                
      for (int k = 0; k < tranRecs.mDstIds.length; ++k) {
        int     dstWordId = tranRecs.mDstIds[k];
        double  prob      = tranRecs.mProbs[k];
        SparseVector v = dict.get(dstWordId).copy();
        if (v != null) {
          newVal.add(prob, v);
        }
      }
    }
    double norm = newVal.norm(Norm.One);
    if (norm > 0) newVal.scale(1.0/norm);
    // Get rid of small values
    double[] data = newVal.getData();
    for (int i = 0; i < newVal.getUsed(); ++i) {
      if (data[i] < minProb) {
        data[i]=0.0;
      }
    }
    newVal.compact();
    res.put(srcWordId, newVal);
  }
  
  return res;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Create an un-normalzied sparse word embedding based on translation probabilities.
 * 
 * @param fieldIndex        
 *            an in-memory forward index.
 * @param minProb           
 *            ignore if the translation probability is smalelr than this one.
 * @param answToQuestTran   
 *            a provider of translation tables (computed originally by GIZA).
 * @param srcWordId         
 *            a source word ID
 * @return    an un-normalzied sparse word embedding based on translation probabilities.
 *            
 * @throws Exception
 */
public static SparseVector createOneTranVector(InMemForwardIndex             fieldIndex,
                                               float                         minProb, 
                                               GizaTranTableReaderAndRecoder answToQuestTran,
                                               int srcWordId) throws Exception {
  ArrayList<WordIdProb>   data = new ArrayList<WordIdProb>();
  if (srcWordId >= 0) {     
    GizaOneWordTranRecs tranRecs = answToQuestTran.getTranProbs(srcWordId);
    
    if (null != tranRecs) {
      for (int k = 0; k < tranRecs.mDstIds.length; ++k) {
        int dstWordId = tranRecs.mDstIds[k];
        double val = tranRecs.mProbs[k];
        if (val >= minProb) {
          data.add(new WordIdProb(dstWordId, val));
        }
      }
    }
  }
    
  WordIdProb dataArr[] = new WordIdProb[data.size()];
  dataArr = data.toArray(dataArr);
  Arrays.sort(dataArr);
  
  int     wordIds[] = new int[data.size()];
  double  vals[] = new double[data.size()];

  int indx = -1;
  int prevWordId = -1;
  for (WordIdProb e: dataArr) {
    if (e.mWordId != prevWordId) indx++;
    else 
      throw new Exception("Bug, repeating wordId " + prevWordId + ", translation record for wordId=" + srcWordId);
    wordIds[indx] = e.mWordId;
    vals[indx] += e.mProb;
    prevWordId = e.mWordId;
  }
  
  return new SparseVector(fieldIndex.getMaxWordId()+1, wordIds, vals, false);
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public static boolean ApproxEqual(SparseVector x, SparseVector y) {
  boolean res = true;

  if (x.getUsed() == y.getUsed()) {
    double dx[] = x.getData();
    int    ix[] = x.getIndex();
    double dy[] = y.getData();
    int    iy[] = y.getIndex();
    for (int i = 0; i < x.getUsed(); ++i) {
      if (ix[i] != iy[i]) {
        System.out.println(String.format("Different indices detected %d vs %d i=%d", ix[i], iy[i], i));
        res = false;
        break;
      }
      if (Math.abs(dx[i]-dy[i]) > 2* Float.MIN_NORMAL) {
        System.out.println(String.format("Different values detected %f vs %f i=%d index=%d", dx[i], dy[i], i,ix[i]));
        res = false;
        break;
      }
    }
  } else {
    System.err.println(String.format("Different # of elements %d vs %d", x.getUsed(), y.getUsed()));
    res = false;
  }

  if (!res) {
    System.out.println("Different vectors:");
    printSparseVector(x);
    printSparseVector(y);
  }

  return res;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public static synchronized void printSparseVector(SparseVector queryEmbedVector) {
  if (null == queryEmbedVector) return;
  
  double [] data = queryEmbedVector.getData();
  int    [] indx = queryEmbedVector.getIndex();
  
  
  System.out.println("The number of elements: " + queryEmbedVector.getUsed()); 
  
  for (int i = 0; i < queryEmbedVector.getUsed(); ++i) {
    System.out.print(indx[i]+":"+data[i]+" ");
  }
  System.out.println();
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Computes a square root of each sparse-vector element. 
 * 
 * @param v
 *      sparse vector
 * @return
 *      a vector of element-wise square roots.
 */
public static SparseVector sqrt(SparseVector v) {
  SparseVector res = v.copy();
  
  double [] data = res.getData();
  for (int k = 0; k < res.getUsed(); ++k) data[k] = Math.sqrt(data[k]);

  return res;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
public void add(StringSparseVector vector, String header) {
	if(vector.size() == 0)return;
	SparseVector featureVector = new SparseVector(params.fvSize());
	for(Entry<String, Double> entry:vector.getFv().entrySet()){
		featureVector.add(h2i.mapToInt(header.concat(entry.getKey())), entry.getValue());
	}
	featureVectors.add(new VectorInfo(0, 1., featureVector));
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
Map<Integer, Integer> getCooccurrence(int index){
    Map<Integer, Integer> result = new HashMap<>();
    SparseVector vec=cooccurrence.getRow(index);
    int[] nonZeroIndexes=vec.getIndex();

    for(int i=0; i<nonZeroIndexes.length; i++){
        int idx=nonZeroIndexes[i];
        double v = cooccurrence.get(index,idx);
        result.put(idx, (int)v);
    }

    return result;
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
@Override
public void subtractVector(double[] vectorAsArray) {
   setIsChanged();
   internalVector = (SparseVector) internalVector.add(-1, new SparseDoubleVector(vectorAsArray).internalVector);
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
@Override
public void addVector(IVector vector) {
   setIsChanged();
   internalVector = (SparseVector) internalVector.add(vector.toSparseVector().internalVector);
}
 

import no.uib.cipr.matrix.sparse.SparseVector; //导入依赖的package包/类
/**
 * Constructor, which doesn't really initialize: a separate function will do the real initialization;
 * this is done on purpose: an uninitialized instance can be used to check capabilities.
 * 
 * 
 * @param dirTranPrefix
 *            a prefix of the directories containing translation table.
 * @param gizaIterQty
 *            the number of GIZA++ iteration.
 * @param indexDir
 *            the directory that keeps serialized in-memory index.
 * @param embedDir
 *            the directory that keeps word embeddings.
 * @param embedFiles
 *            the list of dense word embedding files (relative to embedDir)
 * @param highOrderModelFiles
 *            the list of sparse word embedding files based on translation probabilities (relative to embedDir)
 *                       
 * @throws Exception
 */
public InMemIndexFeatureExtractor(String dirTranPrefix, 
                                  int gizaIterQty, 
                                  String indexDir,
                                  @Nullable String    embedDir,
                                  @Nullable String[]  embedFiles,
                                  @Nullable String[]  highOrderModelFiles) {
  mDirTranPrefix       = dirTranPrefix;
  mGizaIterQty         = gizaIterQty;
  mIndexDir            = indexDir;
  mEmbedDir            = embedDir;
  mEmbedFiles          = embedFiles;
  mHighOrderModelFiles = highOrderModelFiles;
  
  mHighOrderModels = new ArrayList<ArrayList<HashIntObjMap<SparseVector>>>();
  for (int fieldId = 0; fieldId < mFieldIndex.length; ++fieldId) {
    mHighOrderModels.add(null);
  }
}