import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
private static void loadSparseDoublePartition(SparseDoubleModel model, FSDataInputStream input,
    ModelPartitionMeta partMeta) throws IOException {
  int rowNum = input.readInt();
  int rowId = 0;
  int nnz = 0;
  int totalNNZ = 0;
  Int2DoubleOpenHashMap row = null;
  for (int i = 0; i < rowNum; i++) {
    rowId = input.readInt();
    nnz = input.readInt();
    totalNNZ = (int) (nnz * (model.col) / (partMeta.getEndCol() - partMeta.getStartCol()));
    row = model.getRow(rowId, partMeta.getPartId(), totalNNZ);
    for (int j = 0; j < nnz; j++) {
      row.put(input.readInt(), input.readDouble());
    }
  }
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
/**
 * Load dense double model to int->double maps
 *
 * @param modelDir model save directory path
 * @return model data
 */
public static Int2DoubleOpenHashMap[] loadToDoubleMaps(String modelDir, Configuration conf)
    throws IOException {
  // Load model meta
  ModelFilesMeta meta = getMeta(modelDir, conf);

  // Check row type
  if (meta.getRowType() != SPARSE_DOUBLE) {
    throw new IOException("model row type is not sparse double, you should check it");
  }

  // Load model
  SparseDoubleModel model = new SparseDoubleModel(meta.getRow(), meta.getCol());
  loadModel(modelDir, model, meta, conf);

  return model.getModel();
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
private static void convertSparseDoubleModel(Configuration conf, FSDataOutputStream output,
  String modelInputDir, ModelLineConvert lineConvert) throws IOException {
  Int2DoubleOpenHashMap [] data = ModelLoader.loadToDoubleMaps(modelInputDir, conf);
  for(int i = 0; i < data.length; i++) {
    Int2DoubleOpenHashMap row = data[i];
    data[i] = null;
    if(row == null) {
      continue;
    }

    lineConvert.convertRowIndex(output, i);
    int [] indexes = row.keySet().toIntArray();
    double [] values = row.values().toDoubleArray();
    row = null;
    Sort.quickSort(indexes, values, 0, indexes.length - 1);
    for(int j = 0; j < indexes.length; j++) {
      lineConvert.convertDouble(output, indexes[j], values[j]);
    }
  }
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
private double dot(SparseDoubleVector other) {
  double ret = 0.0;

  Int2DoubleOpenHashMap smallMap = this.hashMap;
  Int2DoubleOpenHashMap largeMap = other.hashMap;

  if (smallMap.size() > largeMap.size()) {
    smallMap = other.hashMap;
    largeMap = this.hashMap;
  }

  ObjectIterator<Int2DoubleMap.Entry> iter = smallMap.int2DoubleEntrySet().fastIterator();

  Int2DoubleMap.Entry entry = null;
  while (iter.hasNext()) {
    entry = iter.next();
    if (largeMap.containsKey(entry.getIntKey())) {
      ret += entry.getDoubleValue() * largeMap.get(entry.getIntKey());
    }
  }
  return ret;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Override
public TIntDoubleVector filter(double x) {
  Int2DoubleOpenHashMap newMap = new Int2DoubleOpenHashMap();

  ObjectIterator<Int2DoubleMap.Entry> iter = hashMap.int2DoubleEntrySet().fastIterator();
  Int2DoubleMap.Entry entry = null;
  while (iter.hasNext()) {
    entry = iter.next();
    double value = entry.getDoubleValue();
    if (Math.abs(value) > x) {
      newMap.put(entry.getIntKey(), value);
    }
  }
  SparseDoubleVector vector = new SparseDoubleVector(dim, newMap);
  vector.setRowId(rowId).setMatrixId(matrixId).setClock(clock);
  return vector;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Test
public void testWriteTo() throws Exception {
  ByteBuf buf = Unpooled.buffer(16);
  buf.writeDouble(0.00);
  buf.writeDouble(1.00);
  buf.writeDouble(2.00);
  serverSparseDoubleRow.update(RowType.T_DOUBLE_DENSE, buf, 3);
  DataOutputStream out = new DataOutputStream(new FileOutputStream("data"));
  serverSparseDoubleRow.writeTo(out);
  out.close();
  DataInputStream in = new DataInputStream(new FileInputStream("data"));
  assertEquals(in.readInt(), 3);
  Int2DoubleOpenHashMap hashMap = new Int2DoubleOpenHashMap();
  hashMap.addTo(0, 0.00);
  hashMap.addTo(1, 1.00);
  hashMap.addTo(2, 2.00);
  assertEquals(serverSparseDoubleRow.getData(), hashMap);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Test
public void testUpdateDoubleSparseToDoubleSparse() throws Exception {
  ServerSparseDoubleRow serverSparseDoubleRow =
      new ServerSparseDoubleRow(rowId, startCol, endCol);
  ByteBuf buf = Unpooled.buffer(16);
  buf.writeInt(0);
  buf.writeDouble(0.00);
  buf.writeInt(1);
  buf.writeDouble(1.00);
  buf.writeInt(2);
  buf.writeDouble(2.00);
  rowUpdater.updateDoubleSparseToDoubleSparse(3, buf, serverSparseDoubleRow);
  Int2DoubleOpenHashMap hashMap = new Int2DoubleOpenHashMap();
  hashMap.addTo(0, 0.00);
  hashMap.addTo(1, 1.00);
  hashMap.addTo(2, 2.00);
  assertEquals(serverSparseDoubleRow.getData(), hashMap);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
/**
 * Distributed matrix set.
 *
 * @param a Row or column index.
 * @param b Row or column index.
 * @param v New value to set.
 */
private void matrixSet(int a, int b, double v) {
    // Remote set on the primary node (where given row or column is stored locally).
    ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, a)).run(() -> {
        IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);

        // Local get.
        Map<Integer, Double> map = cache.localPeek(getCacheKey(a), CachePeekMode.PRIMARY);

        if (map == null) {
            map = cache.get(getCacheKey(a)); //Remote entry get.

            if (map == null)
                map = acsMode == SEQUENTIAL_ACCESS_MODE ? new Int2DoubleRBTreeMap() : new Int2DoubleOpenHashMap();
        }

        if (v != 0.0)
            map.put(b, v);
        else if (map.containsKey(b))
            map.remove(b);

        // Local put.
        cache.put(getCacheKey(a), map);
    });
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
private Int2DoubleMap getProductMap(int uidx) {
    Int2DoubleOpenHashMap productMap = new Int2DoubleOpenHashMap();
    productMap.defaultReturnValue(0.0);

    if (data.useIteratorsPreferentially()) {
        IntIterator iidxs = data.getUidxIidxs(uidx);
        DoubleIterator ivs = data.getUidxVs(uidx);
        while (iidxs.hasNext()) {
            int iidx = iidxs.nextInt();
            double iv = ivs.nextDouble();
            IntIterator vidxs = data.getIidxUidxs(iidx);
            DoubleIterator vvs = data.getIidxVs(iidx);
            while (vidxs.hasNext()) {
                productMap.addTo(vidxs.nextInt(), iv * vvs.nextDouble());
            }
        }
    } else {
        data.getUidxPreferences(uidx)
                .forEach(ip -> data.getIidxPreferences(ip.v1)
                        .forEach(up -> productMap.addTo(up.v1, ip.v2 * up.v2)));
    }

    productMap.remove(uidx);

    return productMap;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Override
public SparseFloatVector newNumberVector(Int2DoubleOpenHashMap dvalues, int maxdim) {
  int[] indexes = new int[dvalues.size()];
  float[] values = new float[dvalues.size()];
  // Import and sort the indexes
  ObjectIterator<Int2DoubleMap.Entry> iter = dvalues.int2DoubleEntrySet().fastIterator();
  for(int i = 0; iter.hasNext(); i++) {
    indexes[i] = iter.next().getIntKey();
  }
  Arrays.sort(indexes);
  // Import the values accordingly
  for(int i = 0; i < dvalues.size(); i++) {
    values[i] = (float) dvalues.get(indexes[i]);
  }
  return new SparseFloatVector(indexes, values, maxdim);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Test
public void sparseAngleDegenerate() {
  NumberVector o1 = new SparseDoubleVector(new double[] {});
  Int2DoubleOpenHashMap v2 = new Int2DoubleOpenHashMap();
  v2.put(3, 0.);
  v2.put(4, 0.);
  v2.put(42, 0.);
  NumberVector o2 = new SparseDoubleVector(v2, 100);
  Int2DoubleOpenHashMap v3 = new Int2DoubleOpenHashMap();
  v3.put(15, 0.);
  v3.put(5, 1.);
  NumberVector v1 = new SparseDoubleVector(v3, 100);
  assertEquals("Angle not exact.", 0., VectorUtil.cosAngle(o1, o1), 0.);
  assertEquals("Angle not exact.", 0., VectorUtil.cosAngle(o1, o2), 0.);
  assertEquals("Angle not exact.", 0., VectorUtil.cosAngle(o2, o2), 0.);
  assertEquals("Angle not exact.", 0., VectorUtil.cosAngle(o1, v1), 0.);
  assertEquals("Angle not exact.", 0., VectorUtil.cosAngle(o2, v1), 0.);
  assertEquals("Angle not exact.", 1., VectorUtil.cosAngle(v1, v1), 0.);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
/**
 * Get a model row use row index
 *
 * @param rowId row index
 * @param partId partition index
 * @return a model row
 */
public Int2DoubleOpenHashMap getRow(int rowId, int partId) {
  synchronized (this) {
    if (tempModel.get(rowId) == null) {
      tempModel.put(rowId, new HashMap<>());
      tempModel.get(rowId).put(partId, new Int2DoubleOpenHashMap());
    } else {
      if (tempModel.get(rowId).get(partId) == null) {
        tempModel.get(rowId).put(partId, new Int2DoubleOpenHashMap());
      }
    }

    return tempModel.get(rowId).get(partId);
  }
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
public static Int2DoubleOpenHashMap loadSparseDoubleRowFromPartition(FSDataInputStream input,
    ModelPartitionMeta partMeta, int rowId) throws IOException {
  RowOffset rowOffset = partMeta.getRowMetas().get(rowId);
  input.seek(rowOffset.getOffset());
  Preconditions.checkState (input.readInt() == rowId);
  int num = input.readInt();
  Int2DoubleOpenHashMap row = new Int2DoubleOpenHashMap();
  for (int i = 0; i < num; i++) {
    row.put(input.readInt(), input.readDouble());
  }
  return row;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
/**
 * init the dim and capacity for vector
 *
 * @param dim
 * @param capacity
 */
public SparseDoubleVector(int dim, int capacity) {
  super();
  if(capacity > 0) {
    this.hashMap = new Int2DoubleOpenHashMap(capacity);
  } else {
    this.hashMap = new Int2DoubleOpenHashMap(INIT_SIZE);
  }
  this.dim = dim;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Override
public void deserialize(ByteBuf buf) {
  int dim = buf.readInt();
  int length = buf.readInt();
  Int2DoubleOpenHashMap data = new Int2DoubleOpenHashMap(length);
  IntStream.range(0,length).forEach(i-> data.put(buf.readInt(), buf.readDouble()));
  this.dim = dim;
  this.hashMap = data;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
private double[] getVluesofServerSparseDoubleRow(ServerRow row, int[] index) {
  Int2DoubleOpenHashMap data = ((ServerSparseDoubleRow) row).getData();
  double[] values = new double[index.length];

  for (int i = 0; i < index.length; i++) {
    values[i] = data.get(index[i]);
  }

  return values;
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
/**
 * Merge this sparse double vector split to a map
 * @param indexToValueMap a index->value map
 */
public void mergeTo(Int2DoubleOpenHashMap indexToValueMap) {
  try {
    lock.readLock().lock();
    indexToValueMap.putAll(hashMap);
  } finally {
    lock.readLock().unlock();
  }
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Test
public void testUpdateDoubleDenseToDoubleSparse() throws Exception {
  ServerSparseDoubleRow serverSparseDoubleRow =
      new ServerSparseDoubleRow(rowId, startCol, endCol);
  ByteBuf buf = Unpooled.buffer(16);
  buf.writeDouble(0.00);
  buf.writeDouble(1.00);
  buf.writeDouble(2.00);
  rowUpdater.updateDoubleDenseToDoubleSparse(3, buf, serverSparseDoubleRow);
  Int2DoubleOpenHashMap hashMap = new Int2DoubleOpenHashMap();
  hashMap.addTo(0, 0.00);
  hashMap.addTo(1, 1.00);
  hashMap.addTo(2, 2.00);
  assertEquals(serverSparseDoubleRow.getData(), hashMap);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
public LearningInstance featurize(JsonNode entity, boolean update) {
    Map<String, List<Feature>> feaMap = new HashMap<>();
    for (FeatureExtractor extractor : featureExtractors) {
        feaMap.putAll(extractor.extract(entity, update,
                indexSpace));
    }
    Int2DoubleMap feas = new Int2DoubleOpenHashMap();
    for (String fea : features) {
        if (feaMap.containsKey(fea)) {
            for (Feature feature : feaMap.get(fea)) {
                feas.put(feature.getIndex(), feature.getValue());
            }
        }
    }
    double label = StandardLearningInstance.defaultLabel;
    double weight = StandardLearningInstance.defaultWeight;
    if (entity.has(labelName)) {
        label = entity.get(labelName).asDouble();
    } else if (feaMap.containsKey(labelName)) {
        label = feaMap.get(labelName).get(0).getValue();
    }
    if (entity.has(weightName)) {
        weight = entity.get(weightName).asDouble();
    } else if (feaMap.containsKey(weightName)) {
        weight = feaMap.get(weightName).get(0).getValue();
    }
    String group = null;
    if (groupKeys != null && groupKeys.size() > 0) {
        group = FeatureExtractorUtilities.composeConcatenatedKey(entity, groupKeys);
    }
    return new StandardLearningInstance(feas, label, weight, group);
}
 

import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; //导入依赖的package包/类
@Override
public Int2DoubleMap scores(int node) {
	int[] succs = super.successors(node).toIntArray();
	return new Int2DoubleOpenHashMap(
			succs,
			rnd.doubles(succs.length).toArray()
			);
}