void c4_FormatB::Define(int, const t4_byte **ptr_) {
  d4_assert(_memos.GetSize() == 0);

  if (ptr_ != 0) {
    _data.PullLocation(*ptr_);
    if (_data.ColSize() > 0)
      _sizeCol.PullLocation(*ptr_);
    _memoCol.PullLocation(*ptr_);
  }

  // everything below this point could be delayed until use
  // in that case, watch out that column space use is properly tracked

  InitOffsets(_sizeCol);

  if (_memoCol.ColSize() > 0) {
    c4_Bytes walk;
    _memoCol.FetchBytes(0, _memoCol.ColSize(), walk, true);

    const t4_byte *p = walk.Contents();

    for (int row = 0; p < walk.Contents() + walk.Size(); ++row) {
      row += c4_Column::PullValue(p);
      d4_assert(row < _memos.GetSize());

      c4_Column *mc = d4_new c4_Column(_data.Persist());
      d4_assert(mc != 0);
      _memos.SetAt(row, mc);

      mc->PullLocation(p);
    }

    d4_assert(p == walk.Contents() + walk.Size());
  }
}

c4_Column *c4_FormatB::GetNthMemoCol(int index_, bool alloc_) {
  t4_i32 start;
  c4_Column *col;
  int n = ItemLenOffCol(index_, start, col);

  if (col ==  &_data && alloc_) {
    col = d4_new c4_Column(_data.Persist());
    _memos.SetAt(index_, col);

    if (n > 0)
    if (_data.IsDirty()) {
      c4_Bytes temp;
      _data.FetchBytes(start, n, temp, true);
      col->SetBuffer(n);
      col->StoreBytes(0, temp);
    } else
      col->SetLocation(_data.Position() + start, n);
  }

  return col;
}

void c4_Differ::ApplyDiff(int id_, c4_Column &col_)const {
  d4_assert(0 <= id_ && id_ < _diffs.GetSize());

  c4_View diff = pDiff(_diffs[id_]);
  t4_i32 offset = 0;

  for (int n = 0; n < diff.GetSize(); ++n) {
    c4_RowRef row(diff[n]);
    offset += pKeep(row);

    c4_Bytes data;
    pBytes(row).GetData(data);

    // the following code is a lot like c4_MemoRef::Modify
    const t4_i32 change = pResize(row);
    if (change < 0)
      col_.Shrink(offset,  - change);
    else if (change > 0)
      col_.Grow(offset, change);

    col_.StoreBytes(offset, data);
    offset += data.Size();
  }

  if (offset > col_.ColSize())
    col_.Shrink(offset, offset - col_.ColSize());
}

bool c4_SaveContext::CommitColumn(c4_Column &col_) {
  bool changed = col_.IsDirty() || _fullScan;

  t4_i32 sz = col_.ColSize();
  StoreValue(sz);
  if (sz > 0) {
    t4_i32 pos = col_.Position();

    if (_differ) {
      if (changed) {
        int n = pos < 0 ? ~pos: _differ->NewDiffID();
        _differ->CreateDiff(n, col_);

        d4_assert(n >= 0);
        pos = ~n;
      }
    } else if (_preflight) {
      if (changed)
        pos = _space->Allocate(sz);

      _nextSpace->Occupy(pos, sz);
      _newPositions.Add(pos);
    } else {
      pos = _newPositions.GetAt(_nextPosIndex++);

      if (changed)
        col_.SaveNow(_strategy, pos);

      if (!_fullScan)
        col_.SetLocation(pos, sz);
    }

    StoreValue(pos);
  }

  return changed;
}

void c4_FormatB::SetOne(int index_, const c4_Bytes &xbuf_, bool ignoreMemos_) {
  // this fixes bug in 2.4.0 when copying string from higher row
  // TODO: this fix is very conservative, figure out when to copy
  // (can probably look at pointer to see whether it's from us)
  int sz = xbuf_.Size();
  c4_Bytes buf_(xbuf_.Contents(), sz, 0 < sz && sz <= c4_Column::kSegMax);

  c4_Column *cp = &_data;
  t4_i32 start = Offset(index_);
  int len = Offset(index_ + 1) - start;

  if (!ignoreMemos_ && _memos.GetAt(index_) != 0)
    len = ItemLenOffCol(index_, start, cp);

  int m = buf_.Size();
  int n = m - len;

  if (n > 0)
    cp->Grow(start, n);
  else if (n < 0)
    cp->Shrink(start,  - n);
  else if (m == 0)
    return ;
  // no size change and no contents

  _recalc = true;

  cp->StoreBytes(start, buf_);

  if (n && cp ==  &_data) {
    // if size has changed
    int k = _offsets.GetSize() - 1;

    // if filling in an empty entry at end: extend offsets first
    if (m > 0 && index_ >= k) {
      _offsets.InsertAt(k, _offsets.GetAt(k), index_ - k + 1);

      k = index_ + 1;
      d4_assert(k == _offsets.GetSize() - 1);
    }

    // adjust following entry offsets
    while (++index_ <= k)
      _offsets.ElementAt(index_) += n;
  }

  d4_assert((t4_i32)_offsets.GetAt(_offsets.GetSize() - 1) == _data.ColSize());
}

void c4_Differ::CreateDiff(int id_, c4_Column &col_) {
  _temp.SetSize(0);
#if 0
  t4_i32 offset = 0;
  t4_i32 savedOff = 0;
  t4_i32 savedLen = 0;

  c4_Strategy *strat = col_.Persist() != 0 ? &col_.Strategy(): 0;

  c4_ColIter iter(col_, 0, col_.ColSize());
  while (iter.Next()) {
    const t4_byte *p = iter.BufLoad();
    if (strat != 0 && strat->_mapStart != 0 && p >= strat->_mapStart && p -
      strat->_mapStart < strat->_dataSize) {
      t4_i32 nextOff = p - strat->_mapStart;
      if (savedLen == 0)
        savedOff = nextOff;
      if (nextOff == savedOff + savedLen) {
        savedLen += iter.BufLen();
        continue;
      }

      if (savedLen > 0)
        AddEntry(savedOff, savedLen, c4_Bytes());

      savedOff = nextOff;
      savedLen = iter.BufLen();
    } else {
      AddEntry(savedOff, savedLen, c4_Bytes(p, iter.BufLen()));
      savedLen = 0;
    }

    offset += iter.BufLen();
  }

  c4_View diff = pDiff(_diffs[id_]);
  if (_temp.GetSize() != diff.GetSize() || _temp != diff)
#else 
    c4_Bytes t1;
  const t4_byte *p = col_.FetchBytes(0, col_.ColSize(), t1, false);
  AddEntry(0, 0, c4_Bytes(p, col_.ColSize()));
#endif 
  pDiff(_diffs[id_]) = _temp;

  pOrig(_diffs[id_]) = col_.Position();
}

bool c4_Persist::LoadIt(c4_Column &walk_) {
  t4_i32 limit = _strategy.FileSize();
  if (_strategy._failure != 0)
    return false;

  if (_strategy.EndOfData(limit) < 0) {
    _strategy.SetBase(limit);
    d4_assert(_strategy._failure == 0); // file is ok, but empty
    return false;
  }

  if (_strategy._rootLen > 0)
    walk_.SetLocation(_strategy._rootPos, _strategy._rootLen);

  // if the file size has increased, we must remap
  if (_strategy._mapStart != 0 && _strategy.FileSize() > _strategy._baseOffset 
    + _strategy._dataSize)
    _strategy.ResetFileMapping();

  return true;
}

void c4_FormatB::Insert(int index_, const c4_Bytes& buf_, int count_)
{
  d4_assert(count_ > 0);

  _recalc = true;

  int m = buf_.Size();
  t4_i32 off = Offset(index_);

  _memos.InsertAt(index_, 0, count_);

    // insert the appropriate number of bytes
  t4_i32 n = count_ * (t4_i32) m;
  if (n > 0) {
    _data.Grow(off, n);

      // store as many copies as needed, but may have to do it in chunks
    int spos = 0;

    c4_ColIter iter (_data, off, off + n);
    while (iter.Next(m - spos)) {
      memcpy(iter.BufSave(), buf_.Contents() + spos, iter.BufLen());

      spos += iter.BufLen();
      if (spos >= m)
        spos = 0;
    }

    d4_assert(spos == 0); // must have copied an exact multiple of the data
  }

    // define offsets of the new entries
  _offsets.InsertAt(index_, 0, count_);
  d4_assert(_offsets.GetSize() <= _memos.GetSize() + 1);

  while (--count_ >= 0) {
    _offsets.SetAt(index_++, off);
    off += m;
  }

  d4_assert(index_ < _offsets.GetSize());

    // adjust all following entries
  while (index_ < _offsets.GetSize())
    _offsets.ElementAt(index_++) += n;

  d4_assert((t4_i32) _offsets.GetAt(index_ - 1) == _data.ColSize());
  d4_assert(index_ <= _memos.GetSize() + 1);
}

d4_inline t4_i32 c4_FormatB::Offset(int index_)const {
  d4_assert((t4_i32)_offsets.GetAt(_offsets.GetSize() - 1) == _data.ColSize());
  d4_assert(_offsets.GetSize() == _memos.GetSize() + 1);
  d4_assert(index_ < _offsets.GetSize());

  // extend offset vectors for missing empty entries at end 
  int n = _offsets.GetSize();
  d4_assert(n > 0);

  if (index_ >= n)
    index_ = n - 1;

  return _offsets.GetAt(index_);
}

void c4_FormatB::InitOffsets(c4_ColOfInts &sizes_) {
  int rows = Owner().NumRows();

  if (sizes_.RowCount() != rows) {
    sizes_.SetRowCount(rows);
  }

  _memos.SetSize(rows);
  _offsets.SetSize(rows + 1);

  if (_data.ColSize() > 0) {
    t4_i32 total = 0;

    for (int r = 0; r < rows; ++r) {
      int n = sizes_.GetInt(r);
      d4_assert(n >= 0);
      total += n;
      _offsets.SetAt(r + 1, total);
    }

    d4_assert(total == _data.ColSize());
  }

}