Golang backend.NewIDSet函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中github.com/restic/restic/backend.NewIDSet函数的典型用法代码示例。如果您正苦于以下问题：Golang NewIDSet函数的具体用法？Golang NewIDSet怎么用？Golang NewIDSet使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。

在下文中一共展示了NewIDSet函数的17个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: Execute

func (cmd CmdOptimize) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("optimize has no arguments")
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("Create exclusive lock for repository\n")
	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	chkr := checker.New(repo)

	cmd.global.Verbosef("Load indexes\n")
	_, errs := chkr.LoadIndex()

	if len(errs) > 0 {
		for _, err := range errs {
			cmd.global.Warnf("error: %v\n", err)
		}
		return fmt.Errorf("LoadIndex returned errors")
	}

	done := make(chan struct{})
	errChan := make(chan error)
	go chkr.Structure(errChan, done)

	for err := range errChan {
		if e, ok := err.(checker.TreeError); ok {
			cmd.global.Warnf("error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				cmd.global.Warnf("  %v\n", treeErr)
			}
		} else {
			cmd.global.Warnf("error: %v\n", err)
		}
	}

	unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...)
	cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs))

	repacker := checker.NewRepacker(repo, unusedBlobs)
	err = repacker.Repack()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("repacking done\n")
	return nil
}

开发者ID:marete，项目名称:restic，代码行数:56，代码来源:cmd_optimize.go

示例2: TestIDSet

func TestIDSet(t *testing.T) {
	set := backend.NewIDSet()
	for i, test := range idsetTests {
		seen := set.Has(test.id)
		if seen != test.seen {
			t.Errorf("IDSet test %v failed: wanted %v, got %v", i, test.seen, seen)
		}
		set.Insert(test.id)
	}
}

开发者ID:marete，项目名称:restic，代码行数:10，代码来源:idset_test.go

示例3: NewMasterIndex

// NewMasterIndex creates a new master index.
func NewMasterIndex() *MasterIndex {
	return &MasterIndex{
		inFlight: struct {
			backend.IDSet
			sync.RWMutex
		}{
			IDSet: backend.NewIDSet(),
		},
	}
}

开发者ID:marete，项目名称:restic，代码行数:11，代码来源:master_index.go

示例4: FindBlobsForPacks

// FindBlobsForPacks returns the set of blobs contained in a pack of packs.
func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) {
	blobs := backend.NewIDSet()

	for packID := range packs {
		for _, packedBlob := range repo.Index().ListPack(packID) {
			blobs.Insert(packedBlob.ID)
		}
	}

	return blobs, nil
}

开发者ID:marete，项目名称:restic，代码行数:12，代码来源:repacker.go

示例5: Packs

// Packs returns all packs in this index
func (idx *Index) Packs() backend.IDSet {
	idx.m.Lock()
	defer idx.m.Unlock()

	packs := backend.NewIDSet()
	for _, entry := range idx.pack {
		packs.Insert(entry.packID)
	}

	return packs
}

开发者ID:jhautefeuille，项目名称:restic，代码行数:12，代码来源:index.go

示例6: FindPacksForBlobs

// FindPacksForBlobs returns the set of packs that contain the blobs.
func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) {
	packs := backend.NewIDSet()
	idx := repo.Index()
	for id := range blobs {
		blob, err := idx.Lookup(id)
		if err != nil {
			return nil, err
		}

		packs.Insert(blob.PackID)
	}

	return packs, nil
}

开发者ID:marete，项目名称:restic，代码行数:15，代码来源:repacker.go

示例7: TestIndexPacks

func TestIndexPacks(t *testing.T) {
	idx := repository.NewIndex()
	packs := backend.NewIDSet()

	for i := 0; i < 20; i++ {
		packID := randomID()
		idx.Store(pack.Data, randomID(), packID, 0, 23)

		packs.Insert(packID)
	}

	idxPacks := idx.Packs()
	Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index")
}

开发者ID:JaCoB1123，项目名称:restic，代码行数:14，代码来源:index_test.go

示例8: TestIndexPacks

func TestIndexPacks(t *testing.T) {
	idx := repository.NewIndex()
	packs := backend.NewIDSet()

	for i := 0; i < 20; i++ {
		packID := randomID()
		idx.Store(repository.PackedBlob{
			Type:   pack.Data,
			ID:     randomID(),
			PackID: packID,
			Offset: 0,
			Length: 23,
		})

		packs.Insert(packID)
	}

	idxPacks := idx.Packs()
	Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index")
}

开发者ID:marete，项目名称:restic，代码行数:20，代码来源:index_test.go

示例9: TestSet

func TestSet(t *testing.T) {
	s := backend.NewIDSet()

	testID := randomID()
	err := s.Find(testID)
	Assert(t, err != nil, "found test ID in IDSet before insertion")

	for i := 0; i < 238; i++ {
		s.Insert(randomID())
	}

	s.Insert(testID)
	OK(t, s.Find(testID))

	for i := 0; i < 80; i++ {
		s.Insert(randomID())
	}

	s.Insert(testID)
	OK(t, s.Find(testID))
}

开发者ID:rawtaz，项目名称:restic，代码行数:21，代码来源:id_set_test.go

示例10: NewArchiver

// NewArchiver returns a new archiver.
func NewArchiver(repo *repository.Repository) *Archiver {
	arch := &Archiver{
		repo:      repo,
		blobToken: make(chan struct{}, maxConcurrentBlobs),
		knownBlobs: struct {
			backend.IDSet
			sync.Mutex
		}{
			IDSet: backend.NewIDSet(),
		},
	}

	for i := 0; i < maxConcurrentBlobs; i++ {
		arch.blobToken <- struct{}{}
	}

	arch.Error = archiverAbortOnAllErrors
	arch.SelectFilter = archiverAllowAllFiles

	return arch
}

开发者ID:jhautefeuille，项目名称:restic，代码行数:22，代码来源:archiver.go

示例11: TestRepacker

func TestRepacker(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)
		OK(t, repo.LoadIndex())

		repo.Backend().Remove(backend.Snapshot, "c2b53c5e6a16db92fbb9aa08bd2794c58b379d8724d661ee30d20898bdfdff22")

		unusedBlobs := backend.IDSet{
			ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{},
			ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{},
			ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{},
			ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{},
		}

		chkr := checker.New(repo)
		_, errs := chkr.LoadIndex()
		OKs(t, errs)

		errs = checkStruct(chkr)
		OKs(t, errs)

		list := backend.NewIDSet(chkr.UnusedBlobs()...)
		if !unusedBlobs.Equals(list) {
			t.Fatalf("expected unused blobs:\n  %v\ngot:\n  %v", unusedBlobs, list)
		}

		repacker := checker.NewRepacker(repo, unusedBlobs)
		OK(t, repacker.Repack())

		chkr = checker.New(repo)
		_, errs = chkr.LoadIndex()
		OKs(t, errs)
		OKs(t, checkPacks(chkr))
		OKs(t, checkStruct(chkr))

		blobs := chkr.UnusedBlobs()
		Assert(t, len(blobs) == 0,
			"expected zero unused blobs, got %v", blobs)
	})
}

开发者ID:marete，项目名称:restic，代码行数:40，代码来源:repacker_test.go

示例12: LoadIndex

// LoadIndex loads all index files.
func (c *Checker) LoadIndex() (hints []error, errs []error) {
	debug.Log("LoadIndex", "Start")
	type indexRes struct {
		Index *repository.Index
		ID    string
	}

	indexCh := make(chan indexRes)

	worker := func(id backend.ID, done <-chan struct{}) error {
		debug.Log("LoadIndex", "worker got index %v", id)
		idx, err := repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeIndex)
		if err == repository.ErrOldIndexFormat {
			debug.Log("LoadIndex", "index %v has old format", id.Str())
			hints = append(hints, ErrOldIndexFormat{id})

			idx, err = repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeOldIndex)
		}

		if err != nil {
			return err
		}

		select {
		case indexCh <- indexRes{Index: idx, ID: id.String()}:
		case <-done:
		}

		return nil
	}

	var perr error
	go func() {
		defer close(indexCh)
		debug.Log("LoadIndex", "start loading indexes in parallel")
		perr = repository.FilesInParallel(c.repo.Backend(), backend.Index, defaultParallelism,
			repository.ParallelWorkFuncParseID(worker))
		debug.Log("LoadIndex", "loading indexes finished, error: %v", perr)
	}()

	done := make(chan struct{})
	defer close(done)

	if perr != nil {
		errs = append(errs, perr)
		return hints, errs
	}

	packToIndex := make(map[backend.ID]backend.IDSet)

	for res := range indexCh {
		debug.Log("LoadIndex", "process index %v", res.ID)
		idxID, err := backend.ParseID(res.ID)
		if err != nil {
			errs = append(errs, fmt.Errorf("unable to parse as index ID: %v", res.ID))
			continue
		}

		c.indexes[idxID] = res.Index
		c.masterIndex.Insert(res.Index)

		debug.Log("LoadIndex", "process blobs")
		cnt := 0
		for blob := range res.Index.Each(done) {
			c.packs[blob.PackID] = struct{}{}
			c.blobs[blob.ID] = struct{}{}
			c.blobRefs.M[blob.ID] = 0
			cnt++

			if _, ok := packToIndex[blob.PackID]; !ok {
				packToIndex[blob.PackID] = backend.NewIDSet()
			}
			packToIndex[blob.PackID].Insert(idxID)
		}

		debug.Log("LoadIndex", "%d blobs processed", cnt)
	}

	debug.Log("LoadIndex", "done, error %v", perr)

	debug.Log("LoadIndex", "checking for duplicate packs")
	for packID := range c.packs {
		debug.Log("LoadIndex", "  check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
		if len(packToIndex[packID]) > 1 {
			hints = append(hints, ErrDuplicatePacks{
				PackID:  packID,
				Indexes: packToIndex[packID],
			})
		}
	}

	c.repo.SetIndex(c.masterIndex)

	return hints, errs
}

开发者ID:jhautefeuille，项目名称:restic，代码行数:96，代码来源:checker.go

示例13: TestRebuildIndexAlwaysFull

		}
	})
}

func TestRebuildIndexAlwaysFull(t *testing.T) {
	repository.IndexFull = func(*repository.Index) bool { return true }
	TestRebuildIndex(t)
}

var optimizeTests = []struct {
	testFilename string
	snapshots    backend.IDSet
}{
	{
		filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz"),
		backend.NewIDSet(ParseID("a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43")),
	},
	{
		filepath.Join("testdata", "old-index-repo.tar.gz"),
		nil,
	},
	{
		filepath.Join("testdata", "old-index-repo.tar.gz"),
		backend.NewIDSet(
			ParseID("f7d83db709977178c9d1a09e4009355e534cde1a135b8186b8b118a3fc4fcd41"),
			ParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"),
		),
	},
}

func TestOptimizeRemoveUnusedBlobs(t *testing.T) {

开发者ID:marete，项目名称:restic，代码行数:31，代码来源:integration_test.go

示例14: fsckTree

func fsckTree(global CmdFsck, repo *repository.Repository, id backend.ID) error {
	debug.Log("restic.fsckTree", "checking tree %v", id.Str())

	tree, err := restic.LoadTree(repo, id)
	if err != nil {
		return err
	}

	// if orphan check is active, record storage id
	if global.o_trees != nil {
		// add ID to list
		global.o_trees.Insert(id)
	}

	var firstErr error

	seenIDs := backend.NewIDSet()

	for i, node := range tree.Nodes {
		if node.Name == "" {
			return fmt.Errorf("node %v of tree %v has no name", i, id.Str())
		}

		if node.Type == "" {
			return fmt.Errorf("node %q of tree %v has no type", node.Name, id.Str())
		}

		switch node.Type {
		case "file":
			if node.Content == nil {
				debug.Log("restic.fsckTree", "file node %q of tree %v has no content: %v", node.Name, id, node)
				return fmt.Errorf("file node %q of tree %v has no content: %v", node.Name, id, node)
			}

			if node.Content == nil && node.Error == "" {
				debug.Log("restic.fsckTree", "file node %q of tree %v has no content", node.Name, id)
				return fmt.Errorf("file node %q of tree %v has no content", node.Name, id)
			}

			// record ids
			for _, id := range node.Content {
				seenIDs.Insert(id)
			}

			debug.Log("restic.fsckTree", "check file %v (%v)", node.Name, id.Str())
			bytes, err := fsckFile(global, repo, node.Content)
			if err != nil {
				return err
			}

			if bytes != node.Size {
				debug.Log("restic.fsckTree", "file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, id, node.Size, bytes)
				return fmt.Errorf("file node %q of tree %v has size %d, but only %d bytes could be found", node.Name, id, node.Size, bytes)
			}
		case "dir":
			if node.Subtree == nil {
				return fmt.Errorf("dir node %q of tree %v has no subtree", node.Name, id)
			}

			// record id
			seenIDs.Insert(node.Subtree)

			err = fsckTree(global, repo, node.Subtree)
			if err != nil {
				firstErr = err
				fmt.Fprintf(os.Stderr, "%v\n", err)
			}
		}
	}

	// check map for unused ids
	// for _, id := range tree.Map.IDs() {
	// 	if seenIDs.Find(id) != nil {
	// 		return fmt.Errorf("tree %v: map contains unused ID %v", id, id)
	// 	}
	// }

	return firstErr
}

开发者ID:rawtaz，项目名称:restic，代码行数:79，代码来源:cmd_fsck.go

示例15: Execute

func (cmd CmdFsck) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("fsck has no arguments")
	}

	if cmd.RemoveOrphaned && !cmd.Orphaned {
		cmd.Orphaned = true
	}

	s, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	err = s.LoadIndex()
	if err != nil {
		return err
	}

	if cmd.Snapshot != "" {
		id, err := restic.FindSnapshot(s, cmd.Snapshot)
		if err != nil {
			return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err)
		}

		err = fsckSnapshot(cmd, s, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
		}

		return err
	}

	if cmd.Orphaned {
		cmd.o_data = backend.NewIDSet()
		cmd.o_trees = backend.NewIDSet()
	}

	done := make(chan struct{})
	defer close(done)

	var firstErr error
	for id := range s.List(backend.Snapshot, done) {
		err = fsckSnapshot(cmd, s, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
			firstErr = err
		}
	}

	if !cmd.Orphaned {
		return firstErr
	}

	debug.Log("restic.fsck", "starting orphaned check\n")

	cnt := make(map[pack.BlobType]*backend.IDSet)
	cnt[pack.Data] = cmd.o_data
	cnt[pack.Tree] = cmd.o_trees

	for blob := range s.Index().Each(done) {
		debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID)

		err = cnt[blob.Type].Find(blob.ID)
		if err != nil {
			debug.Log("restic.fsck", "  blob %v is orphaned\n", blob.ID)

			if !cmd.RemoveOrphaned {
				fmt.Printf("orphaned %v blob %v\n", blob.Type, blob.ID)
				continue
			}

			fmt.Printf("removing orphaned %v blob %v\n", blob.Type, blob.ID)
			// err := s.Remove(d.tpe, name)
			// if err != nil {
			// 	return err
			// }
			return errors.New("not implemented")
		}
	}

	return firstErr
}

开发者ID:rawtaz，项目名称:restic，代码行数:83，代码来源:cmd_fsck.go

示例16: RebuildIndex

func (cmd CmdRebuildIndex) RebuildIndex() error {
	debug.Log("RebuildIndex.RebuildIndex", "start")

	done := make(chan struct{})
	defer close(done)

	indexIDs := backend.NewIDSet()
	for id := range cmd.repo.List(backend.Index, done) {
		indexIDs.Insert(id)
	}

	cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs))

	debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))

	combinedIndex := repository.NewIndex()
	packsDone := backend.NewIDSet()

	type Blob struct {
		id  backend.ID
		tpe pack.BlobType
	}
	blobsDone := make(map[Blob]struct{})

	i := 0
	for indexID := range indexIDs {
		cmd.global.Printf("  loading index %v\n", i)

		debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
		idx, err := repository.LoadIndex(cmd.repo, indexID.String())
		if err != nil {
			return err
		}

		debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())

		for packedBlob := range idx.Each(done) {
			packsDone.Insert(packedBlob.PackID)
			b := Blob{
				id:  packedBlob.ID,
				tpe: packedBlob.Type,
			}
			if _, ok := blobsDone[b]; ok {
				continue
			}

			blobsDone[b] = struct{}{}
			combinedIndex.Store(packedBlob)
		}

		combinedIndex.AddToSupersedes(indexID)

		if repository.IndexFull(combinedIndex) {
			combinedIndex, err = cmd.storeIndex(combinedIndex)
			if err != nil {
				return err
			}
		}

		i++
	}

	var err error
	if combinedIndex.Length() > 0 {
		combinedIndex, err = cmd.storeIndex(combinedIndex)
		if err != nil {
			return err
		}
	}

	cmd.global.Printf("removing %d old indexes\n", len(indexIDs))
	for id := range indexIDs {
		debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())

		err := cmd.repo.Backend().Remove(backend.Index, id.String())
		if err != nil {
			debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
			return err
		}
	}

	cmd.global.Printf("checking for additional packs\n")
	newPacks := 0
	for packID := range cmd.repo.List(backend.Data, done) {
		if packsDone.Has(packID) {
			continue
		}

		debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
		newPacks++

		rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0)
		if err != nil {
			debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err)
			return err
		}

		var readSeeker io.ReadSeeker
		if r, ok := rd.(io.ReadSeeker); ok {
			debug.Log("RebuildIndex.RebuildIndex", "reader is seekable")
//.........这里部分代码省略.........

开发者ID:marete，项目名称:restic，代码行数:101，代码来源:cmd_rebuild_index.go

示例17: RebuildIndex

func (cmd CmdRebuildIndex) RebuildIndex() error {
	debug.Log("RebuildIndex.RebuildIndex", "start")

	done := make(chan struct{})
	defer close(done)

	indexIDs := backend.NewIDSet()
	for id := range cmd.repo.List(backend.Index, done) {
		indexIDs.Insert(id)
	}

	cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs))

	debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))

	combinedIndex := repository.NewIndex()
	packsDone := backend.NewIDSet()

	type Blob struct {
		id  backend.ID
		tpe pack.BlobType
	}
	blobsDone := make(map[Blob]struct{})

	i := 0
	for indexID := range indexIDs {
		cmd.global.Printf("  loading index %v\n", i)

		debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
		idx, err := repository.LoadIndex(cmd.repo, indexID.String())
		if err != nil {
			return err
		}

		debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())

		for packedBlob := range idx.Each(done) {
			packsDone.Insert(packedBlob.PackID)
			b := Blob{
				id:  packedBlob.ID,
				tpe: packedBlob.Type,
			}
			if _, ok := blobsDone[b]; ok {
				continue
			}

			blobsDone[b] = struct{}{}
			combinedIndex.Store(packedBlob)
		}

		combinedIndex.AddToSupersedes(indexID)

		if repository.IndexFull(combinedIndex) {
			combinedIndex, err = cmd.storeIndex(combinedIndex)
			if err != nil {
				return err
			}
		}

		i++
	}

	var err error
	if combinedIndex.Length() > 0 {
		combinedIndex, err = cmd.storeIndex(combinedIndex)
		if err != nil {
			return err
		}
	}

	cmd.global.Printf("removing %d old indexes\n", len(indexIDs))
	for id := range indexIDs {
		debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())

		err := cmd.repo.Backend().Remove(backend.Index, id.String())
		if err != nil {
			debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
			return err
		}
	}

	cmd.global.Printf("checking for additional packs\n")
	newPacks := 0
	var buf []byte
	for packID := range cmd.repo.List(backend.Data, done) {
		if packsDone.Has(packID) {
			continue
		}

		debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
		newPacks++

		var err error

		h := backend.Handle{Type: backend.Data, Name: packID.String()}
		buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf)
		if err != nil {
			debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str())
			return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err)
		}
//.........这里部分代码省略.........

开发者ID:jhautefeuille，项目名称:restic，代码行数:101，代码来源:cmd_rebuild_index.go

注：本文中的github.com/restic/restic/backend.NewIDSet函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。