本文整理汇总了Golang中github.com/pachyderm/pachyderm/src/pfs/pfsutil.PutFile函数的典型用法代码示例。如果您正苦于以下问题:Golang PutFile函数的具体用法?Golang PutFile怎么用?Golang PutFile使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了PutFile函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: TestFromCommit
func TestFromCommit(t *testing.T) {
t.Parallel()
repo := uniqueString("TestFromCommit")
pachClient := getPachClient(t)
seed := time.Now().UnixNano()
rand := rand.New(rand.NewSource(seed))
err := pfsutil.CreateRepo(pachClient, repo)
require.NoError(t, err)
commit1, err := pfsutil.StartCommit(pachClient, repo, "")
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, repo, commit1.Id, "file", 0, workload.NewReader(rand, KB))
require.NoError(t, err)
err = pfsutil.FinishCommit(pachClient, repo, commit1.Id)
require.NoError(t, err)
commit2, err := pfsutil.StartCommit(pachClient, repo, commit1.Id)
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, repo, commit2.Id, "file", 0, workload.NewReader(rand, KB))
require.NoError(t, err)
err = pfsutil.FinishCommit(pachClient, repo, commit2.Id)
require.NoError(t, err)
var buffer bytes.Buffer
require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, commit1.Id, nil, &buffer))
require.Equal(t, buffer.Len(), KB)
buffer = bytes.Buffer{}
require.NoError(t, pfsutil.GetFile(pachClient, repo, commit2.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, buffer.Len(), 2*KB)
}
开发者ID:tv42,项目名称:pachyderm,代码行数:27,代码来源:pachyderm_test.go
示例2: TestJob
func TestJob(t *testing.T) {
dataRepo := uniqueString("TestJob.data")
pfsClient := getPfsClient(t)
require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
require.NoError(t, err)
_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
ppsClient := getPpsClient(t)
job, err := ppsutil.CreateJob(
ppsClient,
"",
[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
"",
1,
[]*pfs.Commit{commit},
"",
)
require.NoError(t, err)
inspectJobRequest := &pps.InspectJobRequest{
Job: job,
BlockOutput: true,
BlockState: true,
}
jobInfo, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
require.NoError(t, err)
var buffer bytes.Buffer
require.NoError(t, pfsutil.GetFile(pfsClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, nil, &buffer))
require.Equal(t, "foo", buffer.String())
}
开发者ID:tjyang,项目名称:pachyderm,代码行数:31,代码来源:pachyderm_test.go
示例3: TestJob
func TestJob(t *testing.T) {
t.Parallel()
dataRepo := uniqueString("TestJob.data")
pachClient := getPachClient(t)
require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo\n"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
job, err := ppsutil.CreateJob(
pachClient,
"",
[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
nil,
1,
[]*pps.JobInput{{Commit: commit}},
"",
)
require.NoError(t, err)
inspectJobRequest := &pps.InspectJobRequest{
Job: job,
BlockOutput: true,
BlockState: true,
}
jobInfo, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
require.NoError(t, err)
require.Equal(t, pps.JobState_JOB_STATE_SUCCESS.String(), jobInfo.State.String())
commitInfo, err := pfsutil.InspectCommit(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id)
require.NoError(t, err)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
var buffer bytes.Buffer
require.NoError(t, pfsutil.GetFile(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, "foo\n", buffer.String())
}
开发者ID:tv42,项目名称:pachyderm,代码行数:35,代码来源:pachyderm_test.go
示例4: doWrites
func doWrites(tb testing.TB, apiClient pfs.APIClient, repoName string, commitID string) {
var wg sync.WaitGroup
defer wg.Wait()
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
_, iErr := pfsutil.PutFile(apiClient, repoName, commitID,
fmt.Sprintf("a/b/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
require.NoError(tb, iErr)
_, iErr = pfsutil.PutFile(apiClient, repoName, commitID,
fmt.Sprintf("a/c/file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
require.NoError(tb, iErr)
}()
}
}
开发者ID:Antobiotics,项目名称:pachyderm,代码行数:17,代码来源:testing_test.go
示例5: Write
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) error {
written, err := pfsutil.PutFile(f.fs.apiClient, f.fs.repositoryName, f.commitID, f.path, request.Offset, bytes.NewReader(request.Data))
if err != nil {
return err
}
response.Size = written
if f.size < request.Offset+int64(written) {
f.size = request.Offset + int64(written)
}
return nil
}
开发者ID:sr,项目名称:pachyderm,代码行数:11,代码来源:filesystem.go
示例6: TestGrep
func TestGrep(t *testing.T) {
t.Skip()
t.Parallel()
dataRepo := uniqueString("TestGrep.data")
pfsClient := getPfsClient(t)
require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
require.NoError(t, err)
for i := 0; i < 100; i++ {
_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
require.NoError(t, err)
}
require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
ppsClient := getPpsClient(t)
job1, err := ppsutil.CreateJob(
ppsClient,
"",
[]string{"bash"},
fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
1,
[]*pps.JobInput{{Commit: commit}},
"",
)
require.NoError(t, err)
job2, err := ppsutil.CreateJob(
ppsClient,
"",
[]string{"bash"},
fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
4,
[]*pps.JobInput{{Commit: commit}},
"",
)
require.NoError(t, err)
inspectJobRequest := &pps.InspectJobRequest{
Job: job1,
BlockOutput: true,
BlockState: true,
}
job1Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
require.NoError(t, err)
inspectJobRequest.Job = job2
job2Info, err := ppsClient.InspectJob(context.Background(), inspectJobRequest)
require.NoError(t, err)
repo1Info, err := pfsutil.InspectRepo(pfsClient, job1Info.OutputCommit.Repo.Name)
require.NoError(t, err)
repo2Info, err := pfsutil.InspectRepo(pfsClient, job2Info.OutputCommit.Repo.Name)
require.NoError(t, err)
require.Equal(t, repo1Info.SizeBytes, repo2Info.SizeBytes)
}
开发者ID:tottok-ug,项目名称:pachyderm,代码行数:50,代码来源:pachyderm_test.go
示例7: Write
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) (retErr error) {
defer func() {
protolog.Info(&FileWrite{&f.Node, errorToString(retErr)})
}()
written, err := pfsutil.PutFile(f.fs.apiClient, f.File.Commit.Repo.Name, f.File.Commit.Id, f.File.Path, request.Offset, bytes.NewReader(request.Data))
if err != nil {
return err
}
response.Size = written
if f.size < request.Offset+int64(written) {
f.size = request.Offset + int64(written)
}
return nil
}
开发者ID:klucar,项目名称:pachyderm,代码行数:14,代码来源:filesystem.go
示例8: TestSharding
func TestSharding(t *testing.T) {
t.Parallel()
repo := uniqueString("TestSharding")
pachClient := getPachClient(t)
err := pfsutil.CreateRepo(pachClient, repo)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(pachClient, repo, "")
require.NoError(t, err)
var wg sync.WaitGroup
for i := 0; i < NUMFILES; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
rand := rand.New(rand.NewSource(int64(i)))
_, err = pfsutil.PutFile(pachClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, KB))
require.NoError(t, err)
}()
}
wg.Wait()
err = pfsutil.FinishCommit(pachClient, repo, commit.Id)
require.NoError(t, err)
wg = sync.WaitGroup{}
for i := 0; i < NUMFILES; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
var buffer1Shard bytes.Buffer
var buffer4Shard bytes.Buffer
shard := &pfs.Shard{FileModulus: 1, BlockModulus: 1}
err := pfsutil.GetFile(pachClient, repo, commit.Id,
fmt.Sprintf("file%d", i), 0, 0, "", shard, &buffer1Shard)
require.NoError(t, err)
shard.BlockModulus = 4
for blockNumber := uint64(0); blockNumber < 4; blockNumber++ {
shard.BlockNumber = blockNumber
err := pfsutil.GetFile(pachClient, repo, commit.Id,
fmt.Sprintf("file%d", i), 0, 0, "", shard, &buffer4Shard)
require.NoError(t, err)
}
require.Equal(t, buffer1Shard.Len(), buffer4Shard.Len())
}()
}
wg.Wait()
}
开发者ID:tv42,项目名称:pachyderm,代码行数:46,代码来源:pachyderm_test.go
示例9: TestBigWrite
func TestBigWrite(t *testing.T) {
t.Parallel()
protolog.SetLevel(protolog.Level_LEVEL_DEBUG)
repo := uniqueString("TestBigWrite")
pfsClient := getPfsClient(t)
err := pfsutil.CreateRepo(pfsClient, repo)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(pfsClient, repo, "")
require.NoError(t, err)
rand := rand.New(rand.NewSource(5))
_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, "file", 0, workload.NewReader(rand, 10000))
require.NoError(t, err)
err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
require.NoError(t, err)
var buffer bytes.Buffer
err = pfsutil.GetFile(pfsClient, repo, commit.Id, "file", 0, 0, nil, &buffer)
require.NoError(t, err)
}
开发者ID:vsayer,项目名称:pachyderm,代码行数:18,代码来源:pachyderm_test.go
示例10: TestSharding
func TestSharding(t *testing.T) {
t.Parallel()
repo := uniqueString("TestSharding")
pfsClient := getPfsClient(t)
err := pfsutil.CreateRepo(pfsClient, repo)
require.NoError(t, err)
commit, err := pfsutil.StartCommit(pfsClient, repo, "")
require.NoError(t, err)
var wg sync.WaitGroup
for i := 0; i < NUMFILES; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
rand := rand.New(rand.NewSource(int64(i)))
_, err = pfsutil.PutFile(pfsClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, 128*1024*1024))
require.NoError(t, err)
}()
}
wg.Wait()
err = pfsutil.FinishCommit(pfsClient, repo, commit.Id)
require.NoError(t, err)
}
开发者ID:tottok-ug,项目名称:pachyderm,代码行数:23,代码来源:pachyderm_test.go
示例11: TestGrep
func TestGrep(t *testing.T) {
t.Parallel()
dataRepo := uniqueString("pachyderm.TestGrep.data")
pfsClient := getPfsClient(t)
require.NoError(t, pfsutil.CreateRepo(pfsClient, dataRepo))
commit, err := pfsutil.StartCommit(pfsClient, dataRepo, "")
require.NoError(t, err)
for i := 0; i < 100; i++ {
_, err = pfsutil.PutFile(pfsClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
require.NoError(t, err)
}
require.NoError(t, pfsutil.FinishCommit(pfsClient, dataRepo, commit.Id))
ppsClient := getPpsClient(t)
_, err = ppsutil.CreateJob(
ppsClient,
"",
[]string{"bash"},
fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
1,
[]*pps.JobInput{{Commit: commit}},
"",
)
require.NoError(t, err)
}
开发者ID:vsayer,项目名称:pachyderm,代码行数:24,代码来源:pachyderm_test.go
示例12: testMount
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient, cluster Cluster) {
repositoryName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repositoryName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter(apiClient)
err = mounter.Mount(repositoryName, directory, "", 0, 1)
require.NoError(t, err)
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
err = mounter.Wait(directory)
require.NoError(t, err)
}
开发者ID:sr,项目名称:pachyderm,代码行数:67,代码来源:testing_test.go
示例13: testMount
func testMount(t *testing.T, apiClient pfs.APIClient, cluster Cluster) {
repoName := "testMountRepo"
err := pfsutil.CreateRepo(apiClient, repoName)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter("localhost", apiClient)
ready := make(chan bool)
go func() {
err = mounter.Mount(directory, &pfs.Shard{Number: 0, Modulus: 1}, nil, ready)
require.NoError(t, err)
}()
<-ready
_, err = os.Stat(filepath.Join(directory, repoName))
require.NoError(t, err)
commit, err := pfsutil.StartCommit(apiClient, repoName, "")
require.NoError(t, err)
require.NotNil(t, commit)
newCommitID := commit.Id
_, err = os.Stat(filepath.Join(directory, repoName, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repoName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, repoName, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repoName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.FinishCommit(apiClient, repoName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, repoName, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, repoName, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
err = mounter.Unmount(directory)
require.NoError(t, err)
}
开发者ID:Antobiotics,项目名称:pachyderm,代码行数:69,代码来源:testing_test.go
示例14: Cmds
//.........这里部分代码省略.........
Short: "Delete a commit.",
Long: "Delete a commit.",
Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
apiClient, err := getAPIClient(address)
if err != nil {
return err
}
return pfsutil.DeleteCommit(apiClient, args[0], args[1])
}),
}
mkdir := &cobra.Command{
Use: "mkdir repo-name commit-id path/to/dir",
Short: "Make a directory.",
Long: "Make a directory. Parent directories need not exist.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
apiClient, err := getAPIClient(address)
if err != nil {
return err
}
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
}),
}
putFile := &cobra.Command{
Use: "put-file repo-name commit-id path/to/file",
Short: "Put a file from stdin",
Long: "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
apiClient, err := getAPIClient(address)
if err != nil {
return err
}
_, err = pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
}),
}
getFile := &cobra.Command{
Use: "get-file repo-name commit-id path/to/file",
Short: "Return the contents of a file.",
Long: "Return the contents of a file.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
apiClient, err := getAPIClient(address)
if err != nil {
return err
}
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, shard(), os.Stdout)
}),
}
getFile.Flags().IntVarP(&number, "shard", "s", 0, "shard to read from")
getFile.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
inspectFile := &cobra.Command{
Use: "inspect-file repo-name commit-id path/to/file",
Short: "Return info about a file.",
Long: "Return info about a file.",
Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
apiClient, err := getAPIClient(address)
if err != nil {
return err
}
fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2], shard())
if err != nil {
return err
}
开发者ID:angelabier1,项目名称:pachyderm,代码行数:67,代码来源:cmds.go
示例15: testReplica
func testReplica(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient) {
masterName := TestRepositoryName()
err := pfsutil.InitRepository(apiClient, masterName, false)
require.NoError(t, err)
replicaName := TestRepositoryName()
err = pfsutil.InitRepository(apiClient, replicaName, true)
require.NoError(t, err)
for i := 0; i < testShardsPerServer*testNumServers; i++ {
var buffer bytes.Buffer
err := pfsutil.PullDiff(internalAPIClient, masterName, "scratch", uint64(i), &buffer)
require.NoError(t, err)
err = pfsutil.PushDiff(internalAPIClient, replicaName, "scratch", uint64(i), &buffer)
require.NoError(t, err)
}
getCommitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, replicaName, "scratch")
require.NoError(t, err)
require.NotNil(t, getCommitInfoResponse)
require.NotNil(t, getCommitInfoResponse.CommitInfo)
require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)
branchResponse, err := pfsutil.Branch(apiClient, masterName, "scratch")
require.NoError(t, err)
require.NotNil(t, branchResponse)
newCommitID := branchResponse.Commit.Id
var wg sync.WaitGroup
for i := 0; i < testSize; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
_, iErr := pfsutil.PutFile(apiClient, masterName, newCommitID,
fmt.Sprintf("file%d", i), 0, strings.NewReader(fmt.Sprintf("hello%d", i)))
require.NoError(t, iErr)
}()
}
wg.Wait()
err = pfsutil.Commit(apiClient, masterName, newCommitID)
require.NoError(t, err)
for i := 0; i < testShardsPerServer*testNumServers; i++ {
var buffer bytes.Buffer
err := pfsutil.PullDiff(internalAPIClient, masterName, newCommitID, uint64(i), &buffer)
require.NoError(t, err)
err = pfsutil.PushDiff(internalAPIClient, replicaName, newCommitID, uint64(i), &buffer)
require.NoError(t, err)
}
getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, replicaName, newCommitID)
require.NoError(t, err)
require.NotNil(t, getCommitInfoResponse)
require.NotNil(t, getCommitInfoResponse.CommitInfo)
require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)
}
开发者ID:henrylee2cn,项目名称:pachyderm,代码行数:62,代码来源:testing_test.go
示例16: do
//.........这里部分代码省略.........
NumArgs: 0,
Run: func(cmd *cobra.Command, args []string) error {
blockInfos, err := pfsutil.ListBlock(apiClient, uint64(shard), uint64(modulus))
if err != nil {
return err
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintBlockInfoHeader(writer)
for _, blockInfo := range blockInfos {
pretty.PrintBlockInfo(writer, blockInfo)
}
return writer.Flush()
},
}.ToCobraCommand()
listBlock.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
listBlock.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
mkdir := cobramainutil.Command{
Use: "mkdir repo-name commit-id path/to/dir",
Short: "Make a directory.",
Long: "Make a directory. Parent directories need not exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
},
}.ToCobraCommand()
putFile := cobramainutil.Command{
Use: "put-file repo-name commit-id path/to/file",
Short: "Put a file from stdin",
Long: "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
},
}.ToCobraCommand()
getFile := cobramainutil.Command{
Use: "get-file repo-name commit-id path/to/file",
Short: "Return the contents of a file.",
Long: "Return the contents of a file.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, os.Stdout)
},
}.ToCobraCommand()
inspectFile := cobramainutil.Command{
Use: "inspect-file repo-name commit-id path/to/file",
Short: "Return info about a file.",
Long: "Return info about a file.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2])
if err != nil {
return err
}
if fileInfo == nil {
return fmt.Errorf("file %s not found", args[2])
}
writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
pretty.PrintFileInfoHeader(writer)
pretty.PrintFileInfo(writer, fileInfo)
return writer.Flush()
},
开发者ID:sr,项目名称:pachyderm,代码行数:67,代码来源:main.go
示例17: testMount
func testMount(t *testing.T, apiClient pfs.ApiClient, internalAPIClient pfs.InternalApiClient) {
repositoryName := TestRepositoryName()
err := pfsutil.InitRepository(apiClient, repositoryName, false)
require.NoError(t, err)
directory := "/compile/testMount"
mounter := fuse.NewMounter()
go func() {
err = mounter.Mount(apiClient, repositoryName, directory, 0, 1)
require.NoError(t, err)
}()
mounter.Ready()
_, err = os.Stat(filepath.Join(directory, "scratch"))
require.NoError(t, err)
branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
require.NoError(t, err)
require.NotNil(t, branchResponse)
newCommitID := branchResponse.Commit.Id
_, err = os.Stat(filepath.Join(directory, newCommitID))
require.NoError(t, err)
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", 0, strings.NewReader("bar"))
require.NoError(t, err)
bigValue := make([]byte, 1024*1024)
for i := 0; i < 1024*1024; i++ {
bigValue[i] = 'a'
}
err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "big1"), bigValue, 0666)
require.NoError(t, err)
_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "big2", 0, bytes.NewReader(bigValue))
require.NoError(t, err)
err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
require.NoError(t, err)
fInfo, err := os.Stat(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, int64(3), fInfo.Size())
data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
require.NoError(t, err)
require.Equal(t, "bar", string(data))
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big1"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "big2"))
require.NoError(t, err)
require.Equal(t, bigValue, data)
}
开发者ID:henrylee2cn,项目名称:pachyderm,代码行数:65,代码来源:testing_test.go
示例18: do
func do(appEnvObj interface{}) error {
appEnv := appEnvObj.(*appEnv)
clientConn, err := grpc.Dial(appEnv.Address)
if err != nil {
return err
}
apiClient := pfs.NewApiClient(clientConn)
var shard int
var modulus int
initCmd := cobramainutil.Command{
Use: "init repository-name",
Long: "Initalize a repository.",
NumArgs: 1,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.InitRepository(apiClient, args[0], false)
},
}.ToCobraCommand()
mkdirCmd := cobramainutil.Command{
Use: "mkdir repository-name commit-id path/to/dir",
Long: "Make a directory. Sub directories must already exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
},
}.ToCobraCommand()
putCmd := cobramainutil.Command{
Use: "put repository-name branch-id path/to/file",
Long: "Put a file from stdin. Directories must exist. branch-id must be a writeable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
_, err := pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
return err
},
}.ToCobraCommand()
getCmd := cobramainutil.Command{
Use: "get repository-name commit-id path/to/file",
Long: "Get a file from stdout. commit-id must be a readable commit.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, pfsutil.GetAll, os.Stdout)
},
}.ToCobraCommand()
lsCmd := cobramainutil.Command{
Use: "ls repository-name branch-id path/to/dir",
Long: "List a directory. Directory must exist.",
NumArgs: 3,
Run: func(cmd *cobra.Command, args []string) error {
listFilesResponse, err := pfsutil.ListFiles(apiClient, args[0], args[1], args[2], uint64(shard), uint64(modulus))
if err != nil {
return err
}
for _, fileInfo := range listFilesResponse.FileInfo {
fmt.Printf("%+v\n", fileInfo)
}
return nil
},
}.ToCobraCommand()
lsCmd.Flags().IntVarP(&shard, "shard", "s", 0, "shard to read from")
lsCmd.Flags().IntVarP(&modulus, "modulus", "m", 1, "modulus of the shards")
branchCmd := cobramainutil.Command{
Use: "branch repository-name commit-id",
Long: "Branch a commit. commit-id must be a readable commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
branchResponse, err := pfsutil.Branch(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Println(branchResponse.Commit.Id)
return nil
},
}.ToCobraCommand()
commitCmd := cobramainutil.Command{
Use: "commit repository-name branch-id",
Long: "Commit a branch. branch-id must be a writeable commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
return pfsutil.Commit(apiClient, args[0], args[1])
},
}.ToCobraCommand()
commitInfoCmd := cobramainutil.Command{
Use: "commit-info repository-name commit-id",
Long: "Get info for a commit.",
NumArgs: 2,
Run: func(cmd *cobra.Command, args []string) error {
commitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, args[0], args[1])
if err != nil {
return err
}
fmt.Printf("%+v\n", commitInfoResponse.CommitInfo)
//.........这里部分代码省略.........
开发者ID:henrylee2cn,项目名称:pachyderm,代码行数:101,代码来源:main.go
示例19: work
func (w *worker) work(pfsClient pfs.APIClient, ppsClient pps.APIClient) error {
opt := w.rand.Float64()
switch {
case opt < repo:
repoName := w.randString(10)
if err := pfsutil.CreateRepo(pfsClient, repoName); err != nil {
return err
}
w.repos = append(w.repos, &pfs.Repo{Name: repoName})
commit, err := pfsutil.StartCommit(pfsClient, repoName, "")
if err != nil {
return err
}
w.started = append(w.started, commit)
case opt < commit:
if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
if len(w.started) == 0 {
return nil
}
i := w.rand.Intn(len(w.started))
commit := w.started[i]
if err := pfsutil.FinishCommit(pfsClient, commit.Repo.Name, commit.Id); err != nil {
return err
}
w.started = append(w.started[:i], w.started[i+1:]...)
w.finished = append(w.finished, commit)
} else {
if len(w.finished) == 0 {
return nil
}
commit := w.finished[w.rand.Intn(len(w.finished))]
commit, err := pfsutil.StartCommit(pfsClient, commit.Repo.Name, commit.Id)
if err != nil {
return err
}
w.started = append(w.started, commit)
}
case opt < file:
if len(w.started) == 0 {
return nil
}
commit := w.started[w.rand.Intn(len(w.started))]
if _, err := pfsutil.PutFile(pfsClient, commit.Repo.Name, commit.Id, w.randString(10), 0, w.reader()); err != nil {
return err
}
case opt < job:
if len(w.startedJobs) >= maxStartedJobs {
job := w.startedJobs[0]
w.startedJobs = w.startedJobs[1:]
jobInfo, err := ppsClient.InspectJob(
context.Background(),
&pps.InspectJobRequest{
Job: job,
BlockState: true,
},
)
if err != nil {
return err
}
if jobInfo.State != pps.JobState_JOB_STATE_SUCCESS {
return fmt.Errorf("job %s failed", job.Id)
}
w.jobs = append(w.jobs, job)
} else {
if len(w.finished) == 0 {
return nil
}
inputs := [5]string{}
var jobInputs []*pps.JobInput
repoSet := make(map[string]bool)
for i := range inputs {
commit := w.finished[w.rand.Intn(len(w.finished))]
if _, ok := repoSet[commit.Repo.Name]; ok {
continue
}
repoSet[commit.Repo.Name] = true
inputs[i] = commit.Repo.Name
jobInputs = append(jobInputs, &pps.JobInput{Commit: commit})
}
var parentJobID string
if len(w.jobs) > 0 {
parentJobID = w.jobs[w.rand.Intn(len(w.jobs))].Id
}
outFilename := w.randString(10)
job, err := ppsutil.CreateJob(
ppsClient,
"",
[]string{"bash"},
w.grepCmd(inputs, outFilename),
1,
jobInputs,
parentJobID,
)
if err != nil {
return err
}
w.startedJobs = append(w.startedJobs, job)
}
case opt < pipeline:
if len(w.repos) == 0 {
//.........这里部分代码省略.........
开发者ID:Manikandan-Selvaganesh,项目名称:pachyderm,代码行数:101,代码来源:workload.go
示例20: TestPipeline
func TestPipeline(t *testing.T) {
t.Parallel()
pachClient := getPachClient(t)
// create repos
dataRepo := uniqueString("TestPipeline.data")
require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
// create pipeline
pipelineName := uniqueString("pipeline")
outRepo := pps.PipelineRepo(ppsutil.NewPipeline(pipelineName))
require.NoError(t, ppsutil.CreatePipeline(
pachClient,
pipelineName,
"",
[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
nil,
1,
[]*pps.PipelineInput{{Repo: &pfs.Repo{Name: dataRepo}}},
))
// Do first commit to repo
commit1, err := pfsutil.StartCommit(pachClient, dataRepo, "")
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, dataRepo, commit1.Id, "file", 0, strings.NewReader("foo\n"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit1.Id))
listCommitRequest := &pfs.ListCommitRequest{
Repo: []*pfs.Repo{outRepo},
CommitType: pfs.CommitType_COMMIT_TYPE_READ,
Block: true,
}
listCommitResponse, err := pachClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
outCommits := listCommitResponse.CommitInfo
require.Equal(t, 1, len(outCommits))
var buffer bytes.Buffer
require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, "foo\n", buffer.String())
// Do second commit to repo
commit2, err := pfsutil.StartCommit(pachClient, dataRepo, commit1.Id)
require.NoError(t, err)
_, err = pfsutil.PutFile(pachClient, dataRepo, commit2.Id, "file", 0, strings.NewReader("bar\n"))
require.NoError(t, err)
require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit2.Id))
listCommitRequest = &pfs.ListCommitRequest{
Repo: []*pfs.Repo{outRepo},
FromCommit: []*pfs.Commit{outCommits[0].Commit},
CommitType: pfs.CommitType_COMMIT_TYPE_READ,
Block: true,
}
listCommitResponse, err = pachClient.ListCommit(
context.Background(),
listCommitRequest,
)
require.NoError(t, err)
require.NotNil(t, listCommitResponse.CommitInfo[0].ParentCommit)
require.Equal(t, outCommits[0].Commit.Id, listCommitResponse.CommitInfo[0].ParentCommit.Id)
outCommits = listCommitResponse.CommitInfo
require.Equal(t, 1, len(outCommits))
buffer = bytes.Buffer{}
require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, "", nil, &buffer))
require.Equal(t, "foo\nbar\n", buffer.String())
}
开发者ID:tv42,项目名称:pachyderm,代码行数:64,代码来源:pachyderm_test.go
注:本文中的github.com/pachyderm/pachyderm/src/pfs/pfsutil.PutFile函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论