Golang scrapinghub.Jobs类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中github.com/scrapinghub/shubc/scrapinghub.Jobs类的典型用法代码示例。如果您正苦于以下问题：Golang Jobs类的具体用法？Golang Jobs怎么用？Golang Jobs使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了Jobs类的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: log_tailing

func log_tailing(conn *scrapinghub.Connection, job_id string) {
	var jobs scrapinghub.Jobs
	jobinfo, err := jobs.JobInfo(conn, job_id)
	if err != nil {
		log.Fatalf("%s\n", err)
	}
	// Number of log lines in the job
	offset := jobinfo.Logs
	if offset > 0 {
		offset -= 1 // start one line before
	}
	count := 10 // ask for this lines in every call
	ls := scrapinghub.LinesStream{Conn: conn, Count: count, Offset: offset}
	for {
		retrieved := 0
		ch_lines, ch_err := ls.LogLines(job_id)
		for line := range ch_lines {
			retrieved++
			fmt.Fprintf(os.Stdout, "%s\n", line)
		}
		for err := range ch_err {
			log.Fatalf("%s\n", err)
		}
		ls.Offset += retrieved
		time.Sleep(time.Second)
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:27，代码来源:shubc.go

示例2: cmd_jobs

func cmd_jobs(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <project_id>\n")
	}
	project_id := args[0]
	filters := equality_list_to_map(args[1:])

	count := flags.Count
	offset := flags.Offset

	if flags.AsJsonLines {
		ls := scrapinghub.LinesStream{Conn: conn, Count: count, Offset: offset}
		ch_jobs, errch := ls.JobsAsJsonLines(project_id, filters)
		for line := range ch_jobs {
			fmt.Println(line)
		}
		for err := range errch {
			log.Fatalf("jobs error: %s\n", err)
		}
	} else {
		var jobs scrapinghub.Jobs
		jobs_list, err := jobs.List(conn, project_id, count, filters)
		if err != nil {
			log.Fatalf("jobs error: %s", err)
		}
		outfmt := "| %10s | %25s | %12s | %10s | %10s | %10s | %20s |\n"
		print_out(flags, outfmt, "id", "spider", "state", "items", "errors", "log lines", "started_time")
		print_out(flags, dashes(106))
		for _, j := range jobs_list.Jobs {
			print_out(flags, "| %10s | %25s | %12s | %10d | %10d | %10d | %20s |\n", j.Id, j.Spider, j.State,
				j.ItemsScraped, j.ErrorsCount, j.Logs, j.StartedTime)
		}
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:34，代码来源:shubc.go

示例3: cmd_jobs_stop

func cmd_jobs_stop(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <job_id>\n")
	}
	var jobs scrapinghub.Jobs
	job_id := args[0]
	err := jobs.Stop(conn, job_id)
	if err != nil {
		log.Fatalf("stop error: %s\n", err)
	} else {
		fmt.Printf("Stopped job: %s\n", job_id)
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:13，代码来源:shubc.go

示例4: cmd_reschedule

func cmd_reschedule(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <job_id>\n")
	}
	job_id := args[0]

	var jobs scrapinghub.Jobs
	new_job_id, err := jobs.Reschedule(conn, job_id)
	if err != nil {
		log.Fatalf("reschedule error: %s\n", err)
	} else {
		fmt.Printf("Re-scheduled job new id: %s\n", new_job_id)
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:14，代码来源:shubc.go

示例5: cmd_jobs_update

func cmd_jobs_update(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <job_id>\n")
	}

	var jobs scrapinghub.Jobs
	job_id := args[0]
	update_data := equality_list_to_map(args[1:])

	err := jobs.Update(conn, job_id, update_data)
	if err != nil {
		log.Fatalf("update error: %s\n", err)
	} else {
		fmt.Printf("Updated job: %s\n", job_id)
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:16，代码来源:shubc.go

示例6: cmd_schedule

func cmd_schedule(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 2 {
		log.Fatalf("Missing arguments: <project_id> and <spider_name>\n")
	}
	var jobs scrapinghub.Jobs
	project_id := args[0]
	spider_name := args[1]
	spider_args := equality_list_to_map(args[2:])
	job_id, err := jobs.Schedule(conn, project_id, spider_name, spider_args)

	if err != nil {
		log.Fatalf("schedule error: %s\n", err)
	} else {
		fmt.Printf("Scheduled job: %s\n", job_id)
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:16，代码来源:shubc.go

示例7: cmd_jobinfo

func cmd_jobinfo(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <job_id>\n")
	}
	job_id := args[0]

	var jobs scrapinghub.Jobs
	jobinfo, err := jobs.JobInfo(conn, job_id)

	if err != nil {
		log.Fatalf("jobinfo error: %s\n", err)
	} else {
		outfmt := "| %-30s | %60s |\n"
		print_out(flags, outfmt, "key", "value")
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "id", jobinfo.Id)
		print_out(flags, outfmt, "spider", jobinfo.Spider)
		print_out(flags, outfmt, "spider_args", "")
		for k, v := range jobinfo.SpiderArgs {
			print_out(flags, outfmt, " ", fmt.Sprintf("%s = %s", k, v))
		}
		print_out(flags, outfmt, "spider_type", jobinfo.SpiderType)
		print_out(flags, outfmt, "state", jobinfo.State)
		print_out(flags, outfmt, "close_reason", jobinfo.CloseReason)
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "responses_received", fmt.Sprintf("%d", jobinfo.ResponsesReceived))
		print_out(flags, outfmt, "items_scraped", fmt.Sprintf("%d", jobinfo.ItemsScraped))
		print_out(flags, outfmt, "errors_count", fmt.Sprintf("%d", jobinfo.ErrorsCount))
		print_out(flags, outfmt, "logs", fmt.Sprintf("%d", jobinfo.Logs))
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "started_time", jobinfo.StartedTime)
		print_out(flags, outfmt, "updated_time", jobinfo.UpdatedTime)
		print_out(flags, outfmt, "elapsed", fmt.Sprintf("%d", jobinfo.Elapsed))
		print_out(flags, outfmt, "tags", " ")
		for _, e := range jobinfo.Tags {
			print_out(flags, outfmt, " ", e)
		}
		print_out(flags, outfmt, "priority", fmt.Sprintf("%d", jobinfo.Priority))
		print_out(flags, outfmt, "version", jobinfo.Version)
		print_out(flags, dashes(97))
	}
}

开发者ID:kevinxusz，项目名称:shubc，代码行数:42，代码来源:shubc.go

注：本文中的github.com/scrapinghub/shubc/scrapinghub.Jobs类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。