Golang types.SetupFunction类代码示例

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Golang中github.com/apoydence/hydra/examples/wordCount/types.SetupFunction类的典型用法代码示例。如果您正苦于以下问题：Golang SetupFunction类的具体用法？Golang SetupFunction怎么用？Golang SetupFunction使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了SetupFunction类的14个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Golang代码示例。

示例1: MimeDetector

func MimeDetector(sf types.SetupFunction) {
	in, out := sf.SetName("MimeDetector").AsFilter("UrlParser").Build()
	defer close(out)

	buffer := make([]byte, 512)

	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBM)
		body := Download(url)
		if body == nil {
			continue
		}
		func(body io.ReadCloser) {
			defer body.Close()
			n, err := body.Read(buffer)

			if err == nil || err == io.EOF {
				encoded := http.DetectContentType(buffer[:n]) + "->" + url
				out <- NewStringMarshaler(encoded)
			}
		}(body)
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:26，代码来源:webCrawler.go

示例2: UrlParser

func UrlParser(sf types.SetupFunction) {
	in, out := sf.SetName("UrlParser").AsFilter("UrlProducer").Build()
	defer close(out)

	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		urlStr := ToString(urlBM)
		if path.Ext(urlStr) == "" && urlStr[len(urlStr)-1] != '/' {
			urlStr += "/"
		}
		url, err := urlParser.Parse(urlStr)
		if err != nil {
			continue
		}

		body := Download(url.String())
		for link := range fetchLinks(body) {
			joinedLink, err := url.Parse(link)
			if err != nil {
				continue
			}

			out <- NewStringMarshaler(joinedLink.String())
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:28，代码来源:webCrawler.go

示例3: pathProducer

func pathProducer(sf types.SetupFunction, argv []string) {
	out := sf.SetName("PathProducer").AsProducer().Build()
	defer close(out)

	for _, path := range argv {
		out <- NewStringMarshaler(path)
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:8，代码来源:fileContentCounter.go

示例4: PathValidator

func PathValidator(sf types.SetupFunction, parent string) {
	in, out := sf.SetName("PathValidator").AsFilter(parent).Build()
	defer close(out)

	for path := range in {
		if _, err := os.Stat(ToString(path)); err == nil {
			out <- path
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:10，代码来源:wordCount.go

示例5: urlLooper

func urlLooper(sf types.SetupFunction, feeder chan<- interface{}, done types.AtomicBool) {
	in := sf.AsConsumer("MimeSplitterHtml").Build()

	for url := range in {
		if !done.Get() {
			u := ToString(url)
			feeder <- u
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:10，代码来源:textDownloaderMain.go

示例6: urlFeeder

func urlFeeder(sf types.SetupFunction, feeder <-chan interface{}, done types.AtomicBool) {
	out := sf.SetName("UrlFeeder").AsProducer().Build()
	defer close(out)

	for url := range feeder {
		if !done.Get() {
			out <- NewStringMarshaler(url.(string))
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:10，代码来源:textDownloaderMain.go

示例7: wordPrinter

func wordPrinter(sf types.SetupFunction, done chan struct{}) {
	defer close(done)
	in := sf.AsConsumer("github.com/apoydence/hydra/examples/wordCount.FinalWordCounter").Build()

	for wordMap := range in {
		for k, v := range ToMap(wordMap) {
			println(k, v)
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:10，代码来源:fileContentCounter.go

示例8: WordCounter

func WordCounter(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.SymbolRemover").Build()
	defer close(out)
	m := make(map[string]uint32)

	for word := range in {
		incMap(ToString(word), 1, m)
	}

	out <- NewWordCountMarshaler(m)
}

开发者ID:jmptrader，项目名称:hydra，代码行数:11，代码来源:wordCount.go

示例9: FinalWordCounter

func FinalWordCounter(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordCounter").Build()
	defer close(out)

	m := make(map[string]uint32)

	for wordMap := range in {
		for k, v := range ToMap(wordMap) {
			incMap(k, v, m)
		}
	}

	out <- NewWordCountMarshaler(m)
}

开发者ID:jmptrader，项目名称:hydra，代码行数:14，代码来源:wordCount.go

示例10: MimeSplitterHtml

func MimeSplitterHtml(sf types.SetupFunction) {
	in, out := sf.SetName("MimeSplitterHtml").AsFilter("MimeDetector").Build()
	defer close(out)
	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBM)
		mime, u := decodeMimeUrl(url)
		if strings.Contains(mime, "html") {
			out <- NewStringMarshaler(u)
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:14，代码来源:webCrawler.go

示例11: SymbolRemover

func SymbolRemover(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordExtractor").Build()
	defer close(out)

	for word := range in {
		str := make([]byte, 0)
		bytes := []byte(strings.ToLower(ToString(word)))
		for _, x := range bytes {
			if (x >= 0x30 && x <= 0x39) || (x >= 0x61 && x <= 0x7a) {
				str = append(str, x)
			}
		}
		if len(str) > 0 {
			out <- NewStringMarshaler(string(str))
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:17，代码来源:wordCount.go

示例12: WordExtractor

func WordExtractor(sf types.SetupFunction) {
	in, out := sf.AsFilter("PathValidator").Build()
	defer close(out)

	for path := range in {
		p := ToString(path)
		f, err := os.Open(p)
		if err == nil {
			scanner := bufio.NewScanner(f)
			scanner.Split(bufio.ScanWords)
			for scanner.Scan() {
				out <- NewStringMarshaler(scanner.Text())
			}
		}

	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:17，代码来源:wordCount.go

示例13: UrlProducer

func UrlProducer(sf types.SetupFunction, parent string) {
	in, out := sf.SetName("UrlProducer").AsFilter(parent).Build()
	defer close(out)

	visitedUrls := make(map[string]interface{})

	for urlBm := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBm)
		if _, visited := visitedUrls[url]; !visited {
			visitedUrls[url] = nil
			out <- NewStringMarshaler(url)
		}
	}
}

开发者ID:jmptrader，项目名称:hydra，代码行数:17，代码来源:webCrawler.go

示例14: textDownloader

func textDownloader(sf types.SetupFunction, download string, mb int, done chan struct{}, closer func()) {
	in := sf.AsConsumer("MimeSplitterText").Build()

	var totalSize int64 = 0

	for url := range in {
		if totalSize >= int64(mb)*1024*1024 {
			closer()
			continue
		}

		u := ToString(url)
		println("download", u, path.Base(u))
		totalSize += saveToFile(Download(u), path.Join(download, path.Base(u)))
		println("Size", totalSize)
	}

	close(done)
}

开发者ID:jmptrader，项目名称:hydra，代码行数:19，代码来源:textDownloaderMain.go

注：本文中的github.com/apoydence/hydra/examples/wordCount/types.SetupFunction类示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。