本文整理汇总了Golang中github.com/sburnett/transformer.GroupRecords函数的典型用法代码示例。如果您正苦于以下问题:Golang GroupRecords函数的具体用法?Golang GroupRecords怎么用?Golang GroupRecords使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了GroupRecords函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: joinMacWithFlowId
func joinMacWithFlowId(inputChan, outputChan chan *store.Record) {
var (
session SessionKey
localIp []byte
)
grouper := transformer.GroupRecords(inputChan, &session, &localIp)
for grouper.NextGroup() {
var macAddress []byte
for grouper.NextRecord() {
record := grouper.Read()
switch record.DatabaseIndex {
case 0:
lex.DecodeOrDie(record.Value, &macAddress)
case 1:
if macAddress != nil {
var (
remoteIp []byte
sequenceNumber int32
timestamp int64
flowId int32
)
lex.DecodeOrDie(record.Key, &sequenceNumber, &remoteIp, ×tamp, &flowId)
outputChan <- &store.Record{
Key: lex.EncodeOrDie(&session, macAddress, remoteIp, timestamp, int64(math.MaxInt64), sequenceNumber, flowId),
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:30,代码来源:bytesperdomain.go
示例2: joinDomainsWithWhitelist
func joinDomainsWithWhitelist(inputChan, outputChan chan *store.Record) {
var session SessionKey
grouper := transformer.GroupRecords(inputChan, &session)
for grouper.NextGroup() {
var whitelist []string
for grouper.NextRecord() {
record := grouper.Read()
switch record.DatabaseIndex {
case 0:
lex.DecodeOrDie(record.Value, &whitelist)
sort.Sort(sort.StringSlice(whitelist))
case 1:
if whitelist == nil {
continue
}
var domain string
remainder := lex.DecodeOrDie(record.Key, &domain)
for i := 0; i < len(domain); i++ {
if i > 0 && domain[i-1] != '.' {
continue
}
idx := sort.SearchStrings(whitelist, domain[i:])
if idx >= len(whitelist) || whitelist[idx] != domain[i:] {
continue
}
outputChan <- &store.Record{
Key: lex.Concatenate(grouper.CurrentGroupPrefix, remainder, lex.EncodeOrDie(whitelist[idx])),
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:34,代码来源:bytesperdomain.go
示例3: summarizeFilesystemUsage
func summarizeFilesystemUsage(inputChan, outputChan chan *store.Record) {
var filesystem string
var timestamp int64
grouper := transformer.GroupRecords(inputChan, &filesystem, ×tamp)
for grouper.NextGroup() {
usage := make(map[string]int64)
for grouper.NextRecord() {
record := grouper.Read()
var node string
lex.DecodeOrDie(record.Key, &node)
var used int64
lex.DecodeOrDie(record.Value, &used)
if used > usage[node] {
usage[node] = used
}
}
for node, used := range usage {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(filesystem, timestamp, node),
Value: lex.EncodeOrDie(used),
}
}
}
}
开发者ID:sburnett,项目名称:bismark-tools,代码行数:25,代码来源:summarize.go
示例4: joinAddressIdsWithMacAddresses
func joinAddressIdsWithMacAddresses(inputChan, outputChan chan *store.Record) {
var (
session SessionKey
addressId int32
)
grouper := transformer.GroupRecords(inputChan, &session, &addressId)
for grouper.NextGroup() {
var macAddress []byte
for grouper.NextRecord() {
record := grouper.Read()
switch record.DatabaseIndex {
case 0:
macAddress = record.Value
case 1:
if macAddress != nil {
var unusedSequenceNumber int32
remainder := lex.DecodeOrDie(record.Key, &unusedSequenceNumber)
outputChan <- &store.Record{
Key: lex.Concatenate(lex.EncodeOrDie(&session), macAddress, remainder),
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:25,代码来源:bytesperdomain.go
示例5: flattenLookupsToNodeMacAndTimestamp
func flattenLookupsToNodeMacAndTimestamp(inputChan, outputChan chan *store.Record) {
var nodeId, macAddress, domain string
grouper := transformer.GroupRecords(inputChan, &nodeId, &macAddress, &domain)
for grouper.NextGroup() {
totalCounts := make(map[int64]int64)
for grouper.NextRecord() {
record := grouper.Read()
var (
anonymizationContext string
sessionId int64
sequenceNumber int32
)
lex.DecodeOrDie(record.Key, &anonymizationContext, &sessionId, &sequenceNumber)
var count int64
lex.DecodeOrDie(record.Value, &count)
timestamp := truncateTimestampToHour(sessionId + convertSecondsToMicroseconds(30)*int64(sequenceNumber))
totalCounts[timestamp] += count
}
for timestamp, totalCount := range totalCounts {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(nodeId, macAddress, domain, timestamp),
Value: lex.EncodeOrDie(totalCount),
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:26,代码来源:lookupsperdevice.go
示例6: joinMacWithLookups
func joinMacWithLookups(inputChan, outputChan chan *store.Record) {
var (
session SessionKey
addressId int32
)
grouper := transformer.GroupRecords(inputChan, &session, &addressId)
for grouper.NextGroup() {
var macAddress []byte
for grouper.NextRecord() {
record := grouper.Read()
switch record.DatabaseIndex {
case 0:
lex.DecodeOrDie(record.Value, &macAddress)
case 1:
if macAddress != nil {
var (
sequenceNumber int32
domain string
)
lex.DecodeOrDie(record.Key, &sequenceNumber, &domain)
outputChan <- &store.Record{
Key: lex.EncodeOrDie(session.NodeId, macAddress, domain, session.AnonymizationContext, session.SessionId, sequenceNumber),
Value: record.Value,
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:29,代码来源:lookupsperdevice.go
示例7: aggregateStatisticsReducer
func aggregateStatisticsReducer(inputChan, outputChan chan *store.Record) {
var nodeId []byte
grouper := transformer.GroupRecords(inputChan, &nodeId)
for grouper.NextGroup() {
aggregateStatistics := newAggregateStatistics()
for grouper.NextRecord() {
record := grouper.Read()
var statistics AggregateStatistics
if err := proto.Unmarshal(record.Value, &statistics); err != nil {
panic(err)
}
*aggregateStatistics.Traces += *statistics.Traces
*aggregateStatistics.Packets += *statistics.Packets
*aggregateStatistics.PacketSeriesDropped += *statistics.PacketSeriesDropped
*aggregateStatistics.PcapDropped += *statistics.PcapDropped
*aggregateStatistics.InterfaceDropped += *statistics.InterfaceDropped
*aggregateStatistics.Flows += *statistics.Flows
*aggregateStatistics.DroppedFlows += *statistics.DroppedFlows
*aggregateStatistics.Bytes += *statistics.Bytes
}
encodedStatistics, err := proto.Marshal(aggregateStatistics)
if err != nil {
panic(err)
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(nodeId),
Value: encodedStatistics,
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:30,代码来源:statistics.go
示例8: inferReboots
func inferReboots(inputChan, outputChan chan *store.Record) {
var node string
grouper := transformer.GroupRecords(inputChan, &node)
for grouper.NextGroup() {
lastUptime := int64(-1)
maxReboot := int64(-1)
for grouper.NextRecord() {
record := grouper.Read()
var timestamp int64
lex.DecodeOrDie(record.Key, ×tamp)
var uptime int64
lex.DecodeOrDie(record.Value, &uptime)
if lastUptime >= 0 && lastUptime > uptime {
if maxReboot > -1 {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(node, maxReboot),
}
}
maxReboot = int64(-1)
}
reboot := timestamp - uptime
if maxReboot < reboot {
maxReboot = reboot
}
lastUptime = uptime
}
if maxReboot > -1 {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(node, maxReboot),
}
}
}
}
开发者ID:sburnett,项目名称:bismark-tools,代码行数:35,代码来源:reboots.go
示例9: joinMacAndSizes
func joinMacAndSizes(inputChan, outputChan chan *store.Record) {
var session SessionKey
var flowId int32
grouper := transformer.GroupRecords(inputChan, &session, &flowId)
for grouper.NextGroup() {
var currentMacAddresses [][]byte
for grouper.NextRecord() {
record := grouper.Read()
if record.DatabaseIndex == 0 {
lex.DecodeOrDie(record.Value, ¤tMacAddresses)
continue
}
if currentMacAddresses == nil {
continue
}
var sequenceNumber int32
lex.DecodeOrDie(record.Key, &sequenceNumber)
var timestamps, sizes []int64
lex.DecodeOrDie(record.Value, ×tamps, &sizes)
if len(timestamps) != len(sizes) {
panic(fmt.Errorf("timestamps and sizes must be the same size"))
}
for _, currentMacAddress := range currentMacAddresses {
for idx, timestamp := range timestamps {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(&session, currentMacAddress, timestamp, flowId, sequenceNumber),
Value: lex.EncodeOrDie(sizes[idx]),
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:35,代码来源:bytesperdevice.go
示例10: summarizeSizesByTimestamp
func summarizeSizesByTimestamp(inputChan, outputChan chan *store.Record) {
quantileComputer := NewQuantileSample(101)
quantilesToKeep := []int{0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100}
var experiment, node string
var timestamp int64
grouper := transformer.GroupRecords(inputChan, &experiment, &node, ×tamp)
for grouper.NextGroup() {
for grouper.NextRecord() {
record := grouper.Read()
var statsValue StatsValue
lex.DecodeOrDie(record.Value, &statsValue)
quantileComputer.Append(statsValue.Size)
}
count := int64(quantileComputer.Count())
quantiles := quantileComputer.Quantiles()
quantileComputer.Reset()
statistics := []interface{}{count}
for idx := range quantilesToKeep {
statistics = append(statistics, quantiles[idx])
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(experiment, node, timestamp),
Value: lex.EncodeOrDie(statistics...),
}
}
}
开发者ID:sburnett,项目名称:bismark-tools,代码行数:30,代码来源:summary.go
示例11: summarizeInterarrivalTimes
func summarizeInterarrivalTimes(inputChan, outputChan chan *store.Record) {
quantileComputer := NewQuantileSample(21)
var experiment, node string
grouper := transformer.GroupRecords(inputChan, &experiment, &node)
for grouper.NextGroup() {
for grouper.NextRecord() {
record := grouper.Read()
var timestamp int64
lex.DecodeOrDie(record.Key, ×tamp)
quantileComputer.Append(timestamp)
}
count := int64(quantileComputer.Count())
quantiles := quantileComputer.Quantiles()
statistics := []interface{}{count}
for _, q := range quantiles {
statistics = append(statistics, q)
}
quantileComputer.Reset()
outputChan <- &store.Record{
Key: lex.EncodeOrDie(experiment, node),
Value: lex.EncodeOrDie(statistics...),
}
}
}
开发者ID:sburnett,项目名称:bismark-tools,代码行数:27,代码来源:summary.go
示例12: joinMacAndFlowId
func joinMacAndFlowId(inputChan, outputChan chan *store.Record) {
var session SessionKey
var ipAddress []byte
grouper := transformer.GroupRecords(inputChan, &session, &ipAddress)
for grouper.NextGroup() {
var currentMacAddress []byte
for grouper.NextRecord() {
record := grouper.Read()
if record.DatabaseIndex == 0 {
currentMacAddress = record.Value
continue
}
if currentMacAddress == nil {
continue
}
var sequenceNumber int32
lex.DecodeOrDie(record.Key, &sequenceNumber)
var flowIds []int32
lex.DecodeOrDie(record.Value, &flowIds)
for _, flowId := range flowIds {
outputChan <- &store.Record{
Key: lex.Concatenate(lex.EncodeOrDie(&session, flowId, sequenceNumber), currentMacAddress),
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:27,代码来源:bytesperdevice.go
示例13: joinDomainsWithSizes
func joinDomainsWithSizes(inputChan, outputChan chan *store.Record) {
var (
session SessionKey
flowId int32
)
grouper := transformer.GroupRecords(inputChan, &session, &flowId)
for grouper.NextGroup() {
var domains, macAddresses [][]byte
for grouper.NextRecord() {
record := grouper.Read()
switch record.DatabaseIndex {
case 0:
lex.DecodeOrDie(record.Value, &domains, &macAddresses)
case 1:
if domains != nil && macAddresses != nil {
var (
sequenceNumber int32
timestamp int64
)
lex.DecodeOrDie(record.Key, &sequenceNumber, ×tamp)
for idx, domain := range domains {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(session.NodeId, domain, timestamp, macAddresses[idx], session.AnonymizationContext, session.SessionId, flowId, sequenceNumber),
Value: record.Value,
}
}
}
}
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:32,代码来源:bytesperdomain.go
示例14: aggregateStatisticsReduceBySession
func aggregateStatisticsReduceBySession(inputChan, outputChan chan *store.Record) {
var session SessionKey
grouper := transformer.GroupRecords(inputChan, &session)
for grouper.NextGroup() {
aggregateStatistics := newAggregateStatistics()
var pcapDropped, interfaceDropped int64
var lastPcapDropped, lastInterfaceDropped int64
var pcapDroppedBaseline, interfaceDroppedBaseline int64
for grouper.NextRecord() {
record := grouper.Read()
var statistics AggregateStatistics
if err := proto.Unmarshal(record.Value, &statistics); err != nil {
panic(err)
}
if lastPcapDropped > *statistics.PcapDropped {
pcapDroppedBaseline += math.MaxUint32
pcapDropped = 0
}
lastPcapDropped = *statistics.PcapDropped
pcapDropped = maxInt64(pcapDropped, *statistics.PcapDropped)
if lastInterfaceDropped > *statistics.InterfaceDropped {
interfaceDroppedBaseline += math.MaxUint32
interfaceDropped = 0
}
lastInterfaceDropped = *statistics.InterfaceDropped
interfaceDropped = maxInt64(interfaceDropped, *statistics.InterfaceDropped)
*aggregateStatistics.Traces += *statistics.Traces
*aggregateStatistics.Packets += *statistics.Packets
*aggregateStatistics.PacketSeriesDropped += *statistics.PacketSeriesDropped
*aggregateStatistics.Flows += *statistics.Flows
*aggregateStatistics.DroppedFlows += *statistics.DroppedFlows
*aggregateStatistics.Bytes += *statistics.Bytes
}
*aggregateStatistics.PcapDropped = pcapDroppedBaseline + pcapDropped
*aggregateStatistics.InterfaceDropped = interfaceDroppedBaseline + interfaceDropped
encodedStatistics, err := proto.Marshal(aggregateStatistics)
if err != nil {
panic(err)
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(&session),
Value: encodedStatistics,
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:49,代码来源:statistics.go
示例15: flattenLookupsToNodeAndMac
func flattenLookupsToNodeAndMac(inputChan, outputChan chan *store.Record) {
var nodeId, macAddress, domain string
grouper := transformer.GroupRecords(inputChan, &nodeId, &macAddress, &domain)
for grouper.NextGroup() {
var totalCount int64
for grouper.NextRecord() {
record := grouper.Read()
var count int64
lex.DecodeOrDie(record.Value, &count)
totalCount += count
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(nodeId, macAddress, domain),
Value: lex.EncodeOrDie(totalCount),
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:17,代码来源:lookupsperdevice.go
示例16: reduceBytesPerDevice
func reduceBytesPerDevice(inputChan, outputChan chan *store.Record) {
var nodeId, macAddress []byte
var timestamp int64
grouper := transformer.GroupRecords(inputChan, &nodeId, &macAddress, ×tamp)
for grouper.NextGroup() {
var totalSize int64
for grouper.NextRecord() {
record := grouper.Read()
var size int64
lex.DecodeOrDie(record.Value, &size)
totalSize += size
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(nodeId, macAddress, timestamp),
Value: lex.EncodeOrDie(totalSize),
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:18,代码来源:bytesperdevice.go
示例17: flattenMacAddresses
func flattenMacAddresses(inputChan, outputChan chan *store.Record) {
var session SessionKey
var flowId, sequenceNumber int32
grouper := transformer.GroupRecords(inputChan, &session, &flowId, &sequenceNumber)
for grouper.NextGroup() {
macAddresses := [][]byte{}
for grouper.NextRecord() {
record := grouper.Read()
var macAddress []byte
lex.DecodeOrDie(record.Key, &macAddress)
macAddresses = append(macAddresses, macAddress)
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(&session, flowId, sequenceNumber),
Value: lex.EncodeOrDie(macAddresses),
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:18,代码来源:bytesperdevice.go
示例18: reduceBytesPerDeviceSession
func reduceBytesPerDeviceSession(inputChan, outputChan chan *store.Record) {
var session SessionKey
var macAddress []byte
var timestamp int64
grouper := transformer.GroupRecords(inputChan, &session, &macAddress, ×tamp)
for grouper.NextGroup() {
var totalSize int64
for grouper.NextRecord() {
record := grouper.Read()
var size int64
lex.DecodeOrDie(record.Value, &size)
totalSize += size
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(session.NodeId, macAddress, timestamp, session.AnonymizationContext, session.SessionId),
Value: lex.EncodeOrDie(totalSize),
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:19,代码来源:bytesperdevice.go
示例19: flattenIntoBytesPerTimestamp
func flattenIntoBytesPerTimestamp(inputChan, outputChan chan *store.Record) {
var (
nodeId, domain []byte
timestamp int64
)
grouper := transformer.GroupRecords(inputChan, &nodeId, &domain, ×tamp)
for grouper.NextGroup() {
var totalSize int64
for grouper.NextRecord() {
record := grouper.Read()
var size int64
lex.DecodeOrDie(record.Value, &size)
totalSize += size
}
outputChan <- &store.Record{
Key: lex.EncodeOrDie(nodeId, domain, timestamp),
Value: lex.EncodeOrDie(totalSize),
}
}
}
开发者ID:sburnett,项目名称:bismark-passive-server-go,代码行数:20,代码来源:bytesperdomain.go
示例20: summarizeSizesPerDay
func summarizeSizesPerDay(inputChan, outputChan chan *store.Record) {
var experiment, node string
grouper := transformer.GroupRecords(inputChan, &experiment, &node)
for grouper.NextGroup() {
sizePerDay := make(map[int64]int64)
for grouper.NextRecord() {
record := grouper.Read()
var statsValue StatsValue
lex.DecodeOrDie(record.Value, &statsValue)
roundedTimestamp := truncateTimestampToDay(statsValue.ReceivedTimestamp)
sizePerDay[roundedTimestamp] += statsValue.Size
}
for timestamp, size := range sizePerDay {
outputChan <- &store.Record{
Key: lex.EncodeOrDie(experiment, node, timestamp),
Value: lex.EncodeOrDie(size),
}
}
}
}
开发者ID:sburnett,项目名称:bismark-tools,代码行数:21,代码来源:summary.go
注:本文中的github.com/sburnett/transformer.GroupRecords函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论