simplify mapreduce code

This commit is contained in:
kevin
2020-09-16 16:48:51 +08:00
parent 68335ada54
commit df37597ac3
2 changed files with 6 additions and 16 deletions

View File

@@ -79,7 +79,7 @@ func Map(generate GenerateFunc, mapper MapFunc, opts ...Option) chan interface{}
collector := make(chan interface{}, options.workers)
done := syncx.NewDoneChan()
go mapDispatcher(mapper, source, collector, done.Done(), options.workers)
go executeMappers(mapper, source, collector, done.Done(), options.workers)
return collector
}
@@ -126,7 +126,10 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
reducer(collector, writer, cancel)
drain(collector)
}()
go mapperDispatcher(mapper, source, collector, done.Done(), cancel, options.workers)
go executeMappers(func(item interface{}, writer Writer) {
mapper(item, writer, cancel)
}, source, collector, done.Done(), options.workers)
value, ok := <-output
if err := retErr.Load(); err != nil {
@@ -226,20 +229,6 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
}
}
func mapDispatcher(mapper MapFunc, input <-chan interface{}, collector chan<- interface{},
done <-chan lang.PlaceholderType, workers int) {
executeMappers(func(item interface{}, writer Writer) {
mapper(item, writer)
}, input, collector, done, workers)
}
func mapperDispatcher(mapper MapperFunc, input <-chan interface{}, collector chan<- interface{},
done <-chan lang.PlaceholderType, cancel func(error), workers int) {
executeMappers(func(item interface{}, writer Writer) {
mapper(item, writer, cancel)
}, input, collector, done, workers)
}
func newOptions() *mapReduceOptions {
return &mapReduceOptions{
workers: defaultWorkers,