fix: goroutine stuck on edge case (#1495)

* fix: goroutine stuck on edge case

* refactor: simplify mapreduce implementation
This commit is contained in:
Kevin Wan
2022-01-30 13:09:21 +08:00
committed by GitHub
parent 14a902c1a7
commit 6c2abe7474
3 changed files with 115 additions and 19 deletions

View File

@@ -289,33 +289,21 @@ func drain(channel <-chan interface{}) {
func executeMappers(mCtx mapperContext) {
var wg sync.WaitGroup
pc := &onceChan{channel: make(chan interface{})}
defer func() {
// in case panic happens when processing last item, for loop not handling it.
select {
case r := <-pc.channel:
mCtx.panicChan.write(r)
default:
}
wg.Wait()
close(mCtx.collector)
drain(mCtx.source)
}()
var failed int32
pool := make(chan lang.PlaceholderType, mCtx.workers)
writer := newGuardedWriter(mCtx.ctx, mCtx.collector, mCtx.doneChan)
for {
for atomic.LoadInt32(&failed) == 0 {
select {
case <-mCtx.ctx.Done():
return
case <-mCtx.doneChan:
return
case r := <-pc.channel:
// make sure this method quit ASAP,
// without this case branch, all the items from source will be consumed.
mCtx.panicChan.write(r)
return
case pool <- lang.Placeholder:
item, ok := <-mCtx.source
if !ok {
@@ -327,7 +315,8 @@ func executeMappers(mCtx mapperContext) {
go func() {
defer func() {
if r := recover(); r != nil {
pc.write(r)
atomic.AddInt32(&failed, 1)
mCtx.panicChan.write(r)
}
wg.Done()
<-pool