zeromicro/go-zero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: mapreduce generic version (#2827)

Browse Source 
* feat: mapreduce generic version

* fix: gateway mr type issue

---------

Co-authored-by: kevin.wan <kevin.wan@yijinin.com>
This commit is contained in:
Kevin Wan
2023-01-29 18:01:23 +08:00
committed by GitHub
parent 413ee919e6
commit 464ed51728
8 changed files with 166 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
core/mr/mapreduce_test.go
View File

@@ -3,7 +3,7 @@ package mr
import (
"context"
"errors"
"io"
"io/ioutil"
"log"
"runtime"
"sync/atomic"
@@ -17,7 +17,7 @@ import (
var errDummy = errors.New("dummy")
func init() {
log.SetOutput(io.Discard)
log.SetOutput(ioutil.Discard)
}
func TestFinish(t *testing.T) {
@@ -91,11 +91,11 @@ func TestForEach(t *testing.T) {
defer goleak.VerifyNone(t)
var count uint32
ForEach(func(source chan<- any) {
ForEach(func(source chan<- int) {
for i := 0; i < tasks; i++ {
source <- i
}
}, func(item any) {
}, func(item int) {
atomic.AddUint32(&count, 1)
}, WithWorkers(-1))
@@ -106,12 +106,12 @@ func TestForEach(t *testing.T) {
defer goleak.VerifyNone(t)
var count uint32
ForEach(func(source chan<- any) {
ForEach(func(source chan<- int) {
for i := 0; i < tasks; i++ {
source <- i
}
}, func(item any) {
if item.(int)%2 == 0 {
}, func(item int) {
if item%2 == 0 {
atomic.AddUint32(&count, 1)
}
})
@@ -123,11 +123,11 @@ func TestForEach(t *testing.T) {
defer goleak.VerifyNone(t)
assert.PanicsWithValue(t, "foo", func() {
ForEach(func(source chan<- any) {
ForEach(func(source chan<- int) {
for i := 0; i < tasks; i++ {
source <- i
}
}, func(item any) {
}, func(item int) {
panic("foo")
})
})
@@ -139,9 +139,9 @@ func TestGeneratePanic(t *testing.T) {
t.Run("all", func(t *testing.T) {
assert.PanicsWithValue(t, "foo", func() {
ForEach(func(source chan<- any) {
ForEach(func(source chan<- int) {
panic("foo")
}, func(item any) {
}, func(item int) {
})
})
})
@@ -154,14 +154,14 @@ func TestMapperPanic(t *testing.T) {
var run int32
t.Run("all", func(t *testing.T) {
assert.PanicsWithValue(t, "foo", func() {
_, _ = MapReduce(func(source chan<- any) {
_, _ = MapReduce(func(source chan<- int) {
for i := 0; i < tasks; i++ {
source <- i
}
}, func(item any, writer Writer, cancel func(error)) {
}, func(item int, writer Writer[int], cancel func(error)) {
atomic.AddInt32(&run, 1)
panic("foo")
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {
})
})
assert.True(t, atomic.LoadInt32(&run) < tasks/2)
@@ -173,10 +173,10 @@ func TestMapReduce(t *testing.T) {
tests := []struct {
name string
mapper MapperFunc
reducer ReducerFunc
mapper MapperFunc[int, int]
reducer ReducerFunc[int, int]
expectErr error
expectValue any
expectValue int
}{
{
name: "simple",
@@ -185,8 +185,7 @@ func TestMapReduce(t *testing.T) {
},
{
name: "cancel with error",
mapper: func(item any, writer Writer, cancel func(error)) {
v := item.(int)
mapper: func(v int, writer Writer[int], cancel func(error)) {
if v%3 == 0 {
cancel(errDummy)
}
@@ -196,22 +195,20 @@ func TestMapReduce(t *testing.T) {
},
{
name: "cancel with nil",
mapper: func(item any, writer Writer, cancel func(error)) {
v := item.(int)
mapper: func(v int, writer Writer[int], cancel func(error)) {
if v%3 == 0 {
cancel(nil)
}
writer.Write(v * v)
},
expectErr: ErrCancelWithNil,
expectValue: nil,
expectErr: ErrCancelWithNil,
},
{
name: "cancel with more",
reducer: func(pipe <-chan any, writer Writer, cancel func(error)) {
reducer: func(pipe <-chan int, writer Writer[int], cancel func(error)) {
var result int
for item := range pipe {
result += item.(int)
result += item
if result > 10 {
cancel(errDummy)
}
@@ -226,21 +223,20 @@ func TestMapReduce(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.mapper == nil {
test.mapper = func(item any, writer Writer, cancel func(error)) {
v := item.(int)
test.mapper = func(v int, writer Writer[int], cancel func(error)) {
writer.Write(v * v)
}
}
if test.reducer == nil {
test.reducer = func(pipe <-chan any, writer Writer, cancel func(error)) {
test.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {
var result int
for item := range pipe {
result += item.(int)
result += item
}
writer.Write(result)
}
}
value, err := MapReduce(func(source chan<- any) {
value, err := MapReduce(func(source chan<- int) {
for i := 1; i < 5; i++ {
source <- i
}
@@ -256,22 +252,21 @@ func TestMapReduce(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.mapper == nil {
test.mapper = func(item any, writer Writer, cancel func(error)) {
v := item.(int)
test.mapper = func(v int, writer Writer[int], cancel func(error)) {
writer.Write(v * v)
}
}
if test.reducer == nil {
test.reducer = func(pipe <-chan any, writer Writer, cancel func(error)) {
test.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {
var result int
for item := range pipe {
result += item.(int)
result += item
}
writer.Write(result)
}
}
source := make(chan any)
source := make(chan int)
go func() {
for i := 1; i < 5; i++ {
source <- i
@@ -291,13 +286,13 @@ func TestMapReduceWithReduerWriteMoreThanOnce(t *testing.T) {
defer goleak.VerifyNone(t)
assert.Panics(t, func() {
MapReduce(func(source chan<- any) {
MapReduce(func(source chan<- int) {
for i := 0; i < 10; i++ {
source <- i
}
}, func(item any, writer Writer, cancel func(error)) {
}, func(item int, writer Writer[int], cancel func(error)) {
writer.Write(item)
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[string], cancel func(error)) {
drain(pipe)
writer.Write("one")
writer.Write("two")
@@ -311,8 +306,8 @@ func TestMapReduceVoid(t *testing.T) {
var value uint32
tests := []struct {
name string
mapper MapperFunc
reducer VoidReducerFunc
mapper MapperFunc[int, int]
reducer VoidReducerFunc[int]
expectValue uint32
expectErr error
}{
@@ -323,8 +318,7 @@ func TestMapReduceVoid(t *testing.T) {
},
{
name: "cancel with error",
mapper: func(item any, writer Writer, cancel func(error)) {
v := item.(int)
mapper: func(v int, writer Writer[int], cancel func(error)) {
if v%3 == 0 {
cancel(errDummy)
}
@@ -334,8 +328,7 @@ func TestMapReduceVoid(t *testing.T) {
},
{
name: "cancel with nil",
mapper: func(item any, writer Writer, cancel func(error)) {
v := item.(int)
mapper: func(v int, writer Writer[int], cancel func(error)) {
if v%3 == 0 {
cancel(nil)
}
@@ -345,9 +338,9 @@ func TestMapReduceVoid(t *testing.T) {
},
{
name: "cancel with more",
reducer: func(pipe <-chan any, cancel func(error)) {
reducer: func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
result := atomic.AddUint32(&value, uint32(item.(int)))
result := atomic.AddUint32(&value, uint32(item))
if result > 10 {
cancel(errDummy)
}
@@ -362,19 +355,18 @@ func TestMapReduceVoid(t *testing.T) {
atomic.StoreUint32(&value, 0)
if test.mapper == nil {
test.mapper = func(item any, writer Writer, cancel func(error)) {
v := item.(int)
test.mapper = func(v int, writer Writer[int], cancel func(error)) {
writer.Write(v * v)
}
}
if test.reducer == nil {
test.reducer = func(pipe <-chan any, cancel func(error)) {
test.reducer = func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
atomic.AddUint32(&value, uint32(item.(int)))
atomic.AddUint32(&value, uint32(item))
}
}
}
err := MapReduceVoid(func(source chan<- any) {
err := MapReduceVoid(func(source chan<- int) {
for i := 1; i < 5; i++ {
source <- i
}
@@ -392,18 +384,17 @@ func TestMapReduceVoidWithDelay(t *testing.T) {
defer goleak.VerifyNone(t)
var result []int
err := MapReduceVoid(func(source chan<- any) {
err := MapReduceVoid(func(source chan<- int) {
source <- 0
source <- 1
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
if i == 0 {
time.Sleep(time.Millisecond * 50)
}
writer.Write(i)
}, func(pipe <-chan any, cancel func(error)) {
}, func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
i := item.(int)
i := item
result = append(result, i)
}
})
@@ -417,13 +408,12 @@ func TestMapReducePanic(t *testing.T) {
defer goleak.VerifyNone(t)
assert.Panics(t, func() {
_, _ = MapReduce(func(source chan<- any) {
_, _ = MapReduce(func(source chan<- int) {
source <- 0
source <- 1
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
writer.Write(i)
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {
for range pipe {
panic("panic")
}
@@ -435,17 +425,16 @@ func TestMapReducePanicOnce(t *testing.T) {
defer goleak.VerifyNone(t)
assert.Panics(t, func() {
_, _ = MapReduce(func(source chan<- any) {
_, _ = MapReduce(func(source chan<- int) {
for i := 0; i < 100; i++ {
source <- i
}
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
if i == 0 {
panic("foo")
}
writer.Write(i)
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {
for range pipe {
panic("bar")
}
@@ -457,12 +446,12 @@ func TestMapReducePanicBothMapperAndReducer(t *testing.T) {
defer goleak.VerifyNone(t)
assert.Panics(t, func() {
_, _ = MapReduce(func(source chan<- any) {
_, _ = MapReduce(func(source chan<- int) {
source <- 0
source <- 1
}, func(item any, writer Writer, cancel func(error)) {
}, func(item int, writer Writer[int], cancel func(error)) {
panic("foo")
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {
panic("bar")
})
})
@@ -472,18 +461,17 @@ func TestMapReduceVoidCancel(t *testing.T) {
defer goleak.VerifyNone(t)
var result []int
err := MapReduceVoid(func(source chan<- any) {
err := MapReduceVoid(func(source chan<- int) {
source <- 0
source <- 1
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
if i == 1 {
cancel(errors.New("anything"))
}
writer.Write(i)
}, func(pipe <-chan any, cancel func(error)) {
}, func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
i := item.(int)
i := item
result = append(result, i)
}
})
@@ -496,21 +484,19 @@ func TestMapReduceVoidCancelWithRemains(t *testing.T) {
var done int32
var result []int
err := MapReduceVoid(func(source chan<- any) {
err := MapReduceVoid(func(source chan<- int) {
for i := 0; i < defaultWorkers*2; i++ {
source <- i
}
atomic.AddInt32(&done, 1)
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
if i == defaultWorkers/2 {
cancel(errors.New("anything"))
}
writer.Write(i)
}, func(pipe <-chan any, cancel func(error)) {
}, func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
i := item.(int)
result = append(result, i)
result = append(result, item)
}
})
assert.NotNil(t, err)
@@ -522,18 +508,18 @@ func TestMapReduceWithoutReducerWrite(t *testing.T) {
defer goleak.VerifyNone(t)
uids := []int{1, 2, 3}
res, err := MapReduce(func(source chan<- any) {
res, err := MapReduce(func(source chan<- int) {
for _, uid := range uids {
source <- uid
}
}, func(item any, writer Writer, cancel func(error)) {
}, func(item int, writer Writer[int], cancel func(error)) {
writer.Write(item)
}, func(pipe <-chan any, writer Writer, cancel func(error)) {
}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {
drain(pipe)
// not calling writer.Write(...), should not panic
})
assert.Equal(t, ErrReduceNoOutput, err)
assert.Nil(t, res)
assert.Equal(t, 0, res)
}
func TestMapReduceVoidPanicInReducer(t *testing.T) {
@@ -542,15 +528,14 @@ func TestMapReduceVoidPanicInReducer(t *testing.T) {
const message = "foo"
assert.Panics(t, func() {
var done int32
_ = MapReduceVoid(func(source chan<- any) {
_ = MapReduceVoid(func(source chan<- int) {
for i := 0; i < defaultWorkers*2; i++ {
source <- i
}
atomic.AddInt32(&done, 1)
}, func(item any, writer Writer, cancel func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], cancel func(error)) {
writer.Write(i)
}, func(pipe <-chan any, cancel func(error)) {
}, func(pipe <-chan int, cancel func(error)) {
panic(message)
}, WithWorkers(1))
})
@@ -561,13 +546,12 @@ func TestForEachWithContext(t *testing.T) {
var done int32
ctx, cancel := context.WithCancel(context.Background())
ForEach(func(source chan<- any) {
ForEach(func(source chan<- int) {
for i := 0; i < defaultWorkers*2; i++ {
source <- i
}
atomic.AddInt32(&done, 1)
}, func(item any) {
i := item.(int)
}, func(i int) {
if i == defaultWorkers/2 {
cancel()
}
@@ -580,20 +564,19 @@ func TestMapReduceWithContext(t *testing.T) {
var done int32
var result []int
ctx, cancel := context.WithCancel(context.Background())
err := MapReduceVoid(func(source chan<- any) {
err := MapReduceVoid(func(source chan<- int) {
for i := 0; i < defaultWorkers*2; i++ {
source <- i
}
atomic.AddInt32(&done, 1)
}, func(item any, writer Writer, c func(error)) {
i := item.(int)
}, func(i int, writer Writer[int], c func(error)) {
if i == defaultWorkers/2 {
cancel()
}
writer.Write(i)
}, func(pipe <-chan any, cancel func(error)) {
}, func(pipe <-chan int, cancel func(error)) {
for item := range pipe {
i := item.(int)
i := item
result = append(result, i)
}
}, WithContext(ctx))
@@ -604,19 +587,19 @@ func TestMapReduceWithContext(t *testing.T) {
func BenchmarkMapReduce(b *testing.B) {
b.ReportAllocs()
mapper := func(v any, writer Writer, cancel func(error)) {
writer.Write(v.(int64) * v.(int64))
mapper := func(v int64, writer Writer[int64], cancel func(error)) {
writer.Write(v * v)
}
reducer := func(input <-chan any, writer Writer, cancel func(error)) {
reducer := func(input <-chan int64, writer Writer[int64], cancel func(error)) {
var result int64
for v := range input {
result += v.(int64)
result += v
}
writer.Write(result)
}
for i := 0; i < b.N; i++ {
MapReduce(func(input chan<- any) {
MapReduce(func(input chan<- int64) {
for j := 0; j < 2; j++ {
input <- int64(j)
}