initial import

example/mapreduce/countfunc/countfunc.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"zero/core/mapreduce"
+
+	"github.com/google/gops/agent"
+)
+
+var (
+	dir        = flag.String("d", "", "dir to enumerate")
+	stopOnFile = flag.String("s", "", "stop when got file")
+	maxFiles   = flag.Int("m", 0, "at most files to process")
+	mode       = flag.String("mode", "", "simulate mode, can be return|panic")
+	count      uint32
+)
+
+func enumerateLines(filename string) chan string {
+	output := make(chan string)
+	go func() {
+		file, err := os.Open(filename)
+		if err != nil {
+			return
+		}
+		defer file.Close()
+
+		reader := bufio.NewReader(file)
+		for {
+			line, err := reader.ReadString('\n')
+			if err == io.EOF {
+				break
+			}
+
+			if !strings.HasPrefix(line, "#") {
+				output <- line
+			}
+		}
+		close(output)
+	}()
+	return output
+}
+
+func mapper(filename interface{}, writer mapreduce.Writer, cancel func(error)) {
+	if len(*stopOnFile) > 0 && path.Base(filename.(string)) == *stopOnFile {
+		fmt.Printf("Stop on file: %s\n", *stopOnFile)
+		cancel(errors.New("stop on file"))
+		return
+	}
+
+	var result int
+	for line := range enumerateLines(filename.(string)) {
+		if strings.HasPrefix(strings.TrimSpace(line), "func") {
+			result++
+		}
+	}
+
+	switch *mode {
+	case "return":
+		if atomic.AddUint32(&count, 1)%10 == 0 {
+			return
+		}
+	case "panic":
+		if atomic.AddUint32(&count, 1)%10 == 0 {
+			panic("wow")
+		}
+	}
+
+	writer.Write(result)
+}
+
+func reducer(input <-chan interface{}, writer mapreduce.Writer, cancel func(error)) {
+	var result int
+
+	for count := range input {
+		v := count.(int)
+		if *maxFiles > 0 && result >= *maxFiles {
+			fmt.Printf("Reached max files: %d\n", *maxFiles)
+			cancel(errors.New("max files reached"))
+			return
+		}
+		result += v
+	}
+
+	writer.Write(result)
+}
+
+func main() {
+	if err := agent.Listen(agent.Options{}); err != nil {
+		log.Fatal(err)
+	}
+
+	flag.Parse()
+
+	if len(*dir) == 0 {
+		flag.Usage()
+	}
+
+	fmt.Println("Processing, please wait...")
+
+	start := time.Now()
+	result, err := mapreduce.MapReduce(func(source chan<- interface{}) {
+		filepath.Walk(*dir, func(fpath string, f os.FileInfo, err error) error {
+			if !f.IsDir() && path.Ext(fpath) == ".go" {
+				source <- fpath
+			}
+			return nil
+		})
+	}, mapper, reducer)
+	if err != nil {
+		fmt.Println(err)
+	} else {
+		fmt.Println(result)
+		fmt.Println("Elapsed:", time.Since(start))
+		fmt.Println("Done")
+	}
+}

example/mapreduce/finishvoid/finishvoid.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"zero/core/mapreduce"
+	"zero/core/timex"
+)
+
+func main() {
+	start := timex.Now()
+
+	mapreduce.FinishVoid(func() {
+		time.Sleep(time.Second)
+	}, func() {
+		time.Sleep(time.Second * 5)
+	}, func() {
+		time.Sleep(time.Second * 10)
+	}, func() {
+		time.Sleep(time.Second * 6)
+	}, func() {
+		if err := mapreduce.Finish(func() error {
+			time.Sleep(time.Second)
+			return nil
+		}, func() error {
+			time.Sleep(time.Second * 10)
+			return nil
+		}); err != nil {
+			fmt.Println(err)
+		}
+	})
+
+	fmt.Println(timex.Since(start))
+}

example/mapreduce/flatmap/flatmap.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+
+	"zero/core/mapreduce"
+)
+
+var (
+	persons = []string{"john", "mary", "alice", "bob"}
+	friends = map[string][]string{
+		"john":  {"harry", "hermione", "ron"},
+		"mary":  {"sam", "frodo"},
+		"alice": {},
+		"bob":   {"jamie", "tyrion", "cersei"},
+	}
+)
+
+func main() {
+	var allFriends []string
+	for v := range mapreduce.Map(func(source chan<- interface{}) {
+		for _, each := range persons {
+			source <- each
+		}
+	}, func(item interface{}, writer mapreduce.Writer) {
+		writer.Write(friends[item.(string)])
+	}, mapreduce.WithWorkers(100)) {
+		allFriends = append(allFriends, v.([]string)...)
+	}
+	fmt.Println(allFriends)
+}

example/mapreduce/goroutineleak/leak.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"time"
+
+	"zero/core/lang"
+	"zero/core/logx"
+	"zero/core/mapreduce"
+	"zero/core/proc"
+)
+
+func dumpGoroutines() {
+	dumpFile := "goroutines.dump"
+	logx.Infof("Got dump goroutine signal, printing goroutine profile to %s", dumpFile)
+
+	if f, err := os.Create(dumpFile); err != nil {
+		logx.Errorf("Failed to dump goroutine profile, error: %v", err)
+	} else {
+		defer f.Close()
+		pprof.Lookup("goroutine").WriteTo(f, 2)
+	}
+}
+
+func main() {
+	profiler := proc.StartProfile()
+	defer profiler.Stop()
+
+	done := make(chan lang.PlaceholderType)
+	go func() {
+		for {
+			time.Sleep(time.Second)
+			fmt.Println(runtime.NumGoroutine())
+		}
+	}()
+	go func() {
+		time.Sleep(time.Minute)
+		dumpGoroutines()
+		close(done)
+	}()
+	for {
+		select {
+		case <-done:
+			return
+		default:
+			mapreduce.MapReduce(func(source chan<- interface{}) {
+				for i := 0; i < 100; i++ {
+					source <- i
+				}
+			}, func(item interface{}, writer mapreduce.Writer, cancel func(error)) {
+				if item.(int) == 40 {
+					cancel(errors.New("any"))
+					return
+				}
+				writer.Write(item)
+			}, func(pipe <-chan interface{}, writer mapreduce.Writer, cancel func(error)) {
+				list := make([]int, 0)
+				for p := range pipe {
+					list = append(list, p.(int))
+				}
+				writer.Write(list)
+			})
+		}
+	}
+}

example/mapreduce/irregular/irregular.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"zero/core/mapreduce"
+)
+
+func main() {
+	mapreduce.MapReduceVoid(func(source chan<- interface{}) {
+		for i := 0; i < 10; i++ {
+			source <- i
+		}
+	}, func(item interface{}, writer mapreduce.Writer, cancel func(error)) {
+		i := item.(int)
+		if i == 0 {
+			time.Sleep(10 * time.Second)
+		} else {
+			time.Sleep(5 * time.Second)
+		}
+		writer.Write(i)
+	}, func(pipe <-chan interface{}, cancel func(error)) {
+		for i := range pipe {
+			fmt.Println(i)
+		}
+	})
+}