参考
https://pandaychen.github.io/2020/04/09/A-GOLANG-PIPELINE/
代码
package main
import (
"crypto/md5"
"fmt"
"log"
"os"
"path/filepath"
)
type result struct {
path string
sum [md5.Size]byte
}
type fileData struct {
path string
data []byte
}
func walkFiles(done <-chan struct{}, root string) (<-chan string, <-chan error) {
out := make(chan string, 0)
errc := make(chan error, 1)
go func() {
defer close(out)
defer close(errc)
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
select {
case out <- path:
return nil
case <-done:
return fmt.Errorf("walk cancelled")
}
})
errc <- err
}()
return out, errc
}
func readFiles(done <-chan struct{}, paths <-chan string) (<-chan fileData, <-chan error) {
out := make(chan fileData, 100)
errc := make(chan error, 1)
go func() {
defer close(out)
defer close(errc)
for {
select {
case <-done:
return
case path, ok := <-paths:
if !ok {
return
}
data, err := os.ReadFile(path)
if err != nil {
errc <- err
return
}
select {
case out <- fileData{path, data}:
case <-done:
return
}
}
}
}()
return out, errc
}
func md5Files(done <-chan struct{}, fileDatas <-chan fileData) (<-chan result, <-chan error) {
out := make(chan result, 100)
errc := make(chan error, 1)
go func() {
defer close(out)
defer close(errc)
for {
select {
case <-done:
return
case fd, ok := <-fileDatas:
if !ok {
return
}
sum := md5.Sum(fd.data)
select {
case out <- result{fd.path, sum}:
case <-done:
return
}
}
}
}()
return out, errc
}
func MD5All(root string) (map[string][md5.Size]byte, error) {
done := make(chan struct{})
defer close(done)
paths, walkErrc := walkFiles(done, root)
files, readErrc := readFiles(done, paths)
results, hashErrc := md5Files(done, files)
m := make(map[string][md5.Size]byte)
for {
select {
case r, ok := <-results:
if !ok {
results = nil
} else {
m[r.path] = r.sum
}
case err, ok := <-walkErrc:
if ok && err != nil {
return nil, fmt.Errorf("walk error: %w", err)
}
walkErrc = nil
case err, ok := <-readErrc:
if ok && err != nil {
return nil, fmt.Errorf("read error: %w", err)
}
readErrc = nil
case err, ok := <-hashErrc:
if ok && err != nil {
return nil, fmt.Errorf("hash error: %w", err)
}
hashErrc = nil
}
if results == nil && walkErrc == nil && readErrc == nil && hashErrc == nil {
break
}
}
return m, nil
}
func main() {
root := "."
res, err := MD5All(root)
if err != nil {
log.Fatalf("失败: %v", err)
}
for path, sum := range res {
fmt.Printf("%s: %x\n", path, sum)
}
}