Exercise: Web Crawler.go

package main

import (
        "fmt"
        "sync"
)

type Fetcher interface {
        // Fetch returns the body of URL and
        // a slice of URLs found on that page.
        Fetch(url string) (body string, urls []string, err error)
}

type Fetched struct {
        urls  map[string]bool
        mutex sync.Mutex
}

func (v *Fetched) is_fetched(url string) bool {
        defer func() {
                v.urls[url] = true
                v.mutex.Unlock()
        }()
        v.mutex.Lock()
        return v.urls[url]
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher, f Fetched, ch chan string) {
        defer close(ch)
        // This implementation doesn't do either:
        if depth <= 0 {
                return
        }
        body, urls, err := fetcher.Fetch(url)
        if err != nil {
                ch <- err.Error()
                return
        }

        if f.is_fetched(url) {
                ch <- fmt.Sprintf("exist: %s\n", url)
                return
        }

        ch <- fmt.Sprintf("found: %s %q\n", url, body)

        res := make([]chan string, len(urls))
        for i, u := range urls {
                res[i] = make(chan string)
                go Crawl(u, depth-1, fetcher, f, res[i])
        }

        for v := range res {
                for i := range res[v] {
                        ch <- i
                }
        }
        return
}

func main() {
        resch := make(chan string)
        fetched := Fetched{urls: make(map[string]bool)}
        go Crawl("http://golang.org/", 4, fetcher, fetched, resch)
        for v := range resch {
                fmt.Println(v)
        }
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
        body string
        urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
        if res, ok := f[url]; ok {
                return res.body, res.urls, nil
        }
        return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
        "http://golang.org/": &fakeResult{
                "The Go Programming Language",
                []string{
                        "http://golang.org/pkg/",
                        "http://golang.org/cmd/",
                },
        },
        "http://golang.org/pkg/": &fakeResult{
                "Packages",
                []string{
                        "http://golang.org/",
                        "http://golang.org/cmd/",
                        "http://golang.org/pkg/fmt/",
                        "http://golang.org/pkg/os/",
                },
        },
        "http://golang.org/pkg/fmt/": &fakeResult{
                "Package fmt",
                []string{
                        "http://golang.org/",
                        "http://golang.org/pkg/",
                },
        },
        "http://golang.org/pkg/os/": &fakeResult{
                "Package os",
                []string{
                        "http://golang.org/",
                        "http://golang.org/pkg/",
                },
        },
}