You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
57 lines
1.0 KiB
Go
57 lines
1.0 KiB
Go
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Your task is to change the code to limit the crawler to at most one
|
|
// page per second, while maintaining concurrency (in other words,
|
|
// Crawl() must be called concurrently)
|
|
//
|
|
// @hint: you can achieve this by adding 3 lines
|
|
//
|
|
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var mu sync.Mutex
|
|
|
|
// Crawl uses `fetcher` from the `mockfetcher.go` file to imitate a
|
|
// real crawler. It crawls until the maximum depth has reached.
|
|
func Crawl(url string, depth int, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
|
|
if depth <= 0 {
|
|
return
|
|
}
|
|
|
|
mu.Lock()
|
|
time.Sleep(time.Second)
|
|
mu.Unlock()
|
|
body, urls, err := fetcher.Fetch(url)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
return
|
|
}
|
|
|
|
fmt.Printf("found: %s %q\n", url, body)
|
|
|
|
wg.Add(len(urls))
|
|
for _, u := range urls {
|
|
// Do not remove the `go` keyword, as Crawl() must be
|
|
// called concurrently
|
|
go Crawl(u, depth-1, wg)
|
|
|
|
}
|
|
return
|
|
}
|
|
|
|
func main() {
|
|
var wg sync.WaitGroup
|
|
|
|
wg.Add(1)
|
|
Crawl("http://golang.org/", 4, &wg)
|
|
wg.Wait()
|
|
}
|