|
|
|
|
@ -12,6 +12,7 @@ package main
|
|
|
|
|
import (
|
|
|
|
|
"fmt"
|
|
|
|
|
"sync"
|
|
|
|
|
"time"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
var mu sync.Mutex
|
|
|
|
|
@ -25,6 +26,9 @@ func Crawl(url string, depth int, wg *sync.WaitGroup) {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mu.Lock()
|
|
|
|
|
time.Sleep(time.Second)
|
|
|
|
|
mu.Unlock()
|
|
|
|
|
body, urls, err := fetcher.Fetch(url)
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Println(err)
|
|
|
|
|
|