From 4764182d24957f8fd57403e7b3193374ea8f7b92 Mon Sep 17 00:00:00 2001 From: ron Date: Mon, 23 Aug 2021 16:06:24 +0200 Subject: [PATCH] wg --- main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 6e14687..7f8a336 100644 --- a/main.go +++ b/main.go @@ -64,9 +64,12 @@ func main() { os.MkdirAll(*outputDir, os.ModePerm) + wg := sync.WaitGroup{} + c := colly.NewCollector() c.OnHTML("ul.pagination li:nth-last-child(2)", func(e *colly.HTMLElement) { lastPage, _ = strconv.Atoi(strings.TrimSpace(e.Text)) + wg.Done() }) c.OnHTML("div.amw-listing-item:nth-child(1) a[href]", func(e *colly.HTMLElement) { href := e.Attr("href") @@ -82,8 +85,10 @@ func main() { } }) + wg.Add(1) url := "https://theanarchistlibrary.org/latest/1" c.Visit(url) + wg.Wait() newPages := 1 @@ -332,7 +337,6 @@ func get(url string, timeout time.Duration) (content []byte, err error) { defer response.Body.Close() if response.StatusCode == 404 { - // fmt.Fprintln(os.Stderr, url+" NOT FOUND!") missing = append(missing, url) return }