ron/taldl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wg

Browse Source
This commit is contained in:
ron 2021-08-23 16:06:24 +02:00
parent c8abf6fc61
commit 4764182d24
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
main.go
View File

@ -64,9 +64,12 @@ func main() {
os.MkdirAll(*outputDir, os.ModePerm) os.MkdirAll(*outputDir, os.ModePerm)
wg := sync.WaitGroup{}
c := colly.NewCollector() c := colly.NewCollector()
c.OnHTML("ul.pagination li:nth-last-child(2)", func(e *colly.HTMLElement) { c.OnHTML("ul.pagination li:nth-last-child(2)", func(e *colly.HTMLElement) {
lastPage, _ = strconv.Atoi(strings.TrimSpace(e.Text)) lastPage, _ = strconv.Atoi(strings.TrimSpace(e.Text))
wg.Done()
}) })
c.OnHTML("div.amw-listing-item:nth-child(1) a[href]", func(e *colly.HTMLElement) { c.OnHTML("div.amw-listing-item:nth-child(1) a[href]", func(e *colly.HTMLElement) {
href := e.Attr("href") href := e.Attr("href")
@ -82,8 +85,10 @@ func main() {
} }
}) })
wg.Add(1)
url := "https://theanarchistlibrary.org/latest/1" url := "https://theanarchistlibrary.org/latest/1"
c.Visit(url) c.Visit(url)
wg.Wait()
newPages := 1 newPages := 1
@ -332,7 +337,6 @@ func get(url string, timeout time.Duration) (content []byte, err error) {
defer response.Body.Close() defer response.Body.Close()
if response.StatusCode == 404 { if response.StatusCode == 404 {
// fmt.Fprintln(os.Stderr, url+" NOT FOUND!")
missing = append(missing, url) missing = append(missing, url)
return return
} }