ron/taldl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
ron 2020-04-05 02:11:57 +02:00
parent 97abf4d0e4
commit 0bf9308dd7
1 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
main.go
View File

@ -16,22 +16,23 @@ import (
"github.com/gocolly/colly" "github.com/gocolly/colly"
) )
var format = flag.String("format", "epub", "format to download") var store = "~/.taldl.json"
var output = flag.String("output", "epub", "output directory") var output = flag.String("output", "epub", "output directory")
var store = flag.String("store", "~/.tal-scraper.json", "data store") var format = flag.String("format", "epub", "format to download")
func main() { func main() {
flag.Parse() flag.Parse()
store = fixpath(store)
*output = fixpath(*output) *output = fixpath(*output)
*store = fixpath(*store)
os.MkdirAll(*output, os.ModePerm) os.MkdirAll(*output, os.ModePerm)
oldMostRecent := "" oldMostRecent := ""
newMostRecent := "" newMostRecent := ""
data, _ := ioutil.ReadFile(*store) data, _ := ioutil.ReadFile(store)
_ = json.Unmarshal(data, &oldMostRecent) _ = json.Unmarshal(data, &oldMostRecent)
done := false done := false
@ -55,9 +56,8 @@ func main() {
name := filenameForUrl(url) name := filenameForUrl(url)
path := *output + "/" + name path := *output + "/" + name
data, err := HTTPGet(url, time.Second*3) data, err := get(url, time.Second*3)
if err != nil { if err != nil {
fmt.Println(url)
fmt.Println(err) fmt.Println(err)
return return
} }
@ -83,7 +83,7 @@ func main() {
c.Visit("https://theanarchistlibrary.org/latest") c.Visit("https://theanarchistlibrary.org/latest")
data, _ = json.Marshal(newMostRecent) data, _ = json.Marshal(newMostRecent)
_ = ioutil.WriteFile(*store, data, 0644) _ = ioutil.WriteFile(store, data, 0644)
} }
func fixpath(path string) string { func fixpath(path string) string {
@ -104,7 +104,7 @@ func filenameForUrl(url string) string {
return path.Base(url) return path.Base(url)
} }
func HTTPGet(url string, timeout time.Duration) (content []byte, err error) { func get(url string, timeout time.Duration) (content []byte, err error) {
request, err := http.NewRequest("GET", url, nil) request, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return return
@ -121,7 +121,7 @@ func HTTPGet(url string, timeout time.Duration) (content []byte, err error) {
if response.StatusCode != 200 { if response.StatusCode != 200 {
cancel_func() cancel_func()
return nil, fmt.Errorf("INVALID RESPONSE; status: %s", response.Status) return nil, fmt.Errorf("INVALID RESPONSE: %s", response.Status)
} }
return ioutil.ReadAll(response.Body) return ioutil.ReadAll(response.Body)