ron/taldl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

check muse instead of zip

Browse Source
This commit is contained in:
ron 2021-08-25 03:51:02 +02:00
parent 4764182d24
commit 01f3a51f8e
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
main.go
View File

@ -188,7 +188,7 @@ func main() {
}
if *verbose {
fmt.Fprintf(os.Stderr, "Downloading %d entries...\r\n", downloadCount)
fmt.Fprintf(os.Stderr, "Downloading %d files...\r\n", downloadCount)
}
if *progress {
@ -266,9 +266,14 @@ func dirnameForUrl(url string) string {
}
func check(url string, path string, timeout time.Duration) (modified bool, err error) {
if strings.HasSuffix(url, ".zip") {
url = strings.TrimSuffix(url, ".zip") + ".muse"
path = strings.TrimSuffix(path, ".zip") + ".muse"
}
request, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return
return false, err
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -278,12 +283,12 @@ func check(url string, path string, timeout time.Duration) (modified bool, err e
response, err := http.DefaultClient.Do(request)
if err != nil {
return
return false, err
}
defer response.Body.Close()
if response.StatusCode == 404 {
fmt.Fprintln(os.Stderr, url+" was removed?")
fmt.Fprintln(os.Stderr, url+" not found.")
return false, nil
}
@ -429,6 +434,10 @@ func unzip(src string, dest string) ([]string, error) {
func downloader(id int, jobs <-chan string, results chan<- string) {
for href := range jobs {
if strings.HasSuffix(href, ".muse") {
href = strings.TrimSuffix(href, ".muse") + ".zip"
}
result := ""
dir := dirnameForUrl(href)
@ -519,10 +528,17 @@ func checker(id int, jobs <-chan string, results chan<- string) {
name := filenameForUrl(href)
path := filepath.Join(*outputDir, dir, name)
if strings.HasSuffix(name, ".zip") {
if !fileExists(strings.TrimSuffix(path, ".zip") + ".muse") {
results <- href
continue
}
} else {
if !fileExists(path) {
results <- href
continue
}
}
for {
modified, err = check(href, path, time.Second*30)