diff options
Diffstat (limited to 'fetcher.fnl')
| -rw-r--r-- | fetcher.fnl | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/fetcher.fnl b/fetcher.fnl index 6d6d633..bf22abf 100644 --- a/fetcher.fnl +++ b/fetcher.fnl @@ -71,7 +71,7 @@ categories [])) -(fn walk-json-pages [url-formatter path] +(fn walk-json-pages [url-formatter path response-destructor] (fn gather [page knil] (local url (url-formatter path page)) (print (.. "requesting " url)) @@ -84,12 +84,12 @@ "")) (if (= status 200) - (let [items (json.decode content)] + (let [{: items} (response-destructor (json.decode content))] (if (or (= items nil) (= 0 (# items))) knil (do (os.execute "sleep 1") - (gather (+ page 1) (array.concat knil items))))) + (gather (+ 1 page) (array.concat knil items))))) (= status 404) knil (retry #(gather page knil) 3 1))) @@ -103,7 +103,7 @@ result (categorize-many (map #(normalizer $2) - (walk-json-pages url-formatter path)) + (walk-json-pages url-formatter path response-destructor)) category))) categories [])) |