diff options
| author | unwox <me@unwox.com> | 2024-10-15 14:49:46 +0600 |
|---|---|---|
| committer | unwox <me@unwox.com> | 2024-10-15 14:59:04 +0600 |
| commit | 92d95b978bfb10fd29024d0adda7b28f125e08b8 (patch) | |
| tree | 189fd3c9cf9bc08712d795e8119c8df61fd7ce6b /fetcher.fnl | |
| parent | 1204496efa2fcd495bd74ba8ca249b7f082f3ba5 (diff) | |
adapt to the new convention for calling luna module methods
Diffstat (limited to 'fetcher.fnl')
| -rw-r--r-- | fetcher.fnl | 31 |
1 files changed, 17 insertions, 14 deletions
diff --git a/fetcher.fnl b/fetcher.fnl index 6a133f0..b3db8f0 100644 --- a/fetcher.fnl +++ b/fetcher.fnl @@ -8,6 +8,7 @@ (local json (require :vendor.json)) (local parser (require :parser.parser)) (local http (require :lib.http)) +(local {: must} (require :lib.utils)) (fn retry [what times sleep] (var result nil) @@ -33,17 +34,18 @@ (fn gather [page knil] (local url (url-formatter path page)) (print (.. "requesting " url)) - (local (status _ html) - (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} "")) + (local {: status : body} + (must + (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} ""))) - (if (= status 200) - (let [items (parser.match-many html item-peg)] + (if (= 200 status) + (let [items (parser.match-many body item-peg)] (if (or (= items nil) (= 0 (# items))) knil (do - (os.execute "sleep 1") + (os.execute "sleep .3") (gather (+ page 1) (array.concat knil items))))) - (= status 404) + (= 404 status) knil (retry #(gather page knil) 3 1))) @@ -75,16 +77,17 @@ (fn gather [page knil] (local url (url-formatter path page)) (print (.. "requesting " url)) - (local (status _ content) - (luna.http.request - "GET" url - {:User-Agent (http.random-user-agent) - :Content-Type "application/json" - :Accept "application/json"} - "")) + (local {: status : body} + (must + (luna.http.request + "GET" url + {:User-Agent (http.random-user-agent) + :Content-Type "application/json" + :Accept "application/json"} + ""))) (if (= status 200) - (let [{: items} (response-destructor (json.decode content))] + (let [{: items} (response-destructor (json.decode body))] (if (or (= items nil) (= 0 (# items))) knil (do |
