summaryrefslogtreecommitdiff
path: root/fetcher.fnl
diff options
context:
space:
mode:
authorunwox <me@unwox.com>2024-10-15 14:49:46 +0600
committerunwox <me@unwox.com>2024-10-15 14:59:04 +0600
commit92d95b978bfb10fd29024d0adda7b28f125e08b8 (patch)
tree189fd3c9cf9bc08712d795e8119c8df61fd7ce6b /fetcher.fnl
parent1204496efa2fcd495bd74ba8ca249b7f082f3ba5 (diff)
adapt to the new convention for calling luna module methods
Diffstat (limited to 'fetcher.fnl')
-rw-r--r--fetcher.fnl31
1 files changed, 17 insertions, 14 deletions
diff --git a/fetcher.fnl b/fetcher.fnl
index 6a133f0..b3db8f0 100644
--- a/fetcher.fnl
+++ b/fetcher.fnl
@@ -8,6 +8,7 @@
(local json (require :vendor.json))
(local parser (require :parser.parser))
(local http (require :lib.http))
+(local {: must} (require :lib.utils))
(fn retry [what times sleep]
(var result nil)
@@ -33,17 +34,18 @@
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
- (local (status _ html)
- (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} ""))
+ (local {: status : body}
+ (must
+ (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} "")))
- (if (= status 200)
- (let [items (parser.match-many html item-peg)]
+ (if (= 200 status)
+ (let [items (parser.match-many body item-peg)]
(if (or (= items nil) (= 0 (# items)))
knil
(do
- (os.execute "sleep 1")
+ (os.execute "sleep .3")
(gather (+ page 1) (array.concat knil items)))))
- (= status 404)
+ (= 404 status)
knil
(retry #(gather page knil) 3 1)))
@@ -75,16 +77,17 @@
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
- (local (status _ content)
- (luna.http.request
- "GET" url
- {:User-Agent (http.random-user-agent)
- :Content-Type "application/json"
- :Accept "application/json"}
- ""))
+ (local {: status : body}
+ (must
+ (luna.http.request
+ "GET" url
+ {:User-Agent (http.random-user-agent)
+ :Content-Type "application/json"
+ :Accept "application/json"}
+ "")))
(if (= status 200)
- (let [{: items} (response-destructor (json.decode content))]
+ (let [{: items} (response-destructor (json.decode body))]
(if (or (= items nil) (= 0 (# items)))
knil
(do