blob: 6a133f0c6d7708c09f5ca7b55d5770a7a85e3a8e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
(import-macros {: reduce : map} :lib.macro)
(local peg
(if (pick-values 1 (pcall require :lpeg))
(require :lpeg)
(require :lpeglj)))
(local array (require :lib.array))
(local json (require :vendor.json))
(local parser (require :parser.parser))
(local http (require :lib.http))
(fn retry [what times sleep]
(var result nil)
(var stop? false)
(var err nil)
(for [i 1 times &until stop?]
(local (ok? value) (pcall what))
(if ok?
(do
(set result value)
(set stop? true))
(do
(set err value)
(os.execute (.. "sleep " sleep)))))
(when (not stop?)
(error (.. "failed after " times " retries:\n" err)))
result)
(fn walk-html-pages [url-formatter path item-peg]
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
(local (status _ html)
(luna.http.request "GET" url {:User-Agent (http.random-user-agent)} ""))
(if (= status 200)
(let [items (parser.match-many html item-peg)]
(if (or (= items nil) (= 0 (# items)))
knil
(do
(os.execute "sleep 1")
(gather (+ page 1) (array.concat knil items)))))
(= status 404)
knil
(retry #(gather page knil) 3 1)))
(gather 1 []))
(fn categorize-many [items tags]
(map
(fn [_ item]
(tset item :tags
(-> (or tags [])
(array.concat (parser.guess-tags item.title))
(array.unique)))
item)
items))
(fn from-html [categories url-formatter item-peg normalizer]
(reduce
(fn [_ {: tags : path} result]
(array.concat
result
(categorize-many
(map #(normalizer $2)
(walk-html-pages url-formatter path item-peg))
tags)))
categories
[]))
(fn walk-json-pages [url-formatter path response-destructor]
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
(local (status _ content)
(luna.http.request
"GET" url
{:User-Agent (http.random-user-agent)
:Content-Type "application/json"
:Accept "application/json"}
""))
(if (= status 200)
(let [{: items} (response-destructor (json.decode content))]
(if (or (= items nil) (= 0 (# items)))
knil
(do
(os.execute "sleep 1")
(gather (+ 1 page) (array.concat knil items)))))
(= status 404)
knil
(retry #(gather page knil) 3 1)))
(gather 1 []))
(fn from-json [categories url-formatter response-destructor normalizer]
(reduce
(fn [_ {: tags : path} result]
(array.concat
result
(categorize-many
(map #(normalizer $2)
(walk-json-pages url-formatter path response-destructor))
tags)))
categories
[]))
{: from-html
: from-json}
|
