fetcher.fnl


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

(import-macros {: reduce : map} :lib.macro)

(local peg
 (if (pick-values 1 (pcall require :lpeg))
   (require :lpeg)
   (require :lpeglj)))
(local array (require :lib.array))
(local parser (require :parser.parser))
(local http (require :lib.http))
(local {: must} (require :lib.utils))

(fn walk-html-pages [url-formatter path item-peg]
  (fn gather [page knil]
    (local url (url-formatter path page))
    (print (.. "requesting " url))
    (local {: status : body}
      (must
       (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} "")))

    (if (= 200 status)
     (let [items (parser.match-many body item-peg)]
      (if (or (= items nil) (= 0 (# items)))
          knil
          (do
            (os.execute "sleep 1")
            (gather (+ page 1) (array.concat knil items)))))
      knil))

  (gather 1 []))

(fn categorize-many [items tags]
  (map
   (fn [_ item]
    (tset item :tags
      (-> (or tags [])
        (array.concat (parser.guess-tags item.title))
        (array.unique)))
    item)
   items))

(fn from-html [categories url-formatter item-peg normalizer]
  (reduce
   (fn [_ {: tags : path} result]
     (array.concat
       result
       (categorize-many
        (map #(normalizer $2)
             (walk-html-pages url-formatter path item-peg))
        tags)))
   categories
   []))

(fn walk-json-pages [url-formatter path response-destructor]
  (fn gather [page knil]
    (local url (url-formatter path page))
    (print (.. "requesting " url))
    (local {: status : body}
      (must
        (luna.http.request
         "GET" url
         {:User-Agent (http.random-user-agent)
          :Content-Type "application/json"
          :Accept "application/json"}
         "")))

    (if (= status 200)
     (let [{: items} (response-destructor body)]
      (if (or (= items nil) (= 0 (# items)))
        knil
        (do
         (os.execute "sleep 1")
         (gather (+ 1 page) (array.concat knil items)))))
      knil))

  (gather 1 []))

(fn from-json [categories url-formatter response-destructor normalizer]
  (reduce
   (fn [_ {: tags : path} result]
     (array.concat
       result
       (categorize-many
        (map #(normalizer $2)
             (walk-json-pages url-formatter path response-destructor))
        tags)))
   categories
   []))

{: from-html
 : from-json}