diff options
Diffstat (limited to 'parser')
| -rw-r--r-- | parser/artoftea.fnl | 5 | ||||
| -rw-r--r-- | parser/ipuer.fnl | 4 | ||||
| -rw-r--r-- | parser/ozchai.fnl | 68 |
3 files changed, 29 insertions, 48 deletions
diff --git a/parser/artoftea.fnl b/parser/artoftea.fnl index 1f03ed1..205d5d0 100644 --- a/parser/artoftea.fnl +++ b/parser/artoftea.fnl @@ -8,7 +8,7 @@ (local number (require :lib.number)) (local fetcher (require :fetcher)) -(fn url-formatter [path page] +(fn format-url [path page] (.. "https://artoftea.ru/" path "/?page=" page)) (local product-peg @@ -50,6 +50,7 @@ (local price (number.string->number product.price)) {:site "artoftea" :id product.id + :title product.title :url product.url :description product.description :image product.image @@ -64,7 +65,7 @@ (fetcher.from-html [{:path "redtea" :category "Красный чай"} {:path "greentea" :category "Зеленый чай"}] - url-formatter + format-url product-peg normalize)) diff --git a/parser/ipuer.fnl b/parser/ipuer.fnl index 7fefd1b..11e63c0 100644 --- a/parser/ipuer.fnl +++ b/parser/ipuer.fnl @@ -8,7 +8,7 @@ (local parser (require :parser.parser)) (local fetcher (require :fetcher)) -(fn url-formatter [path page] +(fn format-url [path page] (.. "https://ipuer.ru/catalog/" path "/?p=" page)) (local product-peg @@ -63,7 +63,7 @@ {:path "blagovoniya" :category "Благовония"} {:path "posuda" :category "Посуда"} {:path "282" :category "Посуда"}] - url-formatter + format-url product-peg normalize)) diff --git a/parser/ozchai.fnl b/parser/ozchai.fnl index 6bf6286..50bfd6a 100644 --- a/parser/ozchai.fnl +++ b/parser/ozchai.fnl @@ -1,52 +1,28 @@ (import-macros {: map} :lib.macro) -(local http (require :lib.http)) (local array (require :lib.array)) +(local http (require :lib.http)) +(local number (require :lib.number)) +(local fetcher (require :fetcher)) (local json (require :vendor.json)) -(local %all-products-partuid 176163172341) - -(fn string->number [str] - (if str - (tonumber (pick-values 1 (str:gsub "[^0-9.]" ""))) - nil)) - -(fn request [partuid slice] - (print (.. "https://store.tildaapi.com/api/getproductslist/" - "?storepartuid=" - partuid - "&recid=280779251&c=1723216515077" - "&getparts=true&getoptions=true&slice=%d&size=36")) - (let [(status headers body) - (luna.http.request - "GET" - (string.format - (.. "https://store.tildaapi.com/api/getproductslist/" - "?storepartuid=" - partuid - "&recid=280779251&c=1723216515077" - "&getparts=true&getoptions=true&slice=%d&size=36") - slice) - {:Content-Type "application/json" - :User-Agent (http.random-user-agent)} - "")] - (json.decode body))) +(fn format-url [path page] + (.. "https://store.tildaapi.com/api/getproductslist/" + "?storepartuid=" path + "&slice=" page + "&recid=280779251" + "&c=1723216515077" + "&getparts=true" + "&getoptions=true" + "&size=36")) -(fn walk-slices [partuid] - (fn gather [slice knil] - (let [{: nextslice : products} (request partuid slice) - res (array.concat knil products)] - (if (= 0 (# products)) - knil - (do - (os.execute "sleep 1") - (gather (+ slice 1) res))))) - (gather 1 [])) +(fn destruct-response [response] + {:items response.products}) -(fn normalize [_ product] +(fn normalize [product] (local gallery (json.decode product.gallery)) - (local weight (string->number (. (. product.editions 1) :Вес))) - (local price (string->number (. (. product.editions 1) :price))) + (local weight (number.string->number (. product.editions 1 :Вес))) + (local price (number.string->number (. product.editions 1 :price))) {:site "ozchai" :id product.url :url product.url @@ -54,8 +30,8 @@ :description product.descr ;; FIXME: parse all editions into different projects :image (if (< 0 (# gallery)) - (. (. gallery 1) :img) - "") + (. gallery 1 :img) + "") :weight weight :price price :price-per (if (and price weight (< 0 weight)) @@ -64,6 +40,10 @@ :characteristics product.characteristics}) (fn products [] - (map normalize (walk-slices %all-products-partuid))) + (fetcher.from-json + [{:path "176163172341"}] + format-url + destruct-response + normalize)) {: products} |