summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorunwox <me@unwox.com>2024-10-10 11:51:21 +0600
committerunwox <me@unwox.com>2024-10-10 11:51:38 +0600
commitdfcdaab3a84389e610a57fa82ee0d3a216f4821d (patch)
treee0318abc43baaf83987381312a02c97ffe8b6f9b
parentddc11ff1cc31f17ff46523a649917940357825b5 (diff)
allow to specify several tags for a product (instead of one category)
-rw-r--r--bin/fetch.fnl106
-rw-r--r--bin/serve.fnl56
-rw-r--r--fetcher.fnl18
-rw-r--r--lib/array.fnl43
-rw-r--r--lib/string.fnl2
-rw-r--r--parser/artoftea.fnl22
-rw-r--r--parser/chaekshop.fnl52
-rw-r--r--parser/clubcha.fnl80
-rw-r--r--parser/ipuer.fnl10
-rw-r--r--parser/parser.fnl50
10 files changed, 270 insertions, 169 deletions
diff --git a/bin/fetch.fnl b/bin/fetch.fnl
index 3dcaded..c0f07c6 100644
--- a/bin/fetch.fnl
+++ b/bin/fetch.fnl
@@ -2,6 +2,8 @@
(tset package :path (.. package.path ";./vendor/lpeglj/?.lua"))
+(local fennel (require :vendor.fennel))
+
(local array (require :lib.array))
(local ozchai (require :parser.ozchai))
(local ipuer (require :parser.ipuer))
@@ -15,67 +17,125 @@
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
- CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(name, fid, `table`);
+ CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(title, fid, `table`);
- CREATE TABLE IF NOT EXISTS products (
- id TEXT NOT NULL PRIMARY KEY,
+ CREATE TABLE IF NOT EXISTS products(
+ url TEXT NOT NULL PRIMARY KEY,
site TEXT NOT NULL,
- category TEXT NOT NULL,
title TEXT NOT NULL,
description TEXT NOT NULL,
year INT NOT NULL,
image TEXT NOT NULL,
- url TEXT NOT NULL,
price REAL NOT NULL,
weight REAL NOT NULL,
price_per REAL NOT NULL,
- misc TEXT NOT NULL,
+ archived BOOL NOT NULL,
+ creation_time DATETIME NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS product_tags(
+ product TEXT NOT NULL REFERENCES products(url),
+ tag DATETIME NOT NULL REFERENCES tags(title)
+ );
+
+ CREATE UNIQUE INDEX IF NOT EXISTS product_tags_idx ON product_tags(product, tag);
+
+ CREATE TABLE IF NOT EXISTS tags(
+ title TEXT NOT NULL PRIMARY KEY,
creation_time DATETIME NOT NULL
);" [])
(fn now []
(os.date "%Y-%m-%d %H:%M:%S"))
-(fn store-products [products]
+(fn store-tags [tx tags]
+ (when (< 0 (# tags))
+ (local sql
+ (.. "INSERT OR IGNORE INTO tags VALUES "
+ (array.join
+ (map (fn [_ _] "(?, ?)") tags)
+ ", ")))
+ (local vars
+ (reduce
+ (fn [_ tag rest] (array.concat rest [tag (now)]))
+ tags []))
+ (luna.db.exec-tx tx sql vars)))
+
+(fn store-product-tags [tx products]
+ (when (< 0 (# products))
+ ;; flatten product tags
+ (local tags
+ (reduce
+ (fn [_ product res]
+ (array.concat
+ res
+ (map
+ (fn [_ tag] [product.url tag])
+ product.tags)))
+ products
+ []))
+ (local sql
+ (.. "INSERT OR IGNORE INTO product_tags VALUES "
+ (array.join
+ (map (fn [_ _] "(?, ?)")
+ tags)
+ ", ")))
+ (local vars
+ (reduce
+ (fn [_ tags rest]
+ (array.concat rest tags))
+ tags []))
+ (luna.db.exec-tx tx sql vars)))
+
+(fn store-products [tx products]
(when (< 0 (# products))
(local sql
- (.. "INSERT OR REPLACE INTO products VALUES "
- (table.concat
- (map (fn [_ _]
- "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+ (.. "INSERT OR IGNORE INTO products VALUES "
+ (array.join
+ (map (fn [_ _] "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
products)
- ",")))
+ ", ")))
(local vars
(reduce
(fn [_ product rest]
(array.concat rest
- [(or product.id "")
+ [(or product.url "")
(or product.site "")
- (or product.category "")
(or product.title "")
(or product.description "")
(or product.year 0)
(or product.image "")
- (or product.url "")
(or product.price 0)
(or product.weight 0)
(or product.price-per 0)
- (or product.misc "")
+ false
(now)]))
products []))
- (luna.db.exec db sql vars)))
+ (luna.db.exec-tx tx sql vars)
+
+ ;; store tags
+ (store-tags tx (array.unique
+ (array.flatten
+ (map (fn [_ v] v.tags) products))))
+ (store-product-tags tx products)))
(fn populate-search-table []
(local tx (luna.db.begin db))
(luna.db.exec-tx tx "DELETE FROM search" [])
(luna.db.exec-tx tx "INSERT INTO search
- SELECT title, id, 'products'
+ SELECT title, url, 'products'
FROM products;" [])
(luna.db.commit tx))
-(store-products (chaekshop.products))
-(store-products (clubcha.products))
-(store-products (artoftea.products))
-(store-products (ipuer.products))
-(store-products (ozchai.products))
+;; replace with with-tx
+(local tx (luna.db.begin db))
+(luna.db.exec-tx tx "DELETE FROM product_tags;" [])
+(each [_ products (pairs [chaekshop.products
+ clubcha.products
+ artoftea.products
+ ipuer.products
+ ozchai.products])]
+ (store-products tx (products)))
+(luna.db.commit tx)
+
(populate-search-table)
diff --git a/bin/serve.fnl b/bin/serve.fnl
index bec5a94..d4dbf4c 100644
--- a/bin/serve.fnl
+++ b/bin/serve.fnl
@@ -105,9 +105,23 @@
[:div {} "Всего результатов: " [:strong {} (string.format "%d" total)]]]
""))
-(fn base-template [query sort page total ...]
+(fn aside-template [query tags paginator]
+ [:aside {:class "aside"}
+ [:div {:class "aside-content"}
+ (if (~= query "")
+ [:a {:href "/" :style "display: block;"}
+ [:img {:class "logo" :src "static/logo.svg"
+ :alt "Логотип meicha.ru" :title "Логотип meicha.ru"}]]
+ [:img {:class "logo" :src "static/logo.svg"
+ :alt "Логотип meicha.ru" :title "Логотип meicha.ru"}])
+ [:form {:class "form"}
+ [:input {:type :search :name :query :value query
+ :autofocus true :placeholder "Поисковый запрос"}]
+ [:button {:type :submit} "Искать"]]
+ paginator]])
+
+(fn base-template [query tags page total ...]
(local paginator (paginator-template query page 48 total))
-
[:html {:lang "en"}
[:head {}
[:meta {:charset "UTF-8"}]
@@ -116,19 +130,7 @@
[:body {}
[:div {:class "container"}
[:div {:class "content"}
- [:aside {:class "aside"}
- [:div {:class "aside-content"}
- (if (~= query "")
- [:a {:href "/" :style "display: block;"}
- [:img {:class "logo" :src "static/logo.svg"
- :alt "Логотип meicha.ru" :title "Логотип meicha.ru"}]]
- [:img {:class "logo" :src "static/logo.svg"
- :alt "Логотип meicha.ru" :title "Логотип meicha.ru"}])
- [:form {:class "form"}
- [:input {:type :search :name :query :value query
- :autofocus true :placeholder "Поисковый запрос"}]
- [:button {:type :submit} "Искать"]]
- paginator]]
+ (aside-template query tags paginator)
[:section {}
[:div {:class "list"} ...]
[:footer {} paginator]]]]]])
@@ -137,8 +139,7 @@
(assert (< 0 limit) "limit must be > 0")
(luna.db.query*
db
- "SELECT id,
- site,
+ "SELECT site,
title,
description,
image,
@@ -167,14 +168,13 @@
"SELECT count(*)
FROM search
WHERE search.`table` = 'products'
- AND search.name MATCH ?"
+ AND search.title MATCH ?"
[query]))
{:results
(luna.db.query*
db
- "SELECT products.id,
- highlight(search, 0, '<i>', '</i>') AS \"title\",
+ "SELECT highlight(search, 0, '<i>', '</i>') AS \"title\",
products.site,
products.description,
products.image,
@@ -182,11 +182,13 @@
products.price,
products.weight,
products.price_per AS \"price-per\",
- products.year
+ products.year,
+ products.archived,
+ products.creation_time AS \"creation-time\"
FROM search
- INNER JOIN products ON search.fid = products.id
+ INNER JOIN products ON search.fid = products.url
WHERE search.`table` = 'products'
- AND search.name MATCH ?
+ AND search.title MATCH ?
ORDER BY rank
LIMIT 48 OFFSET ?"
[query (* (- page 1) 48)])
@@ -198,17 +200,17 @@
(if (= path "/")
(let [headers {:content-type "text/html"}
page (or (tonumber query.page) 1)
- search (or query.query "")
- sort "ASC"
+ search (str.trim (or query.query ""))
+ tags (or query.tags [])
{: results : total}
(if (~= "" search)
- (query-products page search sort)
+ (query-products page search)
{:total 48 :results (random-products 48)})]
(values
200 headers
(html.render
(base-template
- search sort page total
+ search tags page total
(table.unpack (map #(item-template $2) results)))
true)))
(values 404 {} "not found")))
diff --git a/fetcher.fnl b/fetcher.fnl
index bf22abf..6a133f0 100644
--- a/fetcher.fnl
+++ b/fetcher.fnl
@@ -49,25 +49,25 @@
(gather 1 []))
-(fn categorize-many [items category]
+(fn categorize-many [items tags]
(map
(fn [_ item]
- (tset item :category
- (if category
- category
- (parser.guess-category item.title)))
+ (tset item :tags
+ (-> (or tags [])
+ (array.concat (parser.guess-tags item.title))
+ (array.unique)))
item)
items))
(fn from-html [categories url-formatter item-peg normalizer]
(reduce
- (fn [_ {: category : path} result]
+ (fn [_ {: tags : path} result]
(array.concat
result
(categorize-many
(map #(normalizer $2)
(walk-html-pages url-formatter path item-peg))
- category)))
+ tags)))
categories
[]))
@@ -98,13 +98,13 @@
(fn from-json [categories url-formatter response-destructor normalizer]
(reduce
- (fn [_ {: category : path} result]
+ (fn [_ {: tags : path} result]
(array.concat
result
(categorize-many
(map #(normalizer $2)
(walk-json-pages url-formatter path response-destructor))
- category)))
+ tags)))
categories
[]))
diff --git a/lib/array.fnl b/lib/array.fnl
index 11e4d2a..9ed0d80 100644
--- a/lib/array.fnl
+++ b/lib/array.fnl
@@ -4,10 +4,45 @@
(local copy
(fn [a b]
(reduce
- (fn [_ v c]
- (table.insert c v)
- c)
+ (fn [_ v res]
+ (table.insert res v)
+ res)
b a)))
(copy a b))
-{: concat}
+(fn flip [a]
+ (collect [k v (pairs a)]
+ (values v k)))
+
+(fn diff [a b]
+ (local flip-b (flip b))
+ (reduce
+ (fn [_ v res]
+ (when (not (. flip-b v))
+ (table.insert res v))
+ res)
+ a []))
+
+(fn unique [a]
+ (var hash {})
+ (reduce
+ (fn [_ v res]
+ (when (not (. hash v))
+ (tset hash v true)
+ (table.insert res v))
+ res)
+ a []))
+
+(fn flatten [a]
+ (reduce
+ (fn [_ value rest]
+ (concat
+ rest
+ (if (= (type value) "table")
+ (flatten value)
+ [value])))
+ a []))
+
+(local join table.concat)
+
+{: concat : diff : unique : flatten : join}
diff --git a/lib/string.fnl b/lib/string.fnl
index d82c658..9ac6edd 100644
--- a/lib/string.fnl
+++ b/lib/string.fnl
@@ -19,4 +19,4 @@
(.. (trim (luna.utf.sub str 1 len)) (or ellipsis "...")))
""))
-{: split : ends-with : truncate}
+{: split : ends-with : trim : truncate}
diff --git a/parser/artoftea.fnl b/parser/artoftea.fnl
index 1b8b6e1..f053833 100644
--- a/parser/artoftea.fnl
+++ b/parser/artoftea.fnl
@@ -64,18 +64,18 @@
(fn products []
(fetcher.from-html
- [{:path "redtea" :category "Красный чай"}
- {:path "greentea" :category "Зеленый чай"}
- {:path "puer/shu-puer" :category "Шу пуэр"}
- {:path "puer/sheng-puer" :category "Шен пуэр"}
+ [{:path "redtea" :tags ["Красный чай"]}]
+ {:path "greentea" :tags ["Зеленый чай"]}
+ {:path "puer/shu-puer" :tags ["Шу пуэр"]}
+ {:path "puer/sheng-puer" :tags ["Шен пуэр"]}
{:path "rassypnoi-puer"}
- {:path "oolong" :category "Улун"}
- {:path "whitetea" :category "Белый чай"}
- {:path "yellowtea" :category "Желтый чай"}
- {:path "xej-cha-chernyj-chaj" :category "Хэй ча"}
- {:path "posuda" :category "Посуда"}
- {:path "tipoty-lightking" :category "Посуда"}
- {:path "accesories" :category "Аксессуары"}
+ {:path "oolong" :tags ["Улун"]}
+ {:path "whitetea" :tags ["Белый чай"]}
+ {:path "yellowtea" :tags ["Желтый чай"]}
+ {:path "xej-cha-chernyj-chaj" :tags ["Хэй ча"]}
+ {:path "posuda" :tags ["Посуда"]}
+ {:path "tipoty-lightking" :tags ["Посуда"]}
+ {:path "accesories" :tags ["Аксессуары"]}
{:path "matcha-i-aksessuary"}
{:path "upakovka"}
{:path "eksklyuzivny-chay"}
diff --git a/parser/chaekshop.fnl b/parser/chaekshop.fnl
index b6e012a..13be8f1 100644
--- a/parser/chaekshop.fnl
+++ b/parser/chaekshop.fnl
@@ -52,35 +52,35 @@
(fn products []
(fetcher.from-html
- [{:path "chay/belyy_chay" :category "Белый чай"}
- {:path "chay/guandunskie_uluny" :category "Улун"}
- {:path "chay/zhyeltyy_chay" :category "Желтый чай"}
- {:path "chay/zelyenyy_chay" :category "Зеленый чай"}
- {:path "chay/krasnyy_chay" :category "Красный чай"}
- {:path "chay/severo_futszyanskie_uluny" :category "Улун"}
- {:path "chay/tayvanskie_uluny" :category "Улун"}
- {:path "chay/khey_cha" :category "Хэй ча"}
- {:path "chay/shen_puer" :category "Шэн пуэр"}
- {:path "chay/shu_puery" :category "Шу пуэр"}
- {:path "chay/yuzhno_futszyanskie_uluny_" :category "Улун"}
- {:path "chay/yunnanskie_uluny" :category "Улун"}
- {:path "chay/gaba_chay" :category "Улун"}
+ [{:path "chay/belyy_chay" :tags ["Белый чай"]}
+ {:path "chay/guandunskie_uluny" :tags ["Улун"]}
+ {:path "chay/zhyeltyy_chay" :tags ["Желтый чай"]}
+ {:path "chay/zelyenyy_chay" :tags ["Зеленый чай"]}
+ {:path "chay/krasnyy_chay" :tags ["Красный чай"]}
+ {:path "chay/severo_futszyanskie_uluny" :tags ["Улун"]}
+ {:path "chay/tayvanskie_uluny" :tags ["Улун"]}
+ {:path "chay/khey_cha" :tags ["Хэй ча"]}
+ {:path "chay/shen_puer" :tags ["Шен пуэр"]}
+ {:path "chay/shu_puery" :tags ["Шу пуэр"]}
+ {:path "chay/yuzhno_futszyanskie_uluny_" :tags ["Улун"]}
+ {:path "chay/yunnanskie_uluny" :tags ["Улун"]}
+ {:path "chay/gaba_chay" :tags ["Улун"]}
{:path "chay/na_kazhdyy_den"}
{:path "chay/eksklyuziv"}
- {:path "posuda_i_aksessuary/pialy" :category "Посуда"}
- {:path "posuda_i_aksessuary/isinskaya_glina" :category "Посуда"}
- {:path "posuda_i_aksessuary/keramika_iz_tszindechzhen" :category "Посуда"}
- {:path "posuda_i_aksessuary/keramika_iz_tszyanshuy" :category "Посуда"}
- {:path "posuda_i_aksessuary/kolby_termosy_nabory" :category "Посуда"}
- {:path "posuda_i_aksessuary/chabani" :category "Посуда"}
- {:path "posuda_i_aksessuary/chaynye_figurki" :category "Посуда"}
- {:path "posuda_i_aksessuary/slivniki" :category "Посуда"}
- {:path "posuda_i_aksessuary/gayvani" :category "Посуда"}
- {:path "posuda_i_aksessuary/keramika_iz_tsinchzhou" :category "Посуда"}
+ {:path "posuda_i_aksessuary/pialy" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/isinskaya_glina" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/keramika_iz_tszindechzhen" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/keramika_iz_tszyanshuy" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/kolby_termosy_nabory" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/chabani" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/chaynye_figurki" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/slivniki" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/gayvani" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/keramika_iz_tsinchzhou" :tags ["Посуда"]}
{:path "posuda_i_aksessuary/aksessuary"}
- {:path "posuda_i_aksessuary/pialy_tszindechzhen" :category "Посуда"}
- {:path "posuda_i_aksessuary/keramika_dekhua" :category "Посуда"}
- {:path "posuda_i_aksessuary/chayniki_iz_farfora" :category "Посуда"}]
+ {:path "posuda_i_aksessuary/pialy_tszindechzhen" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/keramika_dekhua" :tags ["Посуда"]}
+ {:path "posuda_i_aksessuary/chayniki_iz_farfora" :tags ["Посуда"]}]
format-url
product-peg
normalize))
diff --git a/parser/clubcha.fnl b/parser/clubcha.fnl
index e260fd1..200be94 100644
--- a/parser/clubcha.fnl
+++ b/parser/clubcha.fnl
@@ -68,53 +68,53 @@
(fn products []
(fetcher.from-html
- [{:path "shu-puer" :category "Шу пуэр"}
- {:path "shen-puer" :category "Шен пуэр"}
+ [{:path "shu-puer" :tags ["Шу пуэр"]}
+ {:path "shen-puer" :tags ["Шен пуэр"]}
{:path "bai-hao-puer"}
- {:path "hei-cha" :category "Хэй ча"}
- {:path "yuzhnofutszyanskij-ulun" :category "Улун"}
- {:path "guandunskij-ulun" :category "Улун"}
- {:path "severofujianskiy-ulun" :category "Улун"}
- {:path "taiwanskiy-ulun" :category "Улун"}
- {:path "gaba-ulun-gaba-alishan-i-dr" :category "Улун"}
- {:path "zelenyi-chay" :category "Зеленый чай"}
- {:path "belyi-chay" :category "Белый чай"}
- {:path "zheltyi-chay" :category "Желтый чай"}
- {:path "kitajskij-krasnyj-chaj" :category "Красный чай"}
- {:path "tajvanskij-krasnyj-chaj" :category "Красный чай"}
+ {:path "hei-cha" :tags ["Хэй ча"]}
+ {:path "yuzhnofutszyanskij-ulun" :tags ["Улун"]}
+ {:path "guandunskij-ulun" :tags ["Улун"]}
+ {:path "severofujianskiy-ulun" :tags ["Улун"]}
+ {:path "taiwanskiy-ulun" :tags ["Улун"]}
+ {:path "gaba-ulun-gaba-alishan-i-dr" :tags ["Улун"]}
+ {:path "zelenyi-chay" :tags ["Зеленый чай"]}
+ {:path "belyi-chay" :tags ["Белый чай"]}
+ {:path "zheltyi-chay" :tags ["Желтый чай"]}
+ {:path "kitajskij-krasnyj-chaj" :tags ["Красный чай"]}
+ {:path "tajvanskij-krasnyj-chaj" :tags ["Красный чай"]}
{:path "Nechaynyye_chai_i_tsvetochnyye_dobavki"}
{:path "upakovka-dlya-puera"}
{:path "upakovka-dlya-posudy"}
{:path "iz-lichnoj-kollektsii"}
- ;; FIXME: expand this category to subcategories because the main one
+ ;; FIXME: expand this tags [to subcategories because the main one]
;; does not contain all the products
- {:path "gaivan" :category "Посуда"}
- {:path "chashka" :category "Посуда"}
- {:path "chaynyi-nabor" :category "Посуда"}
- {:path "chaynik" :category "Посуда"}
- {:path "glina-farfor" :category "Посуда"}
- {:path "chaban-chajnaya-doska" :category "Посуда"}
- {:path "chajnyj-podnos" :category "Посуда"}
- {:path "chajnyj-prud" :category "Посуда"}
- {:path "professionalnye" :category "Посуда"}
- {:path "glinyanye" :category "Посуда"}
- {:path "farforovye" :category "Посуда"}
- {:path "zhestyanye-i-kartonnye" :category "Посуда"}
- {:path "puernitsy" :category "Посуда"}
- {:path "glina" :category "Посуда"}
- {:path "farfor" :category "Посуда"}
- {:path "steklo" :category "Посуда"}
- {:path "kruzhka" :category "Посуда"}
- {:path "Chaynyye_instrumenty" :category "Посуда"}
- {:path "chahe" :category "Посуда"}
- {:path "sito" :category "Посуда"}
- {:path "Chaynyy_suvenir" :category "Посуда"}
- {:path "Prochaya_utvar" :category "Посуда"}
- {:path "farfor-eilong" :category "Посуда"}
- {:path "butylka-dlya-zavarivaniya" :category "Посуда"}
+ {:path "gaivan" :tags ["Посуда"]}
+ {:path "chashka" :tags ["Посуда"]}
+ {:path "chaynyi-nabor" :tags ["Посуда"]}
+ {:path "chaynik" :tags ["Посуда"]}
+ {:path "glina-farfor" :tags ["Посуда"]}
+ {:path "chaban-chajnaya-doska" :tags ["Посуда"]}
+ {:path "chajnyj-podnos" :tags ["Посуда"]}
+ {:path "chajnyj-prud" :tags ["Посуда"]}
+ {:path "professionalnye" :tags ["Посуда"]}
+ {:path "glinyanye" :tags ["Посуда"]}
+ {:path "farforovye" :tags ["Посуда"]}
+ {:path "zhestyanye-i-kartonnye" :tags ["Посуда"]}
+ {:path "puernitsy" :tags ["Посуда"]}
+ {:path "glina" :tags ["Посуда"]}
+ {:path "farfor" :tags ["Посуда"]}
+ {:path "steklo" :tags ["Посуда"]}
+ {:path "kruzhka" :tags ["Посуда"]}
+ {:path "Chaynyye_instrumenty" :tags ["Посуда"]}
+ {:path "chahe" :tags ["Посуда"]}
+ {:path "sito" :tags ["Посуда"]}
+ {:path "Chaynyy_suvenir" :tags ["Посуда"]}
+ {:path "Prochaya_utvar" :tags ["Посуда"]}
+ {:path "farfor-eilong" :tags ["Посуда"]}
+ {:path "butylka-dlya-zavarivaniya" :tags ["Посуда"]}
{:path "Predmety_interyera_i_fenshuy"}
- {:path "fo-chzhu-chetki" :category "Четки"}
- {:path "nefrit-1" :category "Нефрит"}]
+ {:path "fo-chzhu-chetki" :tags ["Четки"]}
+ {:path "nefrit-1" :tags ["Нефрит"]}]
format-url
product-peg
normalize))
diff --git a/parser/ipuer.fnl b/parser/ipuer.fnl
index 8b0f6be..b16ead5 100644
--- a/parser/ipuer.fnl
+++ b/parser/ipuer.fnl
@@ -59,12 +59,12 @@
(fn products []
(fetcher.from-html
- [{:path "shen-puer" :category "Шен пуэр"}
- {:path "shu-puer" :category "Шу пуэр"}
+ [{:path "shen-puer" :tags ["Шен пуэр"]}
+ {:path "shu-puer" :tags ["Шу пуэр"]}
{:path "drugoy-chay"}
- {:path "blagovoniya" :category "Благовония"}
- {:path "posuda" :category "Посуда"}
- {:path "282" :category "Посуда"}]
+ {:path "blagovoniya" :tags ["Благовония"]}
+ {:path "posuda" :tags ["Посуда"]}
+ {:path "282" :tags ["Посуда"]}]
format-url
product-peg
normalize))
diff --git a/parser/parser.fnl b/parser/parser.fnl
index d5baafa..ca74847 100644
--- a/parser/parser.fnl
+++ b/parser/parser.fnl
@@ -105,39 +105,43 @@
:match html))
;; FIXME: make guessing case insensitive
-(fn guess-category [title]
- (local title (if title (luna.utf.lower title) ""))
+(fn guess-tags [text]
+ (local text (if text (luna.utf.lower text) ""))
- (if (: (anywhere (peg.P "зеленый")) :match title)
- "Зеленый чай"
- (: (anywhere (peg.P "улун")) :match title)
- "Улун"
- (: (anywhere (peg.P "белый")) :match title)
- "Белый чай"
- (: (anywhere (peg.P "желтый")) :match title)
- "Желтый чай"
- (: (anywhere (peg.P "красный")) :match title)
- "Красный чай"
- (: (anywhere (peg.P "хэй ча")) :match title)
- "Хэй ча"
- (: (anywhere (+ (peg.P "матча") "маття")) :match title)
- "Матча"
- "Неизвестная категория"))
+ (if (: (anywhere (peg.P "зеленый")) :match text)
+ ["Зеленый чай"]
+ (: (anywhere (peg.P "улун")) :match text)
+ ["Улун"]
+ (: (anywhere (peg.P "белый")) :match text)
+ ["Белый чай"]
+ (: (anywhere (peg.P "желтый")) :match text)
+ ["Желтый чай"]
+ (: (anywhere (peg.P "красный")) :match text)
+ ["Красный чай"]
+ (: (anywhere (peg.P "хэй ча")) :match text)
+ ["Хэй ча"]
+ (: (anywhere (peg.P "шу пуэр")) :match text)
+ ["Шу пуэр"]
+ (: (anywhere (+ (peg.P "шен пуэр") "шэн пуэр")) :match text)
+ ["Шен пуэр"]
+ (: (anywhere (+ (peg.P "матча") "маття")) :match text)
+ ["Матча"]
+ []))
-(fn guess-year [title]
+(fn guess-year [text]
(number.string->number
(: (anywhere
(* (peg.C (^ (peg.R "09") 4))
(maybe " ")
(- (+ (peg.P "г") "год") (peg.P "гр"))))
- :match title)))
+ :match text)))
-(fn guess-weight [title]
- (if title
+(fn guess-weight [text]
+ (if text
(number.string->number
(: (anywhere
(* (peg.C pegs.number) (maybe " ") "гр"))
- :match title))
+ :match text))
nil))
{: match-many
@@ -147,6 +151,6 @@
: maybe
: pegs
:not pnot
- : guess-category
+ : guess-tags
: guess-year
: guess-weight}