summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bin/fetch.fnl32
-rw-r--r--fetcher.fnl31
-rw-r--r--lib/utils.fnl5
-rw-r--r--parser/clubcha.fnl2
-rw-r--r--parser/parser.fnl3
-rw-r--r--spellfix.fnl7
6 files changed, 45 insertions, 35 deletions
diff --git a/bin/fetch.fnl b/bin/fetch.fnl
index c0f07c6..3ef57df 100644
--- a/bin/fetch.fnl
+++ b/bin/fetch.fnl
@@ -10,9 +10,11 @@
(local artoftea (require :parser.artoftea))
(local clubcha (require :parser.clubcha))
(local chaekshop (require :parser.chaekshop))
+(local {: must} (require :lib.utils))
-(local db (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL"))
-(luna.db.exec db "
+(local db (must (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL")))
+(must
+ (luna.db.exec db "
PRAGMA foreign_keys=ON;
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
@@ -43,7 +45,7 @@
CREATE TABLE IF NOT EXISTS tags(
title TEXT NOT NULL PRIMARY KEY,
creation_time DATETIME NOT NULL
- );" [])
+ );" []))
(fn now []
(os.date "%Y-%m-%d %H:%M:%S"))
@@ -59,7 +61,7 @@
(reduce
(fn [_ tag rest] (array.concat rest [tag (now)]))
tags []))
- (luna.db.exec-tx tx sql vars)))
+ (must (luna.db.exec-tx tx sql vars))))
(fn store-product-tags [tx products]
(when (< 0 (# products))
@@ -85,7 +87,7 @@
(fn [_ tags rest]
(array.concat rest tags))
tags []))
- (luna.db.exec-tx tx sql vars)))
+ (must (luna.db.exec-tx tx sql vars))))
(fn store-products [tx products]
(when (< 0 (# products))
@@ -111,7 +113,7 @@
false
(now)]))
products []))
- (luna.db.exec-tx tx sql vars)
+ (must (luna.db.exec-tx tx sql vars))
;; store tags
(store-tags tx (array.unique
@@ -120,22 +122,22 @@
(store-product-tags tx products)))
(fn populate-search-table []
- (local tx (luna.db.begin db))
- (luna.db.exec-tx tx "DELETE FROM search" [])
- (luna.db.exec-tx tx "INSERT INTO search
- SELECT title, url, 'products'
- FROM products;" [])
- (luna.db.commit tx))
+ (local tx (must (luna.db.begin db)))
+ (must (luna.db.exec-tx tx "DELETE FROM search" []))
+ (must (luna.db.exec-tx tx "INSERT INTO search
+ SELECT title, url, 'products'
+ FROM products;" []))
+ (must (luna.db.commit tx)))
;; replace with with-tx
-(local tx (luna.db.begin db))
-(luna.db.exec-tx tx "DELETE FROM product_tags;" [])
+(local tx (must (luna.db.begin db)))
+(must (luna.db.exec-tx tx "DELETE FROM product_tags;" []))
(each [_ products (pairs [chaekshop.products
clubcha.products
artoftea.products
ipuer.products
ozchai.products])]
(store-products tx (products)))
-(luna.db.commit tx)
+(must (luna.db.commit tx))
(populate-search-table)
diff --git a/fetcher.fnl b/fetcher.fnl
index 6a133f0..b3db8f0 100644
--- a/fetcher.fnl
+++ b/fetcher.fnl
@@ -8,6 +8,7 @@
(local json (require :vendor.json))
(local parser (require :parser.parser))
(local http (require :lib.http))
+(local {: must} (require :lib.utils))
(fn retry [what times sleep]
(var result nil)
@@ -33,17 +34,18 @@
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
- (local (status _ html)
- (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} ""))
+ (local {: status : body}
+ (must
+ (luna.http.request "GET" url {:User-Agent (http.random-user-agent)} "")))
- (if (= status 200)
- (let [items (parser.match-many html item-peg)]
+ (if (= 200 status)
+ (let [items (parser.match-many body item-peg)]
(if (or (= items nil) (= 0 (# items)))
knil
(do
- (os.execute "sleep 1")
+ (os.execute "sleep .3")
(gather (+ page 1) (array.concat knil items)))))
- (= status 404)
+ (= 404 status)
knil
(retry #(gather page knil) 3 1)))
@@ -75,16 +77,17 @@
(fn gather [page knil]
(local url (url-formatter path page))
(print (.. "requesting " url))
- (local (status _ content)
- (luna.http.request
- "GET" url
- {:User-Agent (http.random-user-agent)
- :Content-Type "application/json"
- :Accept "application/json"}
- ""))
+ (local {: status : body}
+ (must
+ (luna.http.request
+ "GET" url
+ {:User-Agent (http.random-user-agent)
+ :Content-Type "application/json"
+ :Accept "application/json"}
+ "")))
(if (= status 200)
- (let [{: items} (response-destructor (json.decode content))]
+ (let [{: items} (response-destructor (json.decode body))]
(if (or (= items nil) (= 0 (# items)))
knil
(do
diff --git a/lib/utils.fnl b/lib/utils.fnl
new file mode 100644
index 0000000..3220952
--- /dev/null
+++ b/lib/utils.fnl
@@ -0,0 +1,5 @@
+(fn must [...]
+ (local (ok? result) ...)
+ (if ok? result (error result)))
+
+{: must}
diff --git a/parser/clubcha.fnl b/parser/clubcha.fnl
index 200be94..4bfe8ec 100644
--- a/parser/clubcha.fnl
+++ b/parser/clubcha.fnl
@@ -86,8 +86,6 @@
{:path "upakovka-dlya-puera"}
{:path "upakovka-dlya-posudy"}
{:path "iz-lichnoj-kollektsii"}
- ;; FIXME: expand this tags [to subcategories because the main one]
- ;; does not contain all the products
{:path "gaivan" :tags ["Посуда"]}
{:path "chashka" :tags ["Посуда"]}
{:path "chaynyi-nabor" :tags ["Посуда"]}
diff --git a/parser/parser.fnl b/parser/parser.fnl
index cb026bc..97cc6e7 100644
--- a/parser/parser.fnl
+++ b/parser/parser.fnl
@@ -1,6 +1,7 @@
(import-macros {: map} :lib.macro)
(local number (require :lib.number))
+(local {: must} (require :lib.utils))
(local peg
(if (pick-values 1 (pcall require :lpeg))
@@ -106,7 +107,7 @@
;; FIXME: make guessing case insensitive
(fn guess-tags [text]
- (local text (if text (luna.utf.lower text) ""))
+ (local text (if text (must (luna.utf.lower text)) ""))
(if (: (anywhere (peg.P "зеленый чай")) :match text)
["Зеленый чай"]
diff --git a/spellfix.fnl b/spellfix.fnl
index 3478782..24d381c 100644
--- a/spellfix.fnl
+++ b/spellfix.fnl
@@ -1,6 +1,7 @@
(import-macros {: map : filter} :lib.macro)
(local array (require :lib.array))
(local str (require :lib.string))
+(local {: must} (require :lib.utils))
(local vocabulary ["абрикосовый" "агар" "агарвуд" "агат" "агатис" "алая" "али"
"алишань" "алтарь" "альба" "амазонка" "аметист" "амино"
@@ -278,8 +279,8 @@
"ясный" "яшма"])
(fn levenshtein [str1 str2]
- (let [len1 (luna.utf.len str1)
- len2 (luna.utf.len str2)
+ (let [len1 (must (luna.utf.len str1))
+ len2 (must (luna.utf.len str2))
matrix {}]
(var cost 0)
(if (= len1 0) (lua "return len2")
@@ -289,7 +290,7 @@
(for [j 0 len2] (tset (. matrix 0) j j))
(for [i 1 len1]
(for [j 1 len2]
- (if (= (luna.utf.sub str1 i 1) (luna.utf.sub str2 j 1))
+ (if (= (must (luna.utf.sub str1 i 1)) (must (luna.utf.sub str2 j 1)))
(set cost 0) (set cost 1))
(tset (. matrix i) j
(math.min (+ (. matrix (- i 1) j) 1) (+ (. matrix i (- j 1)) 1)
generated by cgit v1.2.3 (git 2.39.1) at 2026-01-01 11:56:03 +0000