From 1eb0d60cad414b0cafe1c38b3a0b7823c0c8fb28 Mon Sep 17 00:00:00 2001
From: unwox <me@unwox.com>
Date: Wed, 29 Jan 2025 23:12:10 +0600
Subject: do not lock writes to db while running "./run.sh fetch"

---
 bin/fetch.fnl | 125 +++++++++++++++++++++++++++++++---------------------------
 bin/serve.fnl |   2 +-
 2 files changed, 67 insertions(+), 60 deletions(-)

(limited to 'bin')

diff --git a/bin/fetch.fnl b/bin/fetch.fnl
index 213475e..e7ee454 100644
--- a/bin/fetch.fnl
+++ b/bin/fetch.fnl
@@ -19,7 +19,7 @@
 (when _G.unpack
  (tset table :unpack _G.unpack))
 
-(local db (must (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL")))
+(local db (must (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL&_txlock=immediate")))
 (must
  (luna.db.exec db "
   PRAGMA foreign_keys=ON;
@@ -109,63 +109,70 @@
     (must (luna.db.exec-tx tx sql vars))))
 
 (fn store-products [tx products]
-  (when (< 0 (# products))
-    (local sql
-      (.. "INSERT OR REPLACE INTO products"
-          "(url, site, title, description, year, image, price, weight, volume, price_per, archived, creation_time)"
-          " VALUES "
-          (array.join
-            (map (fn [_ _] "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
-                 products)
-            ", ")))
-    (local creation-time (now))
-    (local vars
-      (reduce
-        (fn [_ product rest]
-          (array.concat rest
-            [(or product.url "")
-             (or product.site "")
-             (or product.title "")
-             (or product.description "")
-             (or product.year 0)
-             (or product.image "")
-             (or product.price 0)
-             (or product.weight 0)
-             (or product.volume 0)
-             (or product.price-per 0)
-             (or product.archived false)
-             creation-time]))
-        products []))
-    (must (luna.db.exec-tx tx sql vars))
-
-    ;; store tags
-    (store-tags tx (array.unique
-                    (array.flatten
-                     (map (fn [_ v] v.tags) products))))
-    (store-product-tags tx products)
-
-    ;; archive previous products
-    (local site (. products 1 :site))
-    (when site
-     (must (luna.db.exec-tx
-             tx
-             "UPDATE products
-              SET    archived = true
-              WHERE  site = ?
-              AND    creation_time < ?"
-             [site creation-time])))))
-
-(fn populate-search-table [tx]
+  (local sql
+    (.. "INSERT OR REPLACE INTO products"
+        "(url, site, title, description, year, image, price, weight, volume, price_per, archived, creation_time)"
+        " VALUES "
+        (array.join
+          (map (fn [_ _] "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+               products)
+          ", ")))
+  (local creation-time (now))
+  (local vars
+    (reduce
+      (fn [_ product rest]
+        (array.concat rest
+          [(or product.url "")
+           (or product.site "")
+           (or product.title "")
+           (or product.description "")
+           (or product.year 0)
+           (or product.image "")
+           (or product.price 0)
+           (or product.weight 0)
+           (or product.volume 0)
+           (or product.price-per 0)
+           (or product.archived false)
+           creation-time]))
+      products []))
+  (must (luna.db.exec-tx tx sql vars))
+
+  ;; store tags
+  (store-tags tx (array.unique
+                  (array.flatten
+                   (map (fn [_ v] v.tags) products))))
+  (store-product-tags tx products)
+
+  ;; archive previous products
+  (local site (. products 1 :site))
+  (when site
+   (must (luna.db.exec-tx
+           tx
+           "UPDATE products
+            SET    archived = true
+            WHERE  site = ?
+            AND    creation_time < ?"
+           [site creation-time]))))
+
+(fn populate-search-table [db]
+  (local tx (must (luna.db.begin db)))
   (must (luna.db.exec-tx tx "DELETE FROM search" []))
   (must (luna.db.exec-tx tx "INSERT INTO search
-                             SELECT title, url, 'products'
-                             FROM products;" [])))
-
-;; replace with with-tx
-(local tx (must (luna.db.begin db)))
-(must (luna.db.exec-tx tx "DELETE FROM product_tags;" []))
-(each [_ parser (pairs [suhexuan])]
-  (store-products tx (parser.products)))
-(cache.clear-tx tx "page:")
-(populate-search-table tx)
-(must (luna.db.commit tx))
+                              SELECT title, url, 'products'
+                              FROM products;" []))
+  (must (luna.db.commit tx)))
+
+(each [_ parser (pairs [gorkovchay moychay ozchai suhexuan ipuer artoftea
+                        clubcha chaekshop])]
+  (local products (parser.products))
+  (when (< 0 (# products))
+    ;; replace with with-tx
+    (local tx (must (luna.db.begin db)))
+    (local (ok? err) (pcall store-products tx products))
+    (must (luna.db.commit tx))
+    (when (not ok?)
+     (print (string.format "Error fetching %s:\n%s"
+                          parser.title err)))))
+
+(cache.clear db "page:")
+(populate-search-table db)
diff --git a/bin/serve.fnl b/bin/serve.fnl
index 7385b53..927bd71 100644
--- a/bin/serve.fnl
+++ b/bin/serve.fnl
@@ -36,7 +36,7 @@
         (tset package :loaded module old))))))
 
 (local db
-  (must (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL")))
+  (must (luna.db.open "file:var/db.sqlite?_journal=WAL&_sync=NORMAL&_txlock=immediate")))
 
 (local query-synonyms {
   "шэн" "шен"
-- 
cgit v1.2.3