diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | Cargo.lock | 441 | ||||
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | docker-compose.yaml | 32 | ||||
| -rw-r--r-- | src/bin/update-db.rs | 11 | ||||
| -rw-r--r-- | src/load.rs | 77 | ||||
| -rw-r--r-- | src/types.rs | 40 |
7 files changed, 526 insertions, 79 deletions
@@ -10,3 +10,6 @@ counts # Generated files *.db *.db.zst + +# configuration files +.env @@ -44,6 +44,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] name = "artemis" version = "0.1.0" dependencies = [ @@ -55,6 +61,7 @@ dependencies = [ "include_dir", "indicatif", "itertools", + "meilisearch-sdk", "mime", "mime_guess", "regex", @@ -67,6 +74,17 @@ dependencies = [ ] [[package]] +name = "async-channel" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] name = "async-trait" version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -200,6 +218,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] +name = "castaway" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" + +[[package]] name = "cc" version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -222,7 +246,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time", + "time 0.1.45", "wasm-bindgen", "winapi", ] @@ -238,6 +262,15 @@ dependencies = [ ] [[package]] +name = "concurrent-queue" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] name = "console" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -251,6 +284,15 @@ dependencies = [ ] [[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] name = "core-foundation-sys" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -341,6 +383,37 @@ dependencies = [ ] [[package]] +name = "curl" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.59+curl-7.86.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" +dependencies = [ + "cc", + "libc", + "libnghttp2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + +[[package]] name = "cxx" version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -405,6 +478,9 @@ name = "either" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -413,12 +489,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + +[[package]] name = "event-listener" version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] name = "filetime" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -468,6 +562,21 @@ dependencies = [ ] [[package]] +name = "futures" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] name = "futures-channel" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -506,6 +615,38 @@ dependencies = [ ] [[package]] +name = "futures-io" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "futures-sink" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -523,9 +664,13 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -742,6 +887,42 @@ dependencies = [ ] [[package]] +name = "isahc" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" +dependencies = [ + "async-channel", + "castaway", + "crossbeam-utils", + "curl", + "curl-sys", + "encoding_rs", + "event-listener", + "futures-lite", + "http", + "log", + "mime", + "once_cell", + "polling", + "slab", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + +[[package]] +name = "iso8601-duration" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b51dd97fa24074214b9eb14da518957573f4dec3189112610ae1ccec9ac464" +dependencies = [ + "nom 5.1.2", +] + +[[package]] name = "itertools" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -772,18 +953,53 @@ dependencies = [ ] [[package]] +name = "jsonwebtoken" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" +dependencies = [ + "base64 0.13.1", + "ring", + "serde", + "serde_json", +] + +[[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + +[[package]] name = "libc" version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] +name = "libnghttp2-sys" +version = "0.1.7+1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" +dependencies = [ + "cc", + "libc", +] + +[[package]] name = "libsqlite3-sys" version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -795,6 +1011,18 @@ dependencies = [ ] [[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] name = "link-cplusplus" version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -829,6 +1057,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] +name = "meilisearch-index-setting-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36007bfec8990e7b77d32706a21d9118a437c3800b4ebe3fc04749fcd20dbd70" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "meilisearch-sdk" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534120322f7ac79114569ed86978840794fa86ad2c22fcbdb8adf392efcafd50" +dependencies = [ + "async-trait", + "either", + "futures", + "isahc", + "iso8601-duration", + "js-sys", + "jsonwebtoken", + "log", + "meilisearch-index-setting-macro", + "serde", + "serde_json", + "thiserror", + "time 0.3.17", + "uuid", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yaup", +] + +[[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -879,6 +1145,17 @@ dependencies = [ [[package]] name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check", +] + +[[package]] +name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" @@ -929,6 +1206,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] name = "parking_lot" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1027,6 +1329,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] +name = "polling" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "windows-sys", +] + +[[package]] name = "portable-atomic" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1131,6 +1447,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys", +] + +[[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1234,6 +1559,17 @@ dependencies = [ ] [[package]] +name = "sluice" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" +dependencies = [ + "async-channel", + "futures-core", + "futures-io", +] + +[[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1271,7 +1607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ "itertools", - "nom", + "nom 7.1.3", "unicode_categories", ] @@ -1363,6 +1699,12 @@ dependencies = [ ] [[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] name = "stringprep" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1430,6 +1772,33 @@ dependencies = [ ] [[package]] +name = "time" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "itoa 1.0.5", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + +[[package]] name = "tinyvec" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1553,10 +1922,22 @@ dependencies = [ "cfg-if", "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] [[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "tracing-core" version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1566,6 +1947,16 @@ dependencies = [ ] [[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] name = "try-lock" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1659,6 +2050,15 @@ dependencies = [ ] [[package]] +name = "uuid" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +dependencies = [ + "getrandom", +] + +[[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1671,6 +2071,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] name = "want" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1718,6 +2124,18 @@ dependencies = [ ] [[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] name = "wasm-bindgen-macro" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1776,6 +2194,15 @@ dependencies = [ ] [[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1864,6 +2291,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] +name = "yaup" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59e7d27bed43f7c37c25df5192ea9d435a8092a902e02203359ac9ce3e429d9" +dependencies = [ + "serde", + "url", +] + +[[package]] name = "zip" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -14,6 +14,7 @@ filetime = "0.2.19" include_dir = { version = "0.7.3", features = ["metadata"] } indicatif = "0.17.2" itertools = "0.10.5" +meilisearch-sdk = "0.22.0" mime = "0.3.16" mime_guess = "2.0.4" regex = "1.7.1" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..e009ba0 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,32 @@ +version: "3" + +services: + # postgres: + # image: docker.io/postgis/postgis:14-3.3 + # environment: + # - "POSTGRES_USER=artemis" + # - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" + # - "POSTGRES_DB=artemis" + # - "PGDATA=/var/lib/postgresql/data/pgdata" + # ports: + # - 127.0.0.1:5432:5432 + # volumes: + # - type: volume + # source: postgres + # target: /var/lib/postgresql/data/pgdata + # - ./postgres.conf:/etc/postgresql/postgresql.conf:ro + + meilisearch: + image: docker.io/getmeili/meilisearch:v1.0 + environment: + - "MEILI_HTTP_PAYLOAD_SIZE_LIMIT=10Gb" + ports: + - 127.0.0.1:7700:7700 + volumes: + - type: volume + source: meilisearch + target: /meili_data + +volumes: + # postgres: + meilisearch: diff --git a/src/bin/update-db.rs b/src/bin/update-db.rs index a7f92f4..8e2a6eb 100644 --- a/src/bin/update-db.rs +++ b/src/bin/update-db.rs @@ -112,7 +112,7 @@ fn get_last_updated_header(url: &str) -> Option<DateTime<Utc>> { } } -async fn load_weekly(db: &SqlitePool) -> chrono::DateTime<Utc> { +async fn load_weekly(db: &SqlitePool, meili: &meilisearch_sdk::Client) -> chrono::DateTime<Utc> { let output_file = download_file(WEEKLY_DUMP_URL, None).expect("Error downloading weekly dump file"); // Hardcoding this file name because it might change and I don't want to deal with that @@ -137,7 +137,7 @@ async fn load_weekly(db: &SqlitePool) -> chrono::DateTime<Utc> { artemis::db::delete_indexes(db).await.expect("Error deleting indexes"); - load::load_amateurs(db, true).await; + load::load_amateurs(db, meili, true).await; load::load_comments(db, true).await; load::load_entities(db, true).await; load::load_headers(db, true).await; @@ -163,7 +163,7 @@ async fn load_daily(url: &str, db: &SqlitePool) -> chrono::DateTime<Utc> { unzip_file(&output_file).expect("Error unzipping file"); std::fs::remove_file("counts").expect("Error deleting counts file"); - load::load_amateurs(db, false).await; + // load::load_amateurs(db, false).await; load::load_comments(db, false).await; load::load_entities(db, false).await; load::load_headers(db, false).await; @@ -190,6 +190,7 @@ async fn main() { ) .await .expect("Error connecting to database"); + let meili = meilisearch_sdk::Client::new("http://localhost:7700", env!("MEILISEARCH_MASTER_KEY")); artemis::db::create_db(&db).await.expect("Error creating database"); @@ -203,7 +204,7 @@ async fn main() { if let Some(last_weekly) = last_weekly { if fcc_updates.weekly.is_some() && fcc_updates.weekly.unwrap() > last_weekly.date { println!("New weekly update found, loading weekly dump"); - let update_date = load_weekly(&db).await; + let update_date = load_weekly(&db, &meili).await; meta::insert_update( &db, &Update { @@ -218,7 +219,7 @@ async fn main() { } } else { println!("No weekly updates found, loading weekly dump"); - let update_date = load_weekly(&db).await; + let update_date = load_weekly(&db, &meili).await; meta::insert_update( &db, &Update { diff --git a/src/load.rs b/src/load.rs index cc35c24..d307e3b 100644 --- a/src/load.rs +++ b/src/load.rs @@ -2,11 +2,11 @@ use crate::types::*; use csv::StringRecord; use indicatif::{ProgressBar, ProgressStyle}; use itertools::Itertools; +use regex::Regex; use sqlx::{QueryBuilder, Sqlite, SqlitePool}; +use std::fs; use std::fs::File; use std::io::BufRead; -use regex::Regex; -use std::fs; const INSERT_AMATEUR_SQL: &str = include_str!("sql/insert-amateur.sql"); const INSERT_COMMENT_SQL: &str = include_str!("sql/insert-comment.sql"); @@ -22,7 +22,7 @@ const INSERT_SPECIAL_CONDITION_CODE_SQL: &str = const BIND_LIMIT: usize = 32766; -pub async fn load_amateurs(db: &SqlitePool, clear_first: bool) { +pub async fn load_amateurs(db: &SqlitePool, meili: &meilisearch_sdk::Client, clear_first: bool) { let amateurs_file = File::open("AM.dat"); if amateurs_file.is_err() { println!("AM.dat not found, skipping"); @@ -51,56 +51,29 @@ pub async fn load_amateurs(db: &SqlitePool, clear_first: bool) { ); progress_bar.set_message("AM.dat"); - if clear_first { - QueryBuilder::new("DELETE FROM amateurs") - .build() - .execute(&mut transaction) - .await - .expect("Error deleting amateurs"); - } - - let chunk_size = BIND_LIMIT / 18; - for chunk in &reader.records().chunks(chunk_size) { - let chunk = chunk.collect::<Result<Vec<StringRecord>, _>>().unwrap(); - let chunk = chunk.iter(); - - let mut query_builder: QueryBuilder<Sqlite> = QueryBuilder::new(INSERT_AMATEUR_SQL); - - query_builder.push_values(chunk, |mut builder, entry| { - let amateur: Amateur = entry.deserialize(None).expect("Error deserializing entry"); - builder - .push_bind(amateur.RecordType) - .push_bind(amateur.UniqueSystemIdentifier) - .push_bind(amateur.UlsFileNumber) - .push_bind(amateur.EBFNumber) - .push_bind(amateur.CallSign) - .push_bind(amateur.OperatorClass) - .push_bind(amateur.GroupCode) - .push_bind(amateur.RegionCode) - .push_bind(amateur.TrusteeCallSign) - .push_bind(amateur.TrusteeIndicator) - .push_bind(amateur.PhysicianCertification) - .push_bind(amateur.VESignature) - .push_bind(amateur.SystematicCallSignChange) - .push_bind(amateur.VanityCallSignChange) - .push_bind(amateur.VanityRelationship) - .push_bind(amateur.PreviousCallSign) - .push_bind(amateur.PreviousOperatorClass) - .push_bind(amateur.TrusteeName); - }); - - query_builder - .build() - .execute(&mut transaction) - .await - .expect("Error executing query"); - progress_bar.set_position(progress_bar.position() + chunk_size as u64); - } - - transaction - .commit() + // if clear_first { + // QueryBuilder::new("DELETE FROM amateurs") + // .build() + // .execute(&mut transaction) + // .await + // .expect("Error deleting amateurs"); + // } + + let amatuers: Vec<Amateur> = reader.records().map(|record| { + record.unwrap().deserialize::<Amateur>(None).unwrap() + }).collect(); + + // transaction + // .commit() + // .await + // .expect("Error committing transaction"); + + let meili_task = meili + .index("amateurs") + .add_documents(&amatuers, Some("UniqueSystemIdentifier")) .await - .expect("Error committing transaction"); + .unwrap(); + dbg!(meili_task); std::fs::remove_file("AM.dat").expect("Error deleting AM.dat"); progress_bar.finish(); } diff --git a/src/types.rs b/src/types.rs index 09e5d06..483156d 100644 --- a/src/types.rs +++ b/src/types.rs @@ -3,7 +3,7 @@ use crate::fcc_date; use chrono::NaiveDate; use chrono::{DateTime, Utc}; -use serde::Deserialize; +use serde::{Serialize, Deserialize}; use sqlx::FromRow; #[derive(Debug, Deserialize, FromRow)] @@ -14,26 +14,26 @@ pub struct Update { pub date: DateTime<Utc>, } -#[derive(Debug, Deserialize)] -pub struct Amateur<'a> { - pub RecordType: &'a str, +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Amateur { + pub RecordType: String, pub UniqueSystemIdentifier: u32, - pub UlsFileNumber: &'a str, - pub EBFNumber: &'a str, - pub CallSign: &'a str, - pub OperatorClass: &'a str, - pub GroupCode: &'a str, - pub RegionCode: &'a str, - pub TrusteeCallSign: &'a str, - pub TrusteeIndicator: &'a str, - pub PhysicianCertification: &'a str, - pub VESignature: &'a str, - pub SystematicCallSignChange: &'a str, - pub VanityCallSignChange: &'a str, - pub VanityRelationship: &'a str, - pub PreviousCallSign: &'a str, - pub PreviousOperatorClass: &'a str, - pub TrusteeName: &'a str, + pub UlsFileNumber: String, + pub EBFNumber: String, + pub CallSign: String, + pub OperatorClass: String, + pub GroupCode: String, + pub RegionCode: String, + pub TrusteeCallSign: String, + pub TrusteeIndicator: String, + pub PhysicianCertification: String, + pub VESignature: String, + pub SystematicCallSignChange: String, + pub VanityCallSignChange: String, + pub VanityRelationship: String, + pub PreviousCallSign: String, + pub PreviousOperatorClass: String, + pub TrusteeName: String, } #[derive(Debug, Deserialize)] |