Skip to content

Commit f0224df

Browse files
aecsocketaecsocket
authored
Search backend refactor with typesense impl (modrinth#5528)
* initial elasticsearch impl * working elastic cluster * replace SearchError with ApiError for preparation of search backend * start factoring meili out to trait * move meili to backend * update routes to use search backend trait * wip * Update projects.rs * search backend is only init'd once in config * wip * wip: backend agnostic * change search internal routes to delegate to backend * initial elasticsearch impl * fix filtering * elastic impl * refactor indexing into its own module * clean up elastic code * fix ci * fix tests * fix elastic health check * fix up env rebase * fix compile * dummy commit to update github pr * Fix rebase * Elastic basic https auth * Fix duplicate projects showing up * Fix up tests * Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors * clean up agents files * make index chunk size configurable * make `match_phrase` in elastic case-insensitive * use current/next indices and swap between them * test case for error body * Fix failing case * da merge * factor out common stuff from search backends * allow fetching hit metadata from search results * allow customising elasticsearch search config * bit of docs * add mappings to indices for elastic * Implement Typesense * wip * fix up some sort fields stuff * use different approach to filterable field sets * remove a bunch of search fields which weren't used for filtering * bucket text matches * Bucketing by text_match for typesense * fix tombi lint * fix some sentry errors and dont prioritise 2+ term matches * tweak ts query settings * expose some more search settings * query sort changes * small fixes * should fix pagination stuff * fix healthcheck maybe * ragebait ci * tests * tests * revert environment
1 parent 1c1683a commit f0224df

36 files changed

Lines changed: 3840 additions & 754 deletions

‎Cargo.lock‎

Lines changed: 24 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ dotenv-build = "0.1.1"
7272
dotenvy = "0.15.7"
7373
dunce = "1.0.5"
7474
either = "1.15.0"
75+
elasticsearch = "9.1.0-alpha.1"
7576
encoding_rs = "0.8.35"
7677
enumset = "1.1.10"
7778
eyre = "0.6.12"

‎apps/labrinth/.env.docker-compose‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,18 @@ DATABASE_URL=postgresql://labrinth:labrinth@labrinth-postgres/labrinth
1616
DATABASE_MIN_CONNECTIONS=0
1717
DATABASE_MAX_CONNECTIONS=16
1818

19+
SEARCH_BACKEND=typesense
1920
MEILISEARCH_READ_ADDR=http://localhost:7700
2021
MEILISEARCH_WRITE_ADDRS=http://localhost:7700
2122
MEILISEARCH_KEY=modrinth
23+
ELASTICSEARCH_URL=http://localhost:9200
24+
ELASTICSEARCH_INDEX_PREFIX=labrinth
25+
ELASTICSEARCH_USERNAME=elastic
26+
ELASTICSEARCH_PASSWORD=elastic
27+
SEARCH_INDEX_CHUNK_SIZE=5000
28+
TYPESENSE_URL=http://localhost:8108
29+
TYPESENSE_API_KEY=modrinth
30+
TYPESENSE_INDEX_PREFIX=labrinth
2231

2332
REDIS_URL=redis://labrinth-redis
2433
REDIS_MIN_CONNECTIONS=0

‎apps/labrinth/.env.local‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,36 @@ DATABASE_URL=postgresql://labrinth:labrinth@localhost/labrinth
1616
DATABASE_MIN_CONNECTIONS=0
1717
DATABASE_MAX_CONNECTIONS=16
1818

19+
SEARCH_BACKEND=meilisearch
20+
21+
# Meilisearch configuration
1922
MEILISEARCH_READ_ADDR=http://localhost:7700
2023
MEILISEARCH_WRITE_ADDRS=http://localhost:7700
2124
# 5 minutes in milliseconds
2225
SEARCH_OPERATION_TIMEOUT=300000
2326

27+
ELASTICSEARCH_URL=http://localhost:9200
28+
ELASTICSEARCH_INDEX_PREFIX=labrinth
29+
2430
# # For a sharded Meilisearch setup (sharded-meilisearch docker compose profile)
2531
# MEILISEARCH_READ_ADDR=http://localhost:7710
2632
# MEILISEARCH_WRITE_ADDRS=http://localhost:7700,http://localhost:7701
2733

34+
SEARCH_BACKEND=typesense
35+
2836
MEILISEARCH_KEY=modrinth
37+
MEILISEARCH_META_NAMESPACE=
38+
39+
# Elasticsearch configuration
40+
ELASTICSEARCH_URL=http://localhost:9200
41+
ELASTICSEARCH_INDEX_PREFIX=labrinth
42+
ELASTICSEARCH_USERNAME=
43+
ELASTICSEARCH_PASSWORD=
44+
45+
SEARCH_INDEX_CHUNK_SIZE=5000
46+
TYPESENSE_URL=http://localhost:8108
47+
TYPESENSE_API_KEY=modrinth
48+
TYPESENSE_INDEX_PREFIX=labrinth
2949

3050
REDIS_URL=redis://localhost
3151
REDIS_MIN_CONNECTIONS=0

‎apps/labrinth/Cargo.toml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ async-stripe = { workspace = true, features = [
2626
"billing",
2727
"checkout",
2828
"connect",
29-
"webhook-events",
29+
"webhook-events"
3030
] }
3131
async-trait = { workspace = true }
3232
base64 = { workspace = true }
@@ -44,6 +44,7 @@ deadpool-redis.workspace = true
4444
derive_more = { workspace = true, features = ["deref", "deref_mut"] }
4545
dotenvy = { workspace = true }
4646
either = { workspace = true }
47+
elasticsearch = { workspace = true, features = ["experimental-apis"] }
4748
eyre = { workspace = true }
4849
futures = { workspace = true }
4950
futures-util = { workspace = true }
@@ -86,11 +87,11 @@ reqwest = { workspace = true, features = [
8687
"http2",
8788
"json",
8889
"multipart",
89-
"rustls-tls-webpki-roots",
90+
"rustls-tls-webpki-roots"
9091
] }
9192
rust_decimal = { workspace = true, features = [
9293
"serde-with-float",
93-
"serde-with-str",
94+
"serde-with-str"
9495
] }
9596
rust_iso3166 = { workspace = true }
9697
rust-s3 = { workspace = true }
@@ -114,7 +115,7 @@ sqlx = { workspace = true, features = [
114115
"tls-rustls-aws-lc-rs",
115116
] }
116117
sqlx-tracing = { workspace = true, features = ["postgres"] }
117-
strum = { workspace = true }
118+
strum = { workspace = true, features = ["derive"] }
118119
thiserror = { workspace = true }
119120
tokio = { workspace = true, features = ["rt-multi-thread", "sync"] }
120121
tokio-stream = { workspace = true }

‎apps/labrinth/src/auth/mod.rs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ use thiserror::Error;
2020
pub enum AuthenticationError {
2121
#[error(transparent)]
2222
Internal(#[from] eyre::Report),
23-
#[error("Environment Error")]
24-
Env(#[from] dotenvy::Error),
2523
#[error("An unknown database error occurred: {0}")]
2624
Sqlx(#[from] sqlx::Error),
2725
#[error("Database Error: {0}")]
@@ -58,7 +56,6 @@ impl actix_web::ResponseError for AuthenticationError {
5856
AuthenticationError::Internal(..) => {
5957
StatusCode::INTERNAL_SERVER_ERROR
6058
}
61-
AuthenticationError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
6259
AuthenticationError::Sqlx(..) => StatusCode::INTERNAL_SERVER_ERROR,
6360
AuthenticationError::Database(..) => {
6461
StatusCode::INTERNAL_SERVER_ERROR
@@ -94,7 +91,6 @@ impl AuthenticationError {
9491
pub fn error_name(&self) -> &'static str {
9592
match self {
9693
AuthenticationError::Internal(..) => "internal_error",
97-
AuthenticationError::Env(..) => "environment_error",
9894
AuthenticationError::Sqlx(..) => "database_error",
9995
AuthenticationError::Database(..) => "database_error",
10096
AuthenticationError::SerDe(..) => "invalid_input",

‎apps/labrinth/src/background_task.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::database;
12
use crate::database::PgPool;
23
use crate::database::redis::RedisPool;
34
use crate::queue::analytics::cache::cache_analytics;
@@ -8,9 +9,9 @@ use crate::queue::payouts::{
89
insert_bank_balances_and_webhook, process_affiliate_payouts,
910
process_payout, remove_payouts_for_refunded_charges,
1011
};
11-
use crate::search::indexing::index_projects;
12+
use crate::search::SearchBackend;
1213
use crate::util::anrok;
13-
use crate::{database, search};
14+
use actix_web::web;
1415
use clap::ValueEnum;
1516
use eyre::WrapErr;
1617
use tracing::info;
@@ -42,7 +43,7 @@ impl BackgroundTask {
4243
pool: PgPool,
4344
ro_pool: PgPool,
4445
redis_pool: RedisPool,
45-
search_config: search::SearchConfig,
46+
search_backend: web::Data<dyn SearchBackend>,
4647
clickhouse: clickhouse::Client,
4748
stripe_client: stripe::Client,
4849
anrok_client: anrok::Client,
@@ -53,7 +54,7 @@ impl BackgroundTask {
5354
match self {
5455
Migrations => run_migrations().await,
5556
IndexSearch => {
56-
index_search(ro_pool, redis_pool, search_config).await
57+
index_search(ro_pool, redis_pool, search_backend).await
5758
}
5859
ReleaseScheduled => release_scheduled(pool).await,
5960
UpdateVersions => update_versions(pool, redis_pool).await,
@@ -132,10 +133,10 @@ pub async fn run_migrations() -> eyre::Result<()> {
132133
pub async fn index_search(
133134
ro_pool: PgPool,
134135
redis_pool: RedisPool,
135-
search_config: search::SearchConfig,
136+
search_backend: web::Data<dyn SearchBackend>,
136137
) -> eyre::Result<()> {
137138
info!("Indexing local database");
138-
index_projects(ro_pool, redis_pool, &search_config).await
139+
search_backend.index_projects(ro_pool, redis_pool).await
139140
}
140141

141142
pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {

‎apps/labrinth/src/env.rs‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ where
8282
}
8383

8484
pub fn init() -> eyre::Result<()> {
85+
dotenvy::dotenv().ok();
8586
EnvVars::from_env()?;
8687
LazyLock::force(&ENV);
8788
Ok(())
@@ -128,9 +129,6 @@ vars! {
128129
LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
129130
RATE_LIMIT_IGNORE_KEY: String;
130131
DATABASE_URL: String;
131-
MEILISEARCH_READ_ADDR: String;
132-
MEILISEARCH_WRITE_ADDRS: StringCsv;
133-
MEILISEARCH_KEY: String;
134132
REDIS_URL: String;
135133
BIND_ADDR: String;
136134
SELF_ADDR: String;
@@ -142,6 +140,20 @@ vars! {
142140
ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
143141
ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
144142

143+
// search
144+
SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
145+
MEILISEARCH_READ_ADDR: String;
146+
MEILISEARCH_WRITE_ADDRS: StringCsv;
147+
MEILISEARCH_KEY: String;
148+
ELASTICSEARCH_URL: String;
149+
ELASTICSEARCH_INDEX_PREFIX: String;
150+
ELASTICSEARCH_USERNAME: String = "";
151+
ELASTICSEARCH_PASSWORD: String = "";
152+
SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
153+
TYPESENSE_URL: String = "http://localhost:8108";
154+
TYPESENSE_API_KEY: String = "modrinth";
155+
TYPESENSE_INDEX_PREFIX: String = "labrinth";
156+
145157
// storage
146158
STORAGE_BACKEND: crate::file_hosting::FileHostKind;
147159

‎apps/labrinth/src/lib.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ pub struct LabrinthConfig {
5858
pub file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
5959
pub scheduler: Arc<scheduler::Scheduler>,
6060
pub ip_salt: Pepper,
61-
pub search_config: search::SearchConfig,
61+
pub search_backend: web::Data<dyn search::SearchBackend>,
6262
pub session_queue: web::Data<AuthQueue>,
6363
pub payouts_queue: web::Data<PayoutsQueue>,
6464
pub analytics_queue: Arc<AnalyticsQueue>,
@@ -78,7 +78,7 @@ pub fn app_setup(
7878
pool: PgPool,
7979
ro_pool: ReadOnlyPgPool,
8080
redis_pool: RedisPool,
81-
search_config: search::SearchConfig,
81+
search_backend: actix_web::web::Data<dyn search::SearchBackend>,
8282
clickhouse: &mut Client,
8383
file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
8484
stripe_client: stripe::Client,
@@ -129,21 +129,21 @@ pub fn app_setup(
129129
let local_index_interval =
130130
Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
131131
let pool_ref = pool.clone();
132-
let search_config_ref = search_config.clone();
133132
let redis_pool_ref = redis_pool.clone();
133+
let search_backend_ref = search_backend.clone();
134134
scheduler.run(local_index_interval, move || {
135135
let pool_ref = pool_ref.clone();
136136
let redis_pool_ref = redis_pool_ref.clone();
137-
let search_config_ref = search_config_ref.clone();
137+
let search_backend = search_backend_ref.clone();
138138
async move {
139-
if let Err(e) = background_task::index_search(
139+
if let Err(err) = background_task::index_search(
140140
pool_ref,
141141
redis_pool_ref,
142-
search_config_ref,
142+
search_backend,
143143
)
144144
.await
145145
{
146-
warn!("Local project indexing failed: {e:#}");
146+
warn!("Failed to index search: {err:?}");
147147
}
148148
}
149149
});
@@ -299,7 +299,7 @@ pub fn app_setup(
299299
file_host,
300300
scheduler: Arc::new(scheduler),
301301
ip_salt,
302-
search_config,
302+
search_backend,
303303
session_queue,
304304
payouts_queue: web::Data::new(PayoutsQueue::new()),
305305
analytics_queue,
@@ -338,7 +338,7 @@ pub fn app_config(
338338
.app_data(web::Data::new(labrinth_config.pool.clone()))
339339
.app_data(web::Data::new(labrinth_config.ro_pool.clone()))
340340
.app_data(web::Data::new(labrinth_config.file_host.clone()))
341-
.app_data(web::Data::new(labrinth_config.search_config.clone()))
341+
.app_data(labrinth_config.search_backend.clone())
342342
.app_data(web::Data::new(labrinth_config.gotenberg_client.clone()))
343343
.app_data(labrinth_config.http_client.clone())
344344
.app_data(labrinth_config.session_queue.clone())

‎apps/labrinth/src/main.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ struct Args {
5656

5757
fn main() -> std::io::Result<()> {
5858
color_eyre::install().expect("failed to install `color-eyre`");
59-
dotenvy::dotenv().ok();
6059
modrinth_util::log::init().expect("failed to initialize logging");
6160
env::init().expect("failed to initialize environment variables");
6261

@@ -152,7 +151,8 @@ async fn app() -> std::io::Result<()> {
152151
info!("Initializing clickhouse connection");
153152
let mut clickhouse = clickhouse::init_client().await.unwrap();
154153

155-
let search_config = search::SearchConfig::new(None);
154+
let search_backend =
155+
actix_web::web::Data::from(Arc::from(search::backend(None)));
156156

157157
let stripe_client = stripe::Client::new(ENV.STRIPE_API_KEY.clone());
158158

@@ -171,7 +171,7 @@ async fn app() -> std::io::Result<()> {
171171
pool,
172172
ro_pool.into_inner(),
173173
redis_pool,
174-
search_config,
174+
search_backend,
175175
clickhouse,
176176
stripe_client,
177177
anrok_client.clone(),
@@ -207,7 +207,7 @@ async fn app() -> std::io::Result<()> {
207207
pool.clone(),
208208
ro_pool.clone(),
209209
redis_pool.clone(),
210-
search_config.clone(),
210+
search_backend.clone(),
211211
&mut clickhouse,
212212
file_host.clone(),
213213
stripe_client,

0 commit comments

Comments
 (0)