fix(web-scraping): make Retrack webhook payload size configurable and bump default from 2MB to 10MB

Author: azasypkin

src/config.rs

@@ -400,6 +400,7 @@ mod tests {
                     query: None,
                     fragment: None,
                 },
+                max_webhook_body_size: 10485760,
             },
             platform: PlatformConfig {
                 max_import_file_size: 10485760,

src/config/raw_config.rs

@@ -236,6 +236,7 @@ mod tests {
 
         [retrack]
         host = 'http://localhost:7676/'
+        max_webhook_body_size = 10485760
 
         [platform]
         max_import_file_size = 10485760
@@ -672,6 +673,7 @@ mod tests {
                     query: None,
                     fragment: None,
                 },
+                max_webhook_body_size: 10485760,
             },
             platform: PlatformConfig {
                 max_import_file_size: 10485760,

src/config/retrack_config.rs

@@ -6,13 +6,21 @@ use url::Url;
 pub struct RetrackConfig {
     /// The URL to access the Retrack service.
     pub host: Url,
+    /// Maximum allowed JSON body size (in bytes) for the Retrack webhook route.
+    #[serde(default = "default_max_webhook_body_size")]
+    pub max_webhook_body_size: usize,
+}
+
+fn default_max_webhook_body_size() -> usize {
+    10 * 1024 * 1024
 }
 
 impl Default for RetrackConfig {
     fn default() -> Self {
         Self {
             host: Url::parse("http://localhost:7676")
                 .expect("Cannot parse Retrack host parameter."),
+            max_webhook_body_size: default_max_webhook_body_size(),
         }
     }
 }
@@ -43,6 +51,7 @@ mod tests {
                 query: None,
                 fragment: None,
             },
+            max_webhook_body_size: 10485760,
         }
         "###);
     }
@@ -59,6 +68,25 @@ mod tests {
             config,
             RetrackConfig {
                 host: url::Url::parse("http://localhost:8686").unwrap(),
+                max_webhook_body_size: 10 * 1024 * 1024,
+            }
+        );
+    }
+
+    #[test]
+    fn deserialization_with_custom_body_size() {
+        let config: RetrackConfig = toml::from_str(
+            r#"
+        host = 'http://localhost:8686/'
+        max_webhook_body_size = 52428800
+    "#,
+        )
+        .unwrap();
+        assert_eq!(
+            config,
+            RetrackConfig {
+                host: url::Url::parse("http://localhost:8686").unwrap(),
+                max_webhook_body_size: 50 * 1024 * 1024,
             }
         );
     }

src/server.rs

@@ -109,6 +109,7 @@ pub async fn run(config: Config, http_port: u16) -> Result<(), anyhow::Error> {
     JsRuntime::init_platform();
 
     let max_responder_body_size = config.utils.max_responder_body_size;
+    let max_retrack_webhook_body_size = config.retrack.max_webhook_body_size;
     let max_import_file_size = config.platform.max_import_file_size;
     let session_cookie_name = config.security.session_cookie_name.clone();
     let state = web::Data::new(AppState::new(config, api.clone()));
@@ -249,7 +250,14 @@ pub async fn run(config: Config, http_port: u16) -> Result<(), anyhow::Error> {
                 web::scope("/api")
                     .service(
                         web::scope("/webhooks")
-                            .route("/retrack", web::post().to(handlers::webhooks_retrack))
+                            .service(
+                                web::scope("/retrack")
+                                    .app_data(
+                                        web::JsonConfig::default()
+                                            .limit(max_retrack_webhook_body_size),
+                                    )
+                                    .route("", web::post().to(handlers::webhooks_retrack)),
+                            )
                             .service(
                                 web::scope("")
                                     .app_data(