[evolution-ews/gnome-41] I#174 - Silently retry on I/O errors (like 'Connection terminated unexpectedly')



commit d54b8f319f83e49d8b1729347c1f6bb70d4f191e
Author: Milan Crha <mcrha redhat com>
Date:   Fri Jan 7 10:25:34 2022 +0100

    I#174 - Silently retry on I/O errors (like 'Connection terminated unexpectedly')
    
    Closes https://gitlab.gnome.org/GNOME/evolution-ews/-/issues/174

 src/EWS/common/e-ews-connection-utils.c     |  4 ++--
 src/EWS/common/e-ews-connection.c           | 24 +++++++++++++++++++-----
 src/EWS/common/e-ews-notification.c         |  8 ++++----
 src/EWS/common/e-soap-response.c            |  2 +-
 src/Microsoft365/common/e-m365-connection.c | 23 +++++++++++++++++------
 5 files changed, 43 insertions(+), 18 deletions(-)
---
diff --git a/src/EWS/common/e-ews-connection-utils.c b/src/EWS/common/e-ews-connection-utils.c
index a9616d59..ba548774 100644
--- a/src/EWS/common/e-ews-connection-utils.c
+++ b/src/EWS/common/e-ews-connection-utils.c
@@ -463,7 +463,7 @@ e_ews_connection_utils_authenticate (EEwsConnection *cnc,
                ews_connection_utils_setup_bearer_auth (cnc, session, msg, TRUE, E_SOUP_AUTH_BEARER (auth), 
NULL, &local_error);
 
                if (local_error)
-                       soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+                       soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
 
                g_object_unref (using_bearer_auth);
                g_clear_error (&local_error);
@@ -477,7 +477,7 @@ e_ews_connection_utils_authenticate (EEwsConnection *cnc,
                e_ews_connection_utils_expired_password_to_error (service_url, &local_error);
 
                if (local_error)
-                       soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+                       soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
 
                g_clear_error (&local_error);
                g_free (service_url);
diff --git a/src/EWS/common/e-ews-connection.c b/src/EWS/common/e-ews-connection.c
index 916aab15..ea0ccca9 100644
--- a/src/EWS/common/e-ews-connection.c
+++ b/src/EWS/common/e-ews-connection.c
@@ -30,6 +30,8 @@
 
 #define d(x) x
 
+#define EWS_RETRY_IO_ERROR_SECONDS 3
+
 /* A chunk size limit when moving items in chunks. */
 #define EWS_MOVE_ITEMS_CHUNK_SIZE 500
 
@@ -153,6 +155,8 @@ struct _EwsNode {
 
        GCancellable *cancellable;
        gulong cancel_handler_id;
+
+       gboolean retrying_after_network_error;
 };
 
 struct _EwsUrls {
@@ -922,7 +926,7 @@ ews_response_cb (SoupSession *session,
                  gpointer data)
 {
        EwsNode *enode = (EwsNode *) data;
-       ESoapResponse *response;
+       ESoapResponse *response = NULL;
        ESoapParameter *param;
        const gchar *persistent_auth;
        gint log_level;
@@ -978,6 +982,11 @@ ews_response_cb (SoupSession *session,
                        EWS_CONNECTION_ERROR_AUTHENTICATION_FAILED,
                        _("Authentication failed"));
                goto exit;
+       } else if (!enode->retrying_after_network_error &&
+                  msg->status_code == SOUP_STATUS_IO_ERROR) {
+               wait_ms = EWS_RETRY_IO_ERROR_SECONDS * 1000;
+               enode->retrying_after_network_error = TRUE;
+               goto retrylbl;
        } else if (msg->status_code == SOUP_STATUS_CANT_RESOLVE ||
                   msg->status_code == SOUP_STATUS_CANT_RESOLVE_PROXY ||
                   msg->status_code == SOUP_STATUS_CANT_CONNECT ||
@@ -1015,7 +1024,10 @@ ews_response_cb (SoupSession *session,
                e_soap_response_dump_response (response, stdout);
        }
 
-       param = e_soap_response_get_first_parameter_by_name (response, "detail", NULL);
+       if (!wait_ms && e_ews_connection_get_backoff_enabled (enode->cnc))
+               param = e_soap_response_get_first_parameter_by_name (response, "detail", NULL);
+       else
+               param = NULL;
        if (param)
                param = e_soap_parameter_get_first_child_by_name (param, "ResponseCode");
        if (param) {
@@ -1042,7 +1054,8 @@ ews_response_cb (SoupSession *session,
                g_free (value);
        }
 
-       if (wait_ms > 0 && e_ews_connection_get_backoff_enabled (enode->cnc)) {
+ retrylbl:
+       if (wait_ms > 0) {
                GCancellable *cancellable = enode->cancellable;
                EFlag *flag;
 
@@ -1086,7 +1099,7 @@ ews_response_cb (SoupSession *session,
 
                e_flag_free (flag);
 
-               g_object_unref (response);
+               g_clear_object (&response);
 
                if (g_cancellable_is_cancelled (cancellable) ||
                    msg->status_code == SOUP_STATUS_CANCELLED) {
@@ -1101,6 +1114,7 @@ ews_response_cb (SoupSession *session,
                        new_node->cb = enode->cb;
                        new_node->cnc = enode->cnc;
                        new_node->simple = enode->simple;
+                       new_node->retrying_after_network_error = enode->retrying_after_network_error;
 
                        enode->simple = NULL;
 
@@ -1123,7 +1137,7 @@ ews_response_cb (SoupSession *session,
        if (enode->cb != NULL)
                enode->cb (response, enode->simple);
 
-       g_object_unref (response);
+       g_clear_object (&response);
 
 exit:
        if (enode->simple)
diff --git a/src/EWS/common/e-ews-notification.c b/src/EWS/common/e-ews-notification.c
index 28325298..e0ca7fc0 100644
--- a/src/EWS/common/e-ews-notification.c
+++ b/src/EWS/common/e-ews-notification.c
@@ -877,16 +877,16 @@ e_ews_notification_get_events_sync (EEwsNotification *notification,
        if (e_ews_debug_get_log_level () <= 3)
                soup_message_body_set_accumulate (SOUP_MESSAGE (msg)->response_body, FALSE);
 
-       handler_id = g_signal_connect (
-               SOUP_MESSAGE (msg), "got-chunk",
-               G_CALLBACK (ews_notification_soup_got_chunk), notification);
-
        if (!e_ews_connection_utils_prepare_message (cnc, notification->priv->soup_session, SOUP_MESSAGE 
(msg), notification->priv->cancellable)) {
                g_object_unref (msg);
                g_object_unref (cnc);
                return FALSE;
        }
 
+       handler_id = g_signal_connect (
+               SOUP_MESSAGE (msg), "got-chunk",
+               G_CALLBACK (ews_notification_soup_got_chunk), notification);
+
        /* Unref it early, thus it doesn't keep the connection alive after all backends are freed */
        g_object_unref (cnc);
 
diff --git a/src/EWS/common/e-soap-response.c b/src/EWS/common/e-soap-response.c
index 45de3069..e503a142 100644
--- a/src/EWS/common/e-soap-response.c
+++ b/src/EWS/common/e-soap-response.c
@@ -562,7 +562,7 @@ e_soap_response_get_first_parameter_by_name (ESoapResponse *response,
 
                        g_set_error (
                                error,
-                               SOUP_HTTP_ERROR, SOUP_STATUS_IO_ERROR,
+                               SOUP_HTTP_ERROR, SOUP_STATUS_MALFORMED,
                                "%s", (string != NULL) ? string :
                                "<faultstring> in SOAP response");
 
diff --git a/src/Microsoft365/common/e-m365-connection.c b/src/Microsoft365/common/e-m365-connection.c
index e81d06ed..b32ca8fa 100644
--- a/src/Microsoft365/common/e-m365-connection.c
+++ b/src/Microsoft365/common/e-m365-connection.c
@@ -19,6 +19,7 @@
 #define LOCK(x) g_rec_mutex_lock (&(x->priv->property_lock))
 #define UNLOCK(x) g_rec_mutex_unlock (&(x->priv->property_lock))
 
+#define M365_RETRY_IO_ERROR_SECONDS 3
 #define X_EVO_M365_DATA "X-EVO-M365-DATA"
 
 typedef enum _CSMFlags {
@@ -274,7 +275,7 @@ m365_connection_authenticate (SoupSession *session,
        m365_connection_utils_setup_bearer_auth (cnc, session, msg, TRUE, E_SOUP_AUTH_BEARER (auth), NULL, 
&local_error);
 
        if (local_error)
-               soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+               soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
 
        g_object_unref (using_bearer_auth);
        g_clear_error (&local_error);
@@ -1139,6 +1140,7 @@ m365_connection_send_request_sync (EM365Connection *cnc,
        SoupSession *soup_session;
        gint need_retry_seconds = 5;
        gboolean success = FALSE, need_retry = TRUE;
+       gboolean did_io_error_retry = FALSE;
 
        g_return_val_if_fail (E_IS_M365_CONNECTION (cnc), FALSE);
        g_return_val_if_fail (SOUP_IS_MESSAGE (message), FALSE);
@@ -1237,10 +1239,12 @@ m365_connection_send_request_sync (EM365Connection *cnc,
                        if (success && m365_log_enabled ())
                                input_stream = e_soup_logger_attach (message, input_stream);
 
-                       /* Throttling - https://docs.microsoft.com/en-us/graph/throttling  */
-                       if (message->status_code == 429 ||
+                       if ((!did_io_error_retry && message->status_code == SOUP_STATUS_IO_ERROR) ||
+                           /* Throttling - https://docs.microsoft.com/en-us/graph/throttling  */
+                           message->status_code == 429 ||
                            /* 
https://docs.microsoft.com/en-us/graph/best-practices-concept#handling-expected-errors */
                            message->status_code == SOUP_STATUS_SERVICE_UNAVAILABLE) {
+                               did_io_error_retry = did_io_error_retry || message->status_code == 
SOUP_STATUS_IO_ERROR;
                                need_retry = TRUE;
                        } else if (message->status_code == SOUP_STATUS_SSL_FAILED) {
                                m365_connection_extract_ssl_data (cnc, message);
@@ -1254,6 +1258,8 @@ m365_connection_send_request_sync (EM365Connection *cnc,
 
                                if (retry_after_str && *retry_after_str)
                                        retry_after = g_ascii_strtoll (retry_after_str, NULL, 10);
+                               else if (message->status_code == SOUP_STATUS_IO_ERROR)
+                                       retry_after = M365_RETRY_IO_ERROR_SECONDS;
                                else
                                        retry_after = 0;
 
@@ -1968,7 +1974,7 @@ e_m365_connection_batch_request_internal_sync (EM365Connection *cnc,
                if (!submessage)
                        continue;
 
-               submessage->status_code = SOUP_STATUS_IO_ERROR;
+               submessage->status_code = SOUP_STATUS_MALFORMED;
 
                suri = soup_message_get_uri (submessage);
                uri = suri ? soup_uri_to_string (suri, TRUE) : NULL;
@@ -2086,6 +2092,7 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
        GPtrArray *use_requests;
        gint need_retry_seconds = 5;
        gboolean success, need_retry = TRUE;
+       gboolean did_io_error_retry = FALSE;
 
        g_return_val_if_fail (E_IS_M365_CONNECTION (cnc), FALSE);
        g_return_val_if_fail (requests != NULL, FALSE);
@@ -2110,13 +2117,15 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
                                if (!message)
                                        continue;
 
-                               /* Throttling - https://docs.microsoft.com/en-us/graph/throttling  */
-                               if (message->status_code == 429 ||
+                               if ((!did_io_error_retry && message->status_code == SOUP_STATUS_IO_ERROR) ||
+                                   /* Throttling - https://docs.microsoft.com/en-us/graph/throttling  */
+                                   message->status_code == 429 ||
                                    /* 
https://docs.microsoft.com/en-us/graph/best-practices-concept#handling-expected-errors */
                                    message->status_code == SOUP_STATUS_SERVICE_UNAVAILABLE) {
                                        const gchar *retry_after_str;
                                        gint64 retry_after;
 
+                                       did_io_error_retry = did_io_error_retry || message->status_code == 
SOUP_STATUS_IO_ERROR;
                                        need_retry = TRUE;
 
                                        if (!new_requests)
@@ -2128,6 +2137,8 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
 
                                        if (retry_after_str && *retry_after_str)
                                                retry_after = g_ascii_strtoll (retry_after_str, NULL, 10);
+                                       else if (message->status_code == SOUP_STATUS_IO_ERROR)
+                                               retry_after = M365_RETRY_IO_ERROR_SECONDS;
                                        else
                                                retry_after = 0;
 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]