随机 Azure Function Apps 失败：超过主机阈值 [连接]

Question

我有以下功能应用程序

[FunctionName("SendEmail")]
public static async Task Run([ServiceBusTrigger("%EmailSendMessageQueueName%", AccessRights.Listen, Connection = AzureFunctions.Connection)] EmailMessageDetails messageToSend,
    [ServiceBus("%EmailUpdateQueueName%", AccessRights.Send, Connection = AzureFunctions.Connection)]IAsyncCollector<EmailMessageUpdate> messageResponse,
    //TraceWriter log,
    ILogger log,
    CancellationToken token)
{
    log.LogInformation($"C# ServiceBus queue trigger function processed message: {messageToSend}");

    /* Validate input and initialise Mandrill */
    try
    {
        if (!ValidateMessage(messageToSend, log))   // TODO: finish validation
        {
            log.LogError("Invalid or Unknown Message Content");
            throw new Exception("Invalid message content.");
        }
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to Validate Message data: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }

    DateTime utcTimeToSend;
    try
    {
        var envTag = GetEnvVariable("Environment");
        messageToSend.Tags.Add(envTag);

        utcTimeToSend = messageToSend.UtcTimeToSend.GetNextUtcSendDateTime();
        DateTime utcExpiryDate = messageToSend.UtcTimeToSend.GetUtcExpiryDate();
        DateTime now = DateTime.UtcNow;
        if (now > utcExpiryDate)
        {
            log.LogError($"Stopping sending message because it is expired: {utcExpiryDate}");
            throw new Exception($"Stopping sending message because it is expired: {utcExpiryDate}");
        }
        if (utcTimeToSend > now)
        {
            log.LogError($"Stopping sending message because it is not allowed to be send due to time constraints: next send time: {utcTimeToSend}");
            throw new Exception($"Stopping sending message because it is not allowed to be send due to time constraints: next send time: {utcTimeToSend}");
        }
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to Parse and/or Validate Message Time To Send: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }

    /* Submit message to Mandrill */
    string errorMessage = null;
    IList<MandrillSendMessageResponse> mandrillResult = null;
    DateTime timeSubmitted = default(DateTime);
    DateTime timeUpdateRecieved = default(DateTime);

    try
    {
        var mandrillApi = new MandrillApi(GetEnvVariable("Mandrill:APIKey"));

        var mandrillMessage = new MandrillMessage
        {
            FromEmail = messageToSend.From,
            FromName = messageToSend.FromName,
            Subject = messageToSend.Subject,
            TrackClicks = messageToSend.Track,
            Tags = messageToSend.Tags,
            TrackOpens = messageToSend.Track,
        };

        mandrillMessage.AddTo(messageToSend.To, messageToSend.ToName);
        foreach (var passthrough in messageToSend.PassThroughVariables)
        {
            mandrillMessage.AddGlobalMergeVars(passthrough.Key, passthrough.Value);
        }

        timeSubmitted = DateTime.UtcNow;
        if (String.IsNullOrEmpty(messageToSend.TemplateId))
        {
            log.LogInformation($"No Message Template");
            mandrillMessage.Text = messageToSend.MessageBody;
            mandrillResult = await mandrillApi.Messages.SendAsync(mandrillMessage, async: true, sendAtUtc: utcTimeToSend);
        }
        else
        {
            log.LogInformation($"Using Message Template: {messageToSend.TemplateId}");
            var clock = new Stopwatch();
            clock.Start();
            mandrillResult = await mandrillApi.Messages.SendTemplateAsync(
                mandrillMessage,
                messageToSend.TemplateId,
                async: true,
                sendAtUtc: utcTimeToSend
            );
            clock.Stop();
            log.LogInformation($"Call to mandrill took {clock.Elapsed}");
        }
        timeUpdateRecieved = DateTime.UtcNow;
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to call Mandrill: {ex.Message} => {ex.ReportAllProperties()}");
        errorMessage = ex.Message;
    }

    try
    {
        MandrillSendMessageResponse theResult = null;
        SendMessageStatus status = SendMessageStatus.FailedToSendToProvider;

        if (mandrillResult == null || mandrillResult.Count < 1)
        {
            if (String.IsNullOrEmpty(errorMessage))
            {
                errorMessage = "Invalid Mandrill result.";
            }
        }
        else
        {
            theResult = mandrillResult[0];
            status = FacMandrillUtils.ConvertToSendMessageStatus(theResult.Status);
        }

        var response = new EmailMessageUpdate
        {
            SentEmailInfoId = messageToSend.SentEmailInfoId,
            ExternalProviderId = theResult?.Id ?? String.Empty,
            Track = messageToSend.Track,
            FacDateSentToProvider = timeSubmitted,
            FacDateUpdateRecieved = timeUpdateRecieved,
            FacErrorMessage = errorMessage,
            Status = status,
            StatusDetail = theResult?.RejectReason ?? "Error"
        };

        await messageResponse.AddAsync(response, token).ConfigureAwait(false);
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to push message to the update ({AzureFunctions.EmailUpdateQueueName}) queue: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }
}

当我排队 100 条消息时，一切正常。当我排队 500 多条消息时，发送了其中的 499 条，但最后一条 none 从未发送过。我也开始收到以下错误。

The operation was canceled.

我设置并配置了 Application Insights，并且记录了日志运行。我无法在本地重现，基于以下来自 Application Insights 的端到端事务详细信息，我相信问题正在此时发生：

await messageResponse.AddAsync(response, token).ConfigureAwait(false);

Application Insights 端到端事务

host.json

{
  "logger": {
    "categoryFilter": {
      "defaultLevel": "Information",
      "categoryLevels": {
        "Host": "Warning",
        "Function": "Information",
        "Host.Aggregator": "Information"
      }
    }
  },
  "applicationInsights": {
    "sampling": {
      "isEnabled": true,
      "maxTelemetryItemsPerSecond": 5
    }
  },
  "serviceBus": {
    "maxConcurrentCalls": 32
  }
}

可能相关的还有 Application Insights 中的这个错误。

[

有其他人遇到过这个或类似的问题吗？

Answer 1

如果您从异常 https://aka.ms/functions-thresholds 中遵循 link，您将看到以下限制：

Connections : Number of outbound connections (limit is 300). For information on handling connection limits, see Managing Connections.

你很可能已经击中了那个。

在每个函数调用中，您都会创建一个 MandrillApi 的新实例。您没有提到您正在使用哪个库，但我怀疑它正在为 MandrillApi.

的每个实例创建一个新连接

我为每个实例检查了Mandrill Dot Net and yes, it's creating一个新的HttpClient：

_httpClient = new HttpClient
{
    BaseAddress = new Uri(BaseUrl)
};

Managing Connections推荐：

In many cases, this connection limit can be avoided by re-using client instances rather than creating new ones in each function. .NET clients like the HttpClient, DocumentClient, and Azure storage clients can manage connections if you use a single, static client. If those clients are re-instantiated with every function invocation, there is a high probability that the code is leaking connections.

如果 API 客户端是 thread-safe，请检查该库的文档，如果是，则在函数调用之间重用它。

随机 Azure Function Apps 失败：超过主机阈值 [连接]

Random Azure Function Apps failures: Host thresholds exceeded [Connections]

c#

azure

mandrill

azure-functions