selenium C# wait for background requests(not initiated by c# code)

Question

Im using selenium chrome driver combined with cefsharp, the page im working on sends some kind of ping request(fetch/xhr) everytime an element is clicked the problem is that those request aren't initiated by selenium chrome driver so there is no wayy for me to track them and wait for the finish. Is there any way for selenium or js to track those requests?

Answer 1

With Selenium 4 you actually can intercept requests

Call Enable to start requests tracking

Call WaitForRequestResponse to wait for the specific request

If you can same requests you can clean already catch requests with Clear

    public class WebDriverNetworkController
    {
        private const string RequestWillBeSent = "requestWillBeSent";
        private const string ResponseReceived = "responseReceived";

        private readonly IDriverProvider _driverProvider;
        private readonly ILogger _log;
        private readonly ConcurrentDictionary<DevToolsSession, List<HttpRequestModel>> _sentRequests = new();

        public WebDriverNetworkController(IDriverProvider driverProvider, ILogger log)
        {
            _driverProvider = driverProvider;
            _log = log;
        }

        private void HandleDevToolsEvent(object sender, DevToolsEventReceivedEventArgs e)
        {
            var session = sender as DevToolsSession;
            switch (e.EventName)
            {
                case RequestWillBeSent:
                {
                    OnRequestWillBeSent(session, e.EventData);
                    break;
                }

                case ResponseReceived:
                {
                    OnResponseReceived(session, e.EventData);
                    break;
                }
            }
        }

        public void Enable()
        {
            _log.Information("Enabling network tracing");
            var session = GetDevToolsSession();
            session.DevToolsEventReceived += HandleDevToolsEvent;

            var enableTask = session.Domains.Network.EnableNetwork();
            enableTask.Wait();
        }

        public void Disable()
        {
            _log.Information("Disabling network tracing");
            var session = GetDevToolsSession();
            session.DevToolsEventReceived -= HandleDevToolsEvent;

            var disableTask = session.Domains.Network.DisableNetwork();
            disableTask.Wait();
        }

        public void Clear()
        {
            _log.Information("Clearing captured network tracing requests");
            var sentRequests = GetRequests();
            sentRequests.Clear();
        }

        public HttpRequestModel[] GetAllRequests() => GetRequests().ToArray();

        public HttpRequestModel WaitForRequestResponse(HttpMethod method, string uri)
        {
            _log.Debug($"Waiting for {method.Method} request to {uri}");
            var session = GetDevToolsSession();

            HttpRequestModel request = null;
            var waiter = new ConditionWaiter
            {
                Message = $"{method.Method} request to {uri}"
            };
            waiter.Until(() =>
            {
                request = GetLastSentRequest(method, uri, session);
                return request?.Response != null;
            });

            return request;
        }

        private HttpRequestModel GetLastSentRequest(HttpMethod method, string uri, DevToolsSession session) =>
            GetRequests(session).LastOrDefault(request => request.Method == method && request.Uri.EndsWith(uri));

        private DevToolsSession GetDevToolsSession()
        {
            var driver = _driverProvider.GetDriver();
            if (driver is IDevTools devTools)
            {
                return devTools.GetDevToolsSession();
            }

            throw new WebDriverException($"Could not cast {driver.GetType().Name} to {nameof(IDevTools)}");
        }

        private List<HttpRequestModel> GetRequests(DevToolsSession session = null) =>
            _sentRequests.GetOrAdd(session ?? GetDevToolsSession(), _ => new());

        private void OnRequestWillBeSent(DevToolsSession session, JToken eventData)
        {
            var requestModel = new HttpRequestModel
            {
                Id = eventData.SelectToken("requestId", true)!.Value<string>(),
                Method = new(eventData.SelectToken("request.method", true)!.Value<string>()!),
                Uri = eventData.SelectToken("request.url", true)!.Value<string>(),
                Body = eventData.SelectToken("request")!.Value<string>("postData")
            };
            var sentRequests = _sentRequests.GetOrAdd(session, _ => new());
            sentRequests.Add(requestModel);
        }

        private void OnResponseReceived(DevToolsSession session, JToken eventData)
        {
            var requestId = eventData.SelectToken("requestId", true)!.Value<string>();
            var responseModel = new HttpResponseModel
            {
                Uri = eventData.SelectToken("response.url", true)!.Value<string>(),
                StatusCode = (HttpStatusCode)eventData.SelectToken("response.status", true)!.Value<long>()
            };
            var sentRequests = _sentRequests[session];
            var request = sentRequests.Last(request => request.Id == requestId);
            request.Response = responseModel;
        }
    }

The code above is not ready to be used right away but should give a general understanding of the approach.