Playwright not able to intercept all requests

Question

const { webkit } = require('playwright');

(async () => {
  const browser = await webkit.launch({ headless: false });
  const context = await browser.newContext();
  const page = await context.newPage();

  // Log and continue all network requests
  page.route('**', route => {
    console.log(route.request());
    route.continue();
  });

  await page.goto('https://www.sahibinden.com');
  //await browser.close();
})();

When I open it in incognito window and intercept using fiddler I am able to get these headers:

X-Client-SrcPort: 49784
Accept-Language: en-US,en;q=0.9
X-J3PopQvX-a: eRELp_ivj7vKEvBmbMRSG-kfU55GIHd5Wc4a06qtEa8-W4hUa90nd0GhKDAoOzTOE6nbdU270qt3pEYVVmFKgVTToiLUqAr9yF684URRjFEdbaPsZXZSQYAJBYyvVsSi_K44SKCEGPl3aAksgQ35gw27vJvvBuscZzVhLbW5PQgfeiMKVbRrWpbR5xHUuVtZ2WWJm7HboV5kXXpkiMPA5vOktA-g4mGZm_zh6ag-OMFmLSfnepSm4PEc=xUFop6_-RYgRLI020e8S8uXXWsuQR95QLXnPCQHvZ3yRFRBtHAG6j0natnjXAxodDcm2Z_tiq7H36uRojXmBVE6A=BwWr0j-cXmsXEuSiKPairOZ6lOskntyNSoE0mao=ge0YrWgZnvEWQIb8FPadYSSufxYmxuGVur0v-5RBRcw=VCk6e6d2sbMDmuvr6zR-F_T_A4K-q-8LlnXywUUfpqG-GInbAyExoE5QdJoRiI8IdTX5NHY0uzddRbfuV9ItOwwY-0iYSMgQ=Aqr=XLeqozjSV7r_Enq0FPcQjNklgG6unf=I3VMURNTQb42pvqWK5RSa-FAD3bTCp4a_OAV=Zx3cIV9JTsSpKKMmFo5q4PMPbj9boGSLSqORhvP95PwMAxyEeMUNbnorRlVOlKH3h2h2sH9rf768CgsXo-GMzCWwQXfoEuKGyl2vRQeP_rddkJ3RHuawS=Z38p69xNcjXr9J_zX4YVoAdXxN56Jm=Qo5K8Xsz3ndwSuvBJYnPXfXNr9Et80GYIjcx-TOkr5ZmPAbRIHkZVKacYvNG_utrkSQmn63IYYydikScedTdnUW5ky=gW3sliikzmZh-cRGaGIaA7FD5Un0Gc8TER6x_mKq-vL5nSuCOnHmzRTNBXZteNk8Gghe6V=ZnRNfCWFeiijdqPP0iPohDEhnjtA32h5vvyCgkx6UWBhAqSsU4IJWGPx8v9S3NtcBuYgTdexPHjnpYVF2CjufPFq0YVL5-di_ATRDfQktSh9gnBdYtIP9ejYLrq6kW6bBm5dxJkgGFD2aEz4bO2QEvoMJ=kuQGc34MBpAG2=PmtA9TErE0WfFftgec-sQfDFn2v4kerFl88C6lGdR=QQu3RBYOdW3V2ZDOONEmzXKAWQDpEZtBrID0Sb8mkQFLcOlbN28tcR8kpha_2E=DgBgLETJE_NqLs=60RdznfPKTcbGmrpBpIZGUKYGoqYzu7AHG-U9MtUAzPP-PVrgA=uOHqN_iFtPl_SCREmnCCLmihX=SsiMCGtaTqvw3RHLYhcUociEskbQh8a4wpniDSp5OM82AHGyrRJK0b29J_KzEgzeGlNn-CLTa=8aaJRfXIIBxHs9Jr6o9Q2DKuwKxvLJpN9kkSMLNbn9Q8R7NeeEeR9oECc9D8h8WjCGkCYJXIPtp-0hpRrlkvvF5ezZhFE0D=GewHdENPEj7CTjqfClVQ=gtzPQF6lG8UX9SKYKxOdyH-jKXJNp2KYWNFTWXS6ZHc56hsw9apvtbxuMciLZYjd4ZGFmLOpHsd7liyRk3QalSm0zxCnnOAL88S=VKDvdWm_qHkslUSEYOOznKY6wqfMlUUxIvRQISzWKY56ajlfyx0qEY6VHuiWbph2kwLGyuVZ6zwmO7RtVCiwp3poGfEnaA2ILv2IwjYm_BIcw8W2joV5G5LwkyEoeUquQWGg9EI_NEW4TO=wKOnmnwFI_f43k595lTHhiYCAwamNnFuiTXA4ZjE5J3S_O45altEdQCYJyJey2K5_iUa--qTmB3OLVZcDA05jN9TTer=-8BCZm=B9F=L=oHYSUlUC=TuzgBRH52I9BnRHDhLW8-U5yy-zYuoq55W-pboEVmlpIocbkE5A6v4gZkgxkLMKk9OzhOGNe7IgIZ=rzP7ocA0CfIKUj80anZlQj8FpVapjmpjWfJlxiZQ9oVcxZWWeSZ9ZKbrGBbx4kja_kjZeVhXOtH0MQLdyqtjIuPKK9pFkvpAOBRv4xrDPpZPOzi59F97zhc2IPhNlgEq9-Dvnl2w40xKRnjBS2cihGy_s2HI2Rf8HsrtHqrWuVtZ7clf2pEwHmdEzIr3aIHSwRzIma_ymwUZ3y4mPqs_d=JtpbqFqd6xykIT_=6Ftz_tqDC4xgTsNUL7-f57F45qpTw5QUNi7CpNUR=wvGhEOskiY2A60ID8IzGlz5hAu785vhseZHnv2fIl5KeH=I=2TH7wJGsUQ-=XMMzg5O8f940hyGJ-y4cHvWppYDK6J6LRvU-AR_Ojb4YgGj9DXgWIWCFrbiS0XY53O-_MvUfsLiURAmsfy5ReKw6-i=rTJ-kN_tf7xcOEbHNCVjFR=ZpJV6LIsI2S5loedjIRHWPhqVI5rJe-NXLs9bo53qnYsWFHbMonxaERtC2ZTYAhbWZglUGHAjv-hqvq22kCGb5fCte6Zb5brvE9CPI8Y8PZ7BCFfcbTpLOAesvHNAxJMaBUVhRGOoYVAL7tq5E-FVmH9L=bdbBgOAXR9s5lHwMRleEOIpqHuyC32z5CC75FAJVly8IP7vLjCjKrDHhNu6yc_5MOvbOQO8CCr0TwLj_XsoSkshH9Q5ICGARL64Ts8F7pmDKC9EDyKR7lZ-tkAJXcCgJd_KCIGnHfULSlwoFrj5QPcVYvGMcaY3Jl4rjyIhctEKXkUiGDwzpjnYbhzVBpsIR=LW6VLVxQXinIFkyBob7lFlyX_y=Gm=5garfnBd29OZWjfym0qh0Em24hJXlbWE7ufXNFUAA8dPNfzjsFk=hOtnpqAokP56vsnOnK0Jl_fOl=5ycCZG6rqGeIDXbRbM_AEy=CPJbE_wbbB2S2Mp9g49Bv9tn_S3LPG_aA_aAWGVDSIVs-3=kwdIybN8y
X-J3PopQvX-d: AAaCxAiBBKCMgcGASZAQgICAACIqGSI6QDSXAaoclOq4s6T2AkpRq7ZjyDmvHZIRbYeFd-E
X-Forwarded-For: 39.41.162.210
X-Forwarded-Proto: https
X-J3PopQvX-f: A3r8AMZyAQAABWpomSNWTAA_aOd-ML0d_DenMFOwBhK_GRxSWvW5FLJRxZ2sAScpotKucnW8wH8AAEB3AAAAAA==
X-J3PopQvX-c: ACCn_sVyAQAAdkviREb3CdA4iVdSrR4-Gc4Y3RxJWrLhnUx8XioZIjpANJcB
X-TLS-Version: 771
X-J3PopQvX-z: q

But Im unable to get these headers by executing above code. How do I get these?

Edit: I used the answer and it works fine. There is only one missing part.

page.route('**', route => {
    const request = route.request()
    //console.log(request.url(), JSON.stringify(request.headers()));
    return route.continue();
  });
  let pageRes = await page.goto(url);
  await page.waitForTimeout(3 * 1000)
  //await page.unroute("**");
  return pageRes;

When I print PageRes.request().headers() it doesnt give j3pop headers.It simply gives

{
  accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.2 Safari/605.1.15'
}

Edit: This is what I tried in interactive Playwright

const { webkit } = require('playwright');

(async () => {
  const browser = await webkit.launch();
  const context = await browser.newContext();
  const page = await context.newPage();

  // Log and continue all network requests
  headers = {}
  page.route('**/*', route => {
    const request = route.request()
    console.log(request.url(), JSON.stringify(request.headers()));
    headers = JSON.stringify(request.headers());
    return route.continue();
  });

  let url = await page.goto('https://www.sahibinden.com/en/cars?sorting=date_desc');
  console.log("headerw");
  console.log(headers)
  await page.waitForTimeout(3 * 1000)
  
  await browser.close();
})();

headers isn't giving me the j3pop headers

Answer 1

The issue is, that the requests will be made, after the page is fully loaded from the browser perspective. So you have to either use something like waitForTimeout which waits X milliseconds or use waitForSelector if you have an element which will in the end appear after your request was made.

See this slightly modified example, which waits additional 3 seconds and logs then your request with the special headers to the console.

const { webkit } = require('playwright');

(async () => {
  const browser = await webkit.launch();
  const context = await browser.newContext();
  const page = await context.newPage();

  // Log and continue all network requests
  await page.route('**', route => {
    const request = route.request()
    console.log(request.url(), JSON.stringify(request.headers()));
    return route.continue();
  });

  await page.goto('https://www.sahibinden.com');
  await page.waitForTimeout(3 * 1000)
  await browser.close();
})();

Or interactive here: https://try.playwright.tech/?s=17v5y

Answer 2

You need to add await before page.route(... or else routing may not be setup yet when page.goto runs.

ref: https://playwright.dev/docs/api/class-page#page-route