'Why am I getting an encrypted response when web scrapping using Selenium?
I am trying to perform web scraping on this website. https://www.celcom.com.my/ The response HTML content received is encrypted but when I inspect through the browser (using the developer tools), I could see the HTML content.
import selenium
from selenium import webdriver
import requests
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
url = 'https://www.celcom.com.my/'
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--allow-running-insecure-content')
options.add_argument("--disable-extensions")
options.add_argument("--proxy-server='direct://'")
options.add_argument("--proxy-bypass-list=*")
options.add_argument("--start-maximized")
options.add_argument('--disable-gpu')
options.add_argument("--incognito")
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--no-sandbox')
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
browser = webdriver.Chrome(executable_path='chromedriver.exe', options=options)
browser.get(url)
html_source_code = browser.execute_script("return document.innerHTML;")
browser.set_page_load_timeout(10)
html = browser.page_source
soup = BeautifulSoup(html, "html5lib")
browser.implicitly_wait(10)
soup
I used the above code snippet to perform web scraping. The output I get is encrypted and it is not in the desired format.
<html><head><meta charset="utf-8"/><script>function i700(){}i700.F20=function (){return typeof i700.O20.p60==='function'?i700.O20.p60.apply(i700.O20,arguments):i700.O20.p60;};i700.X70=function (){return typeof i700.v70.p60==='function'?i700.v70.p60.apply(i700.v70,arguments):i700.v70.p60;};i700.Z20=function (){return typeof i700.O20.P20==='function'?i700.O20.P20.apply(i700.O20,arguments):i700.O20.P20;};i700.Q60=function (){return typeof i700.Y60.P20==='function'?i700.Y60.P20.apply(i700.Y60,arguments):i700.Y60.P20;};i700.a70=function (){return typeof i700.v70.d4==='function'?i700.v70.d4.apply(i700.v70,arguments):i700.v70.d4;};i700.P60=function (){return typeof i700.Y60.p60==='function'?i700.Y60.p60.apply(i700.Y60,arguments):i700.Y60.p60;};i700.r60=function (){return typeof i700.Y60.U20==='function'?i700.Y60.U20.apply(i700.Y60,arguments):i700.Y60.U20;};i700.t20=function (){return typeof i700.O20.U20==='function'?i700.O20.U20.apply(i700.O20,arguments):i700.O20.U20;};i700.g60=function (){return typeof i700.Y60.d4==='function'?(P4.charCodeAt(g4+1)&0xff)<<8|(P4.charCodeAt(g4+2)&0xff)<<16|(P4.charCodeAt(g4+3)&0xff)<<24;j4=k4(j4,v4);j4=(j4&0x1ffff)<<15|j4>>>17;j4=k4(j4,G4);V4^=j4;V4=(V4&0x7ffff)<<13|V4>>>19;V4=V4*5+0xe6546b64|0;}j4=0;switch(y4%4){case 3:j4=(P4.charCodeAt(C4+2)&0xff)<<16;case 2:j4|=(P4.charCodeAt(C4+1)&0xff)<<8;case +"1841442348";C80=-255845232;k60.Z20(k60.y60()[38][4]);y80=k60.N20("2",1);for(var G80="1"|0;k60.K4(G80.toString(),G80.toString().length,"18681"|0)!==g80;G80++){Q1={};y80+=2;}function c1(){}if(k60.K4(y80.toString(),y80.toString().length,648)!==C80){Q1={};}Q1[k60.X70("215"-0)+Y1]=T1;console[k60.X70(+"33")](k60.X70(+"503"),Q1);t7[k60.X70(+"368")]({'\x75\x72\x6c':[k60.L70(+"396"),k60.L70(+"91"),m1][k60.L70("45"*1)](k60.X70("319"-0)),'\x68\x65\x61\x64\x65\x72\x73':Q1})[k60.X70("196"*1)](function(f1,F1){if(window[k60.X70(488)][k60.L70(+"385")]==k60.X70("496"-0)){setTimeout(function(){window[k60.L70(+"387")][k60.X70("195"|0)]();},1);}else{window[k60.L70(+"314")]=f1[k60.X70(550)]();}});}O1(window[k60.X70("488"*1)][k60.L70(32)],window[k60.L70("488"-0)][k60.L70("227"-0)]);function O1(q1,l1){var G1,k1,a1,h1;k60.n20(k60.e60()[12][18]);G1=k60.t20("0",1);k60.Z20(k60.e60()[33][21]);k1=k60.N20(q1,G1);a1=s7(k1,l1);h1=function(){var R1;if(a1){R1=[a1,G1,r7,E7,window[k60.L70(+"488")][k60.X70(k60.t20("32",0,k60.n20(k60.e60()[26][2])))]];z1(b7(R1[k60.X70(+"45")](d7))[k60.L70(605)](/\u003d/g,k60.L70(+"450")),k60.X70("314"-0),K1,!{});}else{G1++;k60.Z20(k60.e60()[22][35]);k1=k60.N20(q1,G1);a1=s7(k1,l1);setTimeout(h1,+"0");}};h1();}};}else{window[k60.X70(607)]=function(){return 0x23;};}}());
I am expecting an output in the below format
<div class="container"><!----><div class="row section-header"><h2 class="col-12 col-lg-6 section-title">Not all networks are equal</h2><!----></div><div class="row row-eq-height"><div class="col-sm-12 col-md-12 col-lg-12 col-xl-12 callout aos-init aos-animate" data-aos="slide-left" data-aos-duration="2000"><ngu-carousel class="slider__list ngucarouselrT8TQa" style="width: 100%; src="/sites/default/files/cms_content_images/svg-image/01-Icon%20Expressive%20%CE%93%C3%87%C3%B6%20Ultra%20Wide%204G%20LTE%20Coverage.svg" width="96"></div><div class="col-12 text-center callout-icon-container" style="color: white;"><div class="subContentTitle callout-icon-item__headline" style="color: white;">Ultra wide 4G LTE coverage</div><div class="callout-icon-item__content subContent" style="color: white;">Powered by 9,200 sites to cover 94% of Malaysia’s network population</div><!----><!----></div></div></div></ngu-tile><ngu-tile _nghost-my-test-c30="" class="item"><div _ngcontent-my-test-c30="" class="tile"><div class="col-12 zeroPadding call-out-icon-container"><div class="justify-content-center align-items-center callout-icon-item__image disableBorder" style="border-color: white;"><img alt="" class="img-fluid" id="image" src="/sites/default/files/cms_content_images/svg-image/02-Icon%20Expressive%20%CE%93%C3%87%C3%B6%20Smooth%204K%20Videos.svg" width="96"></div><div class="col-12 text-center callout-icon-container" style="color: white;"><div class="subContentTitle callout-icon-item__headline" style="color: white;">Smooth 4K videos</div><div class="callout-icon-item__content subContent" style="color: white;">Enjoy the best video streaming experience</div><!----><!----></div></div></div></ngu-tile><ngu-tile _nghost-my-test-c30="" class="item"><div _ngcontent-my-test-c30="" class="tile"><div class="col-12 zeroPadding call-out-icon-container"><div class="justify-content-center align-items-center callout-icon-item__image disableBorder" style="border-color: white;"><img alt="" class="img-fluid" id="image" src="/sites/default/files/cms_content_images/svg-image/03-Icon%20Expressive%20%CE%93%C3%87%C3%B6%20Low%20Latency.svg" width="96"></div><div class="col-12 text-center callout-icon-container" style="color: white;"><div class="subContentTitle callout-icon-item__headline" style="color: white;">Lowest latency</div><div class="callout-icon-item__content subContent" style="color: white;">Low latency for streaming and gaming without delays</div><!----><!----></div></div></div></ngu-tile><ngu-tile _nghost-my-test-c30="" class="item"><div _ngcontent-my-test-c30="" class="tile"><div class="col-12 zeroPadding call-out-icon-container"><div class="justify-content-center align-items-center callout-icon-item__image disableBorder" style="border-color: white;"><img alt="" class="img-fluid" id="image" src="/sites/default/files/cms_content_images/svg-image/05-Icon%20Expressive%20%CE%93%C3%87%C3%B6%20%20Rewards.svg" width="96"></div><div class="col-12 text-center callout-icon-container" style="color: white;"><div class="subContentTitle callout-icon-item__headline" style="color: white;">Exciting rewards, exclusively yours</div><div class="callout-icon-item__content subContent" style="color: white;">Year-round deals, freebies and lifestyle privileges</div><!----><!----></div></div></div></ngu-tile></div></div><div _ngcontent-my-test-c29="" class="nguclearFix"></div><button ngucarouselprev="" class="leftRs2"></button><button ngucarouselnext="" class="rightRs2"></button></div><ul ngucarouselpoint="" class="myPoint"><!----><li class="active"></li><li class=""></li></ul><style _ngcontent-my-test-c29="">
With respect to the above issue, can someone help me with
- Why am I getting an encrypted output? What is the reason for that?
- Workaround for getting the expected output as mentioned above
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|