BeautifulSoup / Get content from script tag?

Question

i would like to scrape the genre- and artist-information from this tag:

<script type="text/javascript">
window.rtkGPTSlotsTargeting = [
    [
        ["genre", "pop"],
        ["artist", "a1"]
    ]
];
</script>

I tried it with this code:

from fake_useragent import UserAgent
import requests
from bs4 import BeautifulSoup
import json

link = "https://www.azlyrics.com/lyrics/a1/foreverinlove.html"
ua = UserAgent()
userAgent = ua.random
HEADERS = {"User-Agent": userAgent}

page = requests.get (link, headers=HEADERS)
soup = BeautifulSoup (page.content, "html.parser")
tmpScript = soup.find("script")
data = json.loads(tmpScript.string)
print(data)

But i allways get this error:

$ python collLyrics.py
Traceback (most recent call last):
  File "C:\Users\Polzi\Documents\DEV\Fiverr\TRY\kushabateni\collLyrics.py", line 14, in <module>
    data = json.loads(tmpScript.string)
  File "c:\users\polzi\appdata\local\programs\python\python39\lib\json\__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "c:\users\polzi\appdata\local\programs\python\python39\lib\json\decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "c:\users\polzi\appdata\local\programs\python\python39\lib\json\decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 2 column 1 (char 2)
(NormalScraping) 

How can i get this 2 informations from the tag?

Answer 1

You can use a regular expression pattern to find the correct data: \[.*\] will search for all the text within brackets:

from fake_useragent import UserAgent
import requests
from bs4 import BeautifulSoup
import re

link = "https://www.azlyrics.com/lyrics/a1/foreverinlove.html"
ua = UserAgent()
userAgent = ua.random
HEADERS = {"User-Agent": userAgent}

page = requests.get(link, headers=HEADERS)
soup = BeautifulSoup(page.content, "html.parser")
tmpScript = soup.find("script")

for tag in re.findall(r"(\[.*\])", tmpScript.string):
    print(tag)

Output:

["genre", "pop"]
["artist", "a1"]

Answer 2

A little "weirder" solution but still gets the desired output.

import json
import re

import requests
from bs4 import BeautifulSoup

data = (
    json.loads(
        re.search(
            r"= \[(.*)\]",
            BeautifulSoup(
                requests.get(
                    "https://www.azlyrics.com/lyrics/a1/foreverinlove.html"
                ).content,
                "html.parser"
            )
            .find("script")
            .string,
            re.S
        ).group(1)
    )
)

print(json.dumps(data, indent=2)

Output:

[
  [
    "genre",
    "pop"
  ],
  [
    "artist",
    "a1"
  ]
]

Answer 3

Using named groups in regex pattern as another example

from fake_useragent import UserAgent
import requests
import re

link = "https://www.azlyrics.com/lyrics/a1/foreverinlove.html"
ua = UserAgent()
userAgent = ua.random
HEADERS = {"User-Agent": userAgent}

page = requests.get(link, headers=HEADERS)
match = re.search(r'genre["\s,]+(?P<genre>(.*?))".*?artist["\s,]+(?P<artist>(.*?))"', page.text, flags = re.S)

for target in ['genre', 'artist']:
    print(f'{target} = {match.group(target)}')

Explanation: