Web scraper using dart

Question

I am trying to create a web scraper for a single website picking out just a title, Image and link to the website. The title comes out fine but the image and link are not properly working can anyone please help me out in this.Here is my code and dependencies I used in .yaml

import 'package:flutter/material.dart';
import 'package:flutter_staggered_animations/flutter_staggered_animations.dart';
import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as parser;
import 'package:http/http.dart' as http;
import 'package:url_launcher/url_launcher.dart';

void main() {
  runApp(MyApp());
}

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      home: MyHomePage(),
    );
  }
}

class MyHomePage extends StatefulWidget {
  @override
  _MyHomePageState createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> {
  List<String> title;
  List<String> image;
  List<String> link;

  @override
  void initState() {
    _getdataFromWeb();
  }

  void _getdataFromWeb() async {
    final response = await http.get('https://www.bewakoof.com/men-shirts');
    dom.Document document = parser.parse(response.body);
    final pictures = document.getElementsByClassName('productGrid');
    final description = document.getElementsByClassName('productCardDetail');
    final nextPage =
      //document.getElementsByClassName('entry-title');
      document.getElementsByClassName('col-sm-4 col-xs-6');

    image = pictures
      .map((element) =>
          element.getElementsByTagName("img")[0].attributes['src'])
      .toList();
    title = description
      .map((element) => element.getElementsByTagName("h3")[0].innerHtml)
      .toList();
    link = nextPage
      .map((element) => element.getElementsByTagName("a")[0].attributes['href'])
      .toList();
    print(link);
  }

  @override
  Widget build(BuildContext context) {
    print("hello");
    if (image == null)
      print("null");
    else
      print(image);
    return SafeArea(
      child: Scaffold(
      backgroundColor: Colors.black87,
      body: title == null || title.length == 0
        ? Text(
            "No data",
            style: TextStyle(
              color: Colors.white,
            ),
          )
        : ListView.builder(
            itemCount: title.length,
            itemBuilder: (context, index) {
              return AnimationConfiguration.staggeredList(
                position: index,
                duration: const Duration(milliseconds: 375),
                child: SlideAnimation(
                  child: FadeInAnimation(
                    child: GestureDetector(
                      onTap: () async {
                        dynamic url = link[index];
                        if (await canLaunch(url))
                          launch(url);
                        else {
                          print('error');
                        }
                      },
                      child: Padding(
                        padding: const EdgeInsets.all(10),
                        child: Card(
                          child: Container(
                            color: Colors.black87,
                            child: Column(
                              children: <Widget>[
                                Align(
                                  alignment: Alignment.centerLeft,
                                  child: Text(
                                    title[index],
                                    style: TextStyle(
                                      fontWeight: FontWeight.bold,
                                      color: Colors.cyan,
                                      fontSize: 20,
                                    ),
                                  ),
                                ),
                                SizedBox(height: 15),
                                Image.network(image[0]),
                              ],
                            ),
                          ),
                        ),
                      ),
                    ),
                  ),
                ),
              );
            },
          ),
      ),
    );
  }
}

cupertino_icons: ^1.0.1
http: ^0.12.0+4
html: ^0.14.0+3
flutter_staggered_animations: "^0.1.2"
url_launcher: ^5.4.0

I may need it tomorrow if it can be possible

Answer 1

enter image description here

Here I added the html pic for the elements rest part I debugged only the link is not working. Here is the debugged code:

import 'package:flutter/material.dart';
import 'package:flutter_staggered_animations/flutter_staggered_animations.dart';
import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as parser;
import 'package:http/http.dart' as http;
import 'package:url_launcher/url_launcher.dart';

void main() {
  runApp(MyApp());
}

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      home: MyHomePage(),
    );
  }
}

class MyHomePage extends StatefulWidget {
  @override
  _MyHomePageState createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> {
  List<String> title;
  List<String> image;
  List<String> link;

  @override
  void initState() {
    _getdataFromWeb();
}

      void _getdataFromWeb() async {
    final response = await http.get(
        'https://www.bewakoof.com/');
    dom.Document document = parser.parse(response.body);
    final pictures =
        document.getElementsByClassName('chWrprInner');
    //document.getElementsByClassName('entry-header blog-entry-header');
final description =
    //document.getElementsByClassName('entry-content');
    document.getElementsByClassName('chWrprInner');
final nextPage =
    //document.getElementsByClassName('entry-title');
    document.getElementsByClassName('chWrprInner');

image = pictures
    .map((element) =>
          element.getElementsByTagName("img")[0]
          .attributes['src'])
      .toList();
  title = description
      .map((element) => element.getElementsByTagName("p")[0]
          .innerHtml)
       .toList();
  link = nextPage
      .map((element) =>
          element.getElementsByTagName("a")[0]
          .attributes['href'])
       .toList();
  print(link);
  }

  @override
  Widget build(BuildContext context) {
print("hello");
if (image == null)
  print("null");
else
  print(image);
return SafeArea(
  child: Scaffold(
    backgroundColor: Colors.black87,
    body: title == null || title.length == 0
        ? Text(
            "No data",
            style: TextStyle(
              color: Colors.white,
            ),
          )
        : ListView.builder(
            itemCount: title.length,
            itemBuilder: (context, index) {
              return AnimationConfiguration.staggeredList(
                position: index,
                duration: const Duration(milliseconds: 375),
                child: SlideAnimation(
                    child: FadeInAnimation(
                        child: GestureDetector(
                  onTap: () async {
                    dynamic url = link[index];
                    if (await canLaunch(url))
                      launch(url);
                    else {
                      print('error');
                    }
                  },
                  child: Padding(
                    padding: const EdgeInsets.all(10),
                    child: Card(
                      child: Container(
                        color: Colors.black87,
                        child: Column(
                          children: <Widget>[
                            Align(
                              alignment: Alignment.centerLeft,
                              child: Text(
                                title[index],
                                style: TextStyle(
                                  fontWeight: FontWeight.bold,
                                  color: Colors.cyan,
                                  fontSize: 20,
                                ),
                              ),
                            ),
                            SizedBox(height: 15),
                            Text(
                                 title[index],
                                 style: TextStyle(
                               color: Colors.white,
                              ),
                            ),
                            Image.network(image[index]),
                          ],
                        ),
                      ),
                    ),
                  ),
                ))),
              );
            },
          ),
  ),
);
}

}

Answer 2

you can use this package for web scraping:

  web_scraper: ^0.0.9

if need see sample , you can visit this repository:

https://github.com/esmaeil-ahmadipour/Flutter_Web_Scraper