Saturday, August 20, 2011

Web Crawler or Site grabber code in java with Page downloader

/**
 * @Web Crawler *
 *
 * @author sharma
 * @version 1.00 2010/7/16
 */

import java.io.BufferedInputStream;

import java.io.IOException;

import java.net.MalformedURLException;

import java.util.*;

import java.net.URLConnection;



import java.awt.BorderLayout;

import java.awt.Color;

import javax.swing.JFrame;

import javax.swing.JTabbedPane;

import javax.swing.JLabel;

import javax.swing.JPanel;

import javax.swing.JButton;

import javax.swing.SwingConstants;

import javax.swing.*;

import java.awt.*;

import java.awt.event.*;

import java.awt.Transparency;

import javax.swing.event.*;

import java.sql.*;

import java.util.*;

import java.net.URL;

import javax.imageio.ImageIO;

import java.lang.*;



import java.io.*;

import javax.swing.BoxLayout;



public class assign_3 extends JFrame {

   

    String name="";

    String str[] = new String[1000];

    int k;

    StringBuilder builder;



    JLabel Send_lebel = new JLabel();

    JTextField send_bit = new JTextField();

   

    JLabel Rec_lebel = new JLabel();

    JTextArea rec_bit = new JTextArea();

    JTextField err = new JTextField();

   

    JButton gen = new JButton();

    JButton sen = new JButton();

    JButton cal = new JButton();

   

    int a[] = new int[100];

    int b[] = new int[100];

    WindowListener l;



    public assign_3(){

       

       

        setTitle("ASSIGNMENT #3");

        setSize(600,453);

        setLocation(220,100);



        l = new WindowAdapter() {

            public void windowClosing(WindowEvent e) {

                setEnabled(true);

            }

        };

         setUndecorated(true);

        getRootPane().setWindowDecorationStyle(JRootPane.FRAME);

        setVisible(true);

        jbinit();

       

    }

   

   

       void jbinit(){

             this.setResizable(false);

             this.getContentPane().setLayout(null);

            

             Send_lebel.setText("ADDRESS:");

             Send_lebel.setBounds(new Rectangle(8, 18, 150, 42));

             send_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             send_bit.setText("");

             send_bit.setBounds(new Rectangle(18, 48, 156, 21));

            

             Rec_lebel.setText("LINKS:");

             Rec_lebel.setBounds(new Rectangle(350, 48, 132, 21));

             rec_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             rec_bit.setText("");

             rec_bit.setBounds(new Rectangle(345,88,226,311));

            

             gen.setBounds(new Rectangle(48, 100, 88, 24));

             gen.setFont(new java.awt.Font("Dialog", 1, 12));

             gen.setBorder(BorderFactory.createRaisedBevelBorder());

             gen.setMnemonic('G');

             gen.setText("GO!");    

             gen.addActionListener(new gen_actionAdapter(this));

           

             sen.setBounds(new Rectangle(208, 100, 88, 24));

             sen.setFont(new java.awt.Font("Dialog", 1, 12));

             sen.setBorder(BorderFactory.createRaisedBevelBorder());

             sen.setMnemonic('S');

             sen.setText("DOWNLOAD");    

             sen.addActionListener(new sen_actionAdapter(this));

           

                    

                       

            

             this.getContentPane().add(Send_lebel, null);

             this.getContentPane().add(Rec_lebel, null);

             this.getContentPane().add(rec_bit, null);

             this.getContentPane().add(send_bit, null); 

             this.getContentPane().add(gen,null);

             this.getContentPane().add(sen,null);     

              



              



    }



    public static void main(String[] args) {

       assign_3 jmain = new assign_3();



       jmain.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);



    }

   

  

// all button action

void gen_actionPerformed(ActionEvent e)

{



 int head,tail,num;

 boolean flag=true; 

 head = tail = num= 0;

 Vector<String> vc = new Vector<String>();

 Map m1 = new HashMap();



 String pre ="";

          name = send_bit.getText().toLowerCase();

          vc.addElement(name);

              System.out.println(name); 

                  m1.put(name,true);

                  ++tail;

              while(flag)

              {

                  if(head == tail){break;}

                  name = (String)vc.elementAt(head);

                 

                  System.out.println(name);

                  ++head;

            try {

           

       

            URL url = new URL("http://"+name);

            URLConnection urlc = url.openConnection();

            String st ="";

            BufferedInputStream buffer = new BufferedInputStream(urlc.getInputStream());

           

            builder = new StringBuilder();

            int byteRead;

            while ((byteRead = buffer.read()) != -1)

            {

                

                 builder.append((char) byteRead);

                 st += (char)byteRead;

            }

            buffer.close();

            System.out.println("URL VALID");

                  try {

                      ++num;

                BufferedWriter out = new BufferedWriter(new FileWriter("D:\\html\\"+num+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"+ep); 

                    } 

          

           // finding URL in given Page

            int r = 0;

            int i,j,l,s;

           

            for(i=0;i<1000;i++)

                str[i] = "";

               

             k = 0;   

             for(i=0;i<st.length();i++)

             {

               r = l = 0;

               for(j=i;j<st.length();j++)

               {

                  if(st.charAt(j) == 'w' && r == 0)  

                      ++l;

             

                 

                  if(l == 3)

                  {

                    str[k]+="www."; 

                      for(s=j+2;;s++)

                      {

                        if(st.charAt(s) == ' ' || st.charAt(s) == '<' || st.charAt(s) == '"')break;   

                        str[k] +=  st.charAt(s);

                       

                      }

                     

                      k++;

                      break;

                  }

                  else if(st.charAt(j) != 'w'){ r= 1;break;}

                }       

             }

            

            

             for(i=0;i<k;i++)

             {

                if(m1.containsValue(str[i]) != true)

                {

                

                    pre += str[i];

                    vc.addElement(str[i]);

                    m1.put(str[i],true);

                    ++tail;

                     pre += "\n";

                     rec_bit.setText(str[i]+"\n");

                }

             }   

            // rec_bit.setText(pre);

            }

            catch (MalformedURLException ex) {

             pre += "Sorry ! URL ERROR";

             rec_bit.setText(pre);

           

             }

            catch (IOException ex) {

            pre += "Sorry ! There Is no Site Like that";

            rec_bit.setText(pre);   

           

        }

              }

           

           

}



void sen_actionPerformed(ActionEvent e)

{

            try {

                BufferedWriter out = new BufferedWriter(new FileWriter(name+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"); 

                    } 

}







}







// all button class



class gen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    gen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.gen_actionPerformed(e);

  }

}





class sen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    sen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.sen_actionPerformed(e);

  }

}




No comments:

Post a Comment

How to enable hotspot in TPG iPhone

 By default, the hotspot does not work on the phone. It will ask you to contact the provider. This video will help you bypass the network ...