Skip to main content

Web Crawler or Site grabber code in java with Page downloader

/**
 * @Web Crawler *
 *
 * @author sharma
 * @version 1.00 2010/7/16
 */

import java.io.BufferedInputStream;

import java.io.IOException;

import java.net.MalformedURLException;

import java.util.*;

import java.net.URLConnection;



import java.awt.BorderLayout;

import java.awt.Color;

import javax.swing.JFrame;

import javax.swing.JTabbedPane;

import javax.swing.JLabel;

import javax.swing.JPanel;

import javax.swing.JButton;

import javax.swing.SwingConstants;

import javax.swing.*;

import java.awt.*;

import java.awt.event.*;

import java.awt.Transparency;

import javax.swing.event.*;

import java.sql.*;

import java.util.*;

import java.net.URL;

import javax.imageio.ImageIO;

import java.lang.*;



import java.io.*;

import javax.swing.BoxLayout;



public class assign_3 extends JFrame {

   

    String name="";

    String str[] = new String[1000];

    int k;

    StringBuilder builder;



    JLabel Send_lebel = new JLabel();

    JTextField send_bit = new JTextField();

   

    JLabel Rec_lebel = new JLabel();

    JTextArea rec_bit = new JTextArea();

    JTextField err = new JTextField();

   

    JButton gen = new JButton();

    JButton sen = new JButton();

    JButton cal = new JButton();

   

    int a[] = new int[100];

    int b[] = new int[100];

    WindowListener l;



    public assign_3(){

       

       

        setTitle("ASSIGNMENT #3");

        setSize(600,453);

        setLocation(220,100);



        l = new WindowAdapter() {

            public void windowClosing(WindowEvent e) {

                setEnabled(true);

            }

        };

         setUndecorated(true);

        getRootPane().setWindowDecorationStyle(JRootPane.FRAME);

        setVisible(true);

        jbinit();

       

    }

   

   

       void jbinit(){

             this.setResizable(false);

             this.getContentPane().setLayout(null);

            

             Send_lebel.setText("ADDRESS:");

             Send_lebel.setBounds(new Rectangle(8, 18, 150, 42));

             send_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             send_bit.setText("");

             send_bit.setBounds(new Rectangle(18, 48, 156, 21));

            

             Rec_lebel.setText("LINKS:");

             Rec_lebel.setBounds(new Rectangle(350, 48, 132, 21));

             rec_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             rec_bit.setText("");

             rec_bit.setBounds(new Rectangle(345,88,226,311));

            

             gen.setBounds(new Rectangle(48, 100, 88, 24));

             gen.setFont(new java.awt.Font("Dialog", 1, 12));

             gen.setBorder(BorderFactory.createRaisedBevelBorder());

             gen.setMnemonic('G');

             gen.setText("GO!");    

             gen.addActionListener(new gen_actionAdapter(this));

           

             sen.setBounds(new Rectangle(208, 100, 88, 24));

             sen.setFont(new java.awt.Font("Dialog", 1, 12));

             sen.setBorder(BorderFactory.createRaisedBevelBorder());

             sen.setMnemonic('S');

             sen.setText("DOWNLOAD");    

             sen.addActionListener(new sen_actionAdapter(this));

           

                    

                       

            

             this.getContentPane().add(Send_lebel, null);

             this.getContentPane().add(Rec_lebel, null);

             this.getContentPane().add(rec_bit, null);

             this.getContentPane().add(send_bit, null); 

             this.getContentPane().add(gen,null);

             this.getContentPane().add(sen,null);     

              



              



    }



    public static void main(String[] args) {

       assign_3 jmain = new assign_3();



       jmain.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);



    }

   

  

// all button action

void gen_actionPerformed(ActionEvent e)

{



 int head,tail,num;

 boolean flag=true; 

 head = tail = num= 0;

 Vector<String> vc = new Vector<String>();

 Map m1 = new HashMap();



 String pre ="";

          name = send_bit.getText().toLowerCase();

          vc.addElement(name);

              System.out.println(name); 

                  m1.put(name,true);

                  ++tail;

              while(flag)

              {

                  if(head == tail){break;}

                  name = (String)vc.elementAt(head);

                 

                  System.out.println(name);

                  ++head;

            try {

           

       

            URL url = new URL("http://"+name);

            URLConnection urlc = url.openConnection();

            String st ="";

            BufferedInputStream buffer = new BufferedInputStream(urlc.getInputStream());

           

            builder = new StringBuilder();

            int byteRead;

            while ((byteRead = buffer.read()) != -1)

            {

                

                 builder.append((char) byteRead);

                 st += (char)byteRead;

            }

            buffer.close();

            System.out.println("URL VALID");

                  try {

                      ++num;

                BufferedWriter out = new BufferedWriter(new FileWriter("D:\\html\\"+num+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"+ep); 

                    } 

          

           // finding URL in given Page

            int r = 0;

            int i,j,l,s;

           

            for(i=0;i<1000;i++)

                str[i] = "";

               

             k = 0;   

             for(i=0;i<st.length();i++)

             {

               r = l = 0;

               for(j=i;j<st.length();j++)

               {

                  if(st.charAt(j) == 'w' && r == 0)  

                      ++l;

             

                 

                  if(l == 3)

                  {

                    str[k]+="www."; 

                      for(s=j+2;;s++)

                      {

                        if(st.charAt(s) == ' ' || st.charAt(s) == '<' || st.charAt(s) == '"')break;   

                        str[k] +=  st.charAt(s);

                       

                      }

                     

                      k++;

                      break;

                  }

                  else if(st.charAt(j) != 'w'){ r= 1;break;}

                }       

             }

            

            

             for(i=0;i<k;i++)

             {

                if(m1.containsValue(str[i]) != true)

                {

                

                    pre += str[i];

                    vc.addElement(str[i]);

                    m1.put(str[i],true);

                    ++tail;

                     pre += "\n";

                     rec_bit.setText(str[i]+"\n");

                }

             }   

            // rec_bit.setText(pre);

            }

            catch (MalformedURLException ex) {

             pre += "Sorry ! URL ERROR";

             rec_bit.setText(pre);

           

             }

            catch (IOException ex) {

            pre += "Sorry ! There Is no Site Like that";

            rec_bit.setText(pre);   

           

        }

              }

           

           

}



void sen_actionPerformed(ActionEvent e)

{

            try {

                BufferedWriter out = new BufferedWriter(new FileWriter(name+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"); 

                    } 

}







}







// all button class



class gen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    gen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.gen_actionPerformed(e);

  }

}





class sen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    sen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.sen_actionPerformed(e);

  }

}




Comments

Popular posts from this blog

UDP server client in c

Server #include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <stdio.h> #include <unistd.h> #include <errno.h> #include <string.h> #include <stdlib.h> int main() {         int sock;         int addr_len, bytes_read;         char recv_data[1024],send_data[1024];         struct sockaddr_in server_addr , client_addr;         if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) == -1) {             perror("Socket");             exit(1);         }         server_addr.sin_family = AF_INET;         server_addr.sin...

My favourite writer Humayun Ahmed

There is none who can replace him.At least the standard which he create in is life time in the running literature it never be replaceable.The new generation which is all the time busy in playing PC games,chatting ,bands etc. only his writing makes them to take a glance on the literature.For example Himu and Misir ali all the time keep them on track by anti-logic and logic.They also have show-down on the novel.I myself read all of the books of Himu and Misir ali and wait for the new one to come every year.Now I have to wait for life time. Except books he makes our dirty film industry pure by his heart warming ,well versed film.Following him many new producer try to make well and good film (not the dirty one) .He also contribute our drama by his dashing drama's. In writing except romance ,logic ,anti-logic ,he also write many science fiction.His brother Sir Dr. Md. Zafar Iqbal is the man who started science fiction in bangla. The list of books: Selected novels • Lilaboti (2...

[ASTERIK] configure: error: *** uuid support not found (this typically means the uuid development package is missing)

ISSUE: Build error on Asterik , when you want test webrtc feature :) checking for uuid_generate_random in -luuid... no checking for uuid_generate_random in -le2fs-uuid... no checking for uuid_generate_random... no configure: error: *** uuid support not found (this typically means the uuid development package is missing) Fix: This issue arises due to missing of UUID generator specified by rfc4122 . +Linux sudo apt-get install uuid-dev  @Unix yum -y install libuuid-devel Asterik comes with lots of helpful script available on - asterisk/contrib/scripts/ folder of your ASTERIK source. So just use the following command on UNIX console to run the asterik pre-requisite script. contrib/scripts/install_prereq install And you are done! configuring. Now -- Make Asterik.