Skip to main content

Web Crawler or Site grabber code in java with Page downloader

/**
 * @Web Crawler *
 *
 * @author sharma
 * @version 1.00 2010/7/16
 */

import java.io.BufferedInputStream;

import java.io.IOException;

import java.net.MalformedURLException;

import java.util.*;

import java.net.URLConnection;



import java.awt.BorderLayout;

import java.awt.Color;

import javax.swing.JFrame;

import javax.swing.JTabbedPane;

import javax.swing.JLabel;

import javax.swing.JPanel;

import javax.swing.JButton;

import javax.swing.SwingConstants;

import javax.swing.*;

import java.awt.*;

import java.awt.event.*;

import java.awt.Transparency;

import javax.swing.event.*;

import java.sql.*;

import java.util.*;

import java.net.URL;

import javax.imageio.ImageIO;

import java.lang.*;



import java.io.*;

import javax.swing.BoxLayout;



public class assign_3 extends JFrame {

   

    String name="";

    String str[] = new String[1000];

    int k;

    StringBuilder builder;



    JLabel Send_lebel = new JLabel();

    JTextField send_bit = new JTextField();

   

    JLabel Rec_lebel = new JLabel();

    JTextArea rec_bit = new JTextArea();

    JTextField err = new JTextField();

   

    JButton gen = new JButton();

    JButton sen = new JButton();

    JButton cal = new JButton();

   

    int a[] = new int[100];

    int b[] = new int[100];

    WindowListener l;



    public assign_3(){

       

       

        setTitle("ASSIGNMENT #3");

        setSize(600,453);

        setLocation(220,100);



        l = new WindowAdapter() {

            public void windowClosing(WindowEvent e) {

                setEnabled(true);

            }

        };

         setUndecorated(true);

        getRootPane().setWindowDecorationStyle(JRootPane.FRAME);

        setVisible(true);

        jbinit();

       

    }

   

   

       void jbinit(){

             this.setResizable(false);

             this.getContentPane().setLayout(null);

            

             Send_lebel.setText("ADDRESS:");

             Send_lebel.setBounds(new Rectangle(8, 18, 150, 42));

             send_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             send_bit.setText("");

             send_bit.setBounds(new Rectangle(18, 48, 156, 21));

            

             Rec_lebel.setText("LINKS:");

             Rec_lebel.setBounds(new Rectangle(350, 48, 132, 21));

             rec_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             rec_bit.setText("");

             rec_bit.setBounds(new Rectangle(345,88,226,311));

            

             gen.setBounds(new Rectangle(48, 100, 88, 24));

             gen.setFont(new java.awt.Font("Dialog", 1, 12));

             gen.setBorder(BorderFactory.createRaisedBevelBorder());

             gen.setMnemonic('G');

             gen.setText("GO!");    

             gen.addActionListener(new gen_actionAdapter(this));

           

             sen.setBounds(new Rectangle(208, 100, 88, 24));

             sen.setFont(new java.awt.Font("Dialog", 1, 12));

             sen.setBorder(BorderFactory.createRaisedBevelBorder());

             sen.setMnemonic('S');

             sen.setText("DOWNLOAD");    

             sen.addActionListener(new sen_actionAdapter(this));

           

                    

                       

            

             this.getContentPane().add(Send_lebel, null);

             this.getContentPane().add(Rec_lebel, null);

             this.getContentPane().add(rec_bit, null);

             this.getContentPane().add(send_bit, null); 

             this.getContentPane().add(gen,null);

             this.getContentPane().add(sen,null);     

              



              



    }



    public static void main(String[] args) {

       assign_3 jmain = new assign_3();



       jmain.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);



    }

   

  

// all button action

void gen_actionPerformed(ActionEvent e)

{



 int head,tail,num;

 boolean flag=true; 

 head = tail = num= 0;

 Vector<String> vc = new Vector<String>();

 Map m1 = new HashMap();



 String pre ="";

          name = send_bit.getText().toLowerCase();

          vc.addElement(name);

              System.out.println(name); 

                  m1.put(name,true);

                  ++tail;

              while(flag)

              {

                  if(head == tail){break;}

                  name = (String)vc.elementAt(head);

                 

                  System.out.println(name);

                  ++head;

            try {

           

       

            URL url = new URL("http://"+name);

            URLConnection urlc = url.openConnection();

            String st ="";

            BufferedInputStream buffer = new BufferedInputStream(urlc.getInputStream());

           

            builder = new StringBuilder();

            int byteRead;

            while ((byteRead = buffer.read()) != -1)

            {

                

                 builder.append((char) byteRead);

                 st += (char)byteRead;

            }

            buffer.close();

            System.out.println("URL VALID");

                  try {

                      ++num;

                BufferedWriter out = new BufferedWriter(new FileWriter("D:\\html\\"+num+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"+ep); 

                    } 

          

           // finding URL in given Page

            int r = 0;

            int i,j,l,s;

           

            for(i=0;i<1000;i++)

                str[i] = "";

               

             k = 0;   

             for(i=0;i<st.length();i++)

             {

               r = l = 0;

               for(j=i;j<st.length();j++)

               {

                  if(st.charAt(j) == 'w' && r == 0)  

                      ++l;

             

                 

                  if(l == 3)

                  {

                    str[k]+="www."; 

                      for(s=j+2;;s++)

                      {

                        if(st.charAt(s) == ' ' || st.charAt(s) == '<' || st.charAt(s) == '"')break;   

                        str[k] +=  st.charAt(s);

                       

                      }

                     

                      k++;

                      break;

                  }

                  else if(st.charAt(j) != 'w'){ r= 1;break;}

                }       

             }

            

            

             for(i=0;i<k;i++)

             {

                if(m1.containsValue(str[i]) != true)

                {

                

                    pre += str[i];

                    vc.addElement(str[i]);

                    m1.put(str[i],true);

                    ++tail;

                     pre += "\n";

                     rec_bit.setText(str[i]+"\n");

                }

             }   

            // rec_bit.setText(pre);

            }

            catch (MalformedURLException ex) {

             pre += "Sorry ! URL ERROR";

             rec_bit.setText(pre);

           

             }

            catch (IOException ex) {

            pre += "Sorry ! There Is no Site Like that";

            rec_bit.setText(pre);   

           

        }

              }

           

           

}



void sen_actionPerformed(ActionEvent e)

{

            try {

                BufferedWriter out = new BufferedWriter(new FileWriter(name+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"); 

                    } 

}







}







// all button class



class gen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    gen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.gen_actionPerformed(e);

  }

}





class sen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    sen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.sen_actionPerformed(e);

  }

}




Comments

Popular posts from this blog

UDP server client in c

Server #include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <stdio.h> #include <unistd.h> #include <errno.h> #include <string.h> #include <stdlib.h> int main() {         int sock;         int addr_len, bytes_read;         char recv_data[1024],send_data[1024];         struct sockaddr_in server_addr , client_addr;         if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) == -1) {             perror("Socket");             exit(1);         }         server_addr.sin_family = AF_INET;         server_addr.sin...

[ASTERIK] configure: error: *** uuid support not found (this typically means the uuid development package is missing)

ISSUE: Build error on Asterik , when you want test webrtc feature :) checking for uuid_generate_random in -luuid... no checking for uuid_generate_random in -le2fs-uuid... no checking for uuid_generate_random... no configure: error: *** uuid support not found (this typically means the uuid development package is missing) Fix: This issue arises due to missing of UUID generator specified by rfc4122 . +Linux sudo apt-get install uuid-dev  @Unix yum -y install libuuid-devel Asterik comes with lots of helpful script available on - asterisk/contrib/scripts/ folder of your ASTERIK source. So just use the following command on UNIX console to run the asterik pre-requisite script. contrib/scripts/install_prereq install And you are done! configuring. Now -- Make Asterik.

Video Conferencing Project in Java Source Code

My video conferencing project was completed as 300 project for 3rd year.It is not totally completed. There is some bug in here.To solve these bugs and to help other students this project is open.It is first try to make a project open source in this way so it can be modified.Any kind of question against this project will be answered. As this project was created in 3rd year 1st semester and now i nearly completed my BSc. so there will be little description about this.I will try to describe every class and function later when i get the chance.   For more details and how to build and run go to this link   in github.com . Latest code and binary release : VideoConference-v1.1 Older code and binary release : VideoConference-v1.0 Fix: 1. When in Video chat the Text chat option hanged for good. ################################################################################# FEATURE #############################################################################...