Saturday, August 20, 2011

Web Crawler or Site grabber code in java with Page downloader

/**
 * @Web Crawler *
 *
 * @author sharma
 * @version 1.00 2010/7/16
 */

import java.io.BufferedInputStream;

import java.io.IOException;

import java.net.MalformedURLException;

import java.util.*;

import java.net.URLConnection;



import java.awt.BorderLayout;

import java.awt.Color;

import javax.swing.JFrame;

import javax.swing.JTabbedPane;

import javax.swing.JLabel;

import javax.swing.JPanel;

import javax.swing.JButton;

import javax.swing.SwingConstants;

import javax.swing.*;

import java.awt.*;

import java.awt.event.*;

import java.awt.Transparency;

import javax.swing.event.*;

import java.sql.*;

import java.util.*;

import java.net.URL;

import javax.imageio.ImageIO;

import java.lang.*;



import java.io.*;

import javax.swing.BoxLayout;



public class assign_3 extends JFrame {

   

    String name="";

    String str[] = new String[1000];

    int k;

    StringBuilder builder;



    JLabel Send_lebel = new JLabel();

    JTextField send_bit = new JTextField();

   

    JLabel Rec_lebel = new JLabel();

    JTextArea rec_bit = new JTextArea();

    JTextField err = new JTextField();

   

    JButton gen = new JButton();

    JButton sen = new JButton();

    JButton cal = new JButton();

   

    int a[] = new int[100];

    int b[] = new int[100];

    WindowListener l;



    public assign_3(){

       

       

        setTitle("ASSIGNMENT #3");

        setSize(600,453);

        setLocation(220,100);



        l = new WindowAdapter() {

            public void windowClosing(WindowEvent e) {

                setEnabled(true);

            }

        };

         setUndecorated(true);

        getRootPane().setWindowDecorationStyle(JRootPane.FRAME);

        setVisible(true);

        jbinit();

       

    }

   

   

       void jbinit(){

             this.setResizable(false);

             this.getContentPane().setLayout(null);

            

             Send_lebel.setText("ADDRESS:");

             Send_lebel.setBounds(new Rectangle(8, 18, 150, 42));

             send_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             send_bit.setText("");

             send_bit.setBounds(new Rectangle(18, 48, 156, 21));

            

             Rec_lebel.setText("LINKS:");

             Rec_lebel.setBounds(new Rectangle(350, 48, 132, 21));

             rec_bit.setFont(new java.awt.Font("Dialog", 1, 12));

             rec_bit.setText("");

             rec_bit.setBounds(new Rectangle(345,88,226,311));

            

             gen.setBounds(new Rectangle(48, 100, 88, 24));

             gen.setFont(new java.awt.Font("Dialog", 1, 12));

             gen.setBorder(BorderFactory.createRaisedBevelBorder());

             gen.setMnemonic('G');

             gen.setText("GO!");    

             gen.addActionListener(new gen_actionAdapter(this));

           

             sen.setBounds(new Rectangle(208, 100, 88, 24));

             sen.setFont(new java.awt.Font("Dialog", 1, 12));

             sen.setBorder(BorderFactory.createRaisedBevelBorder());

             sen.setMnemonic('S');

             sen.setText("DOWNLOAD");    

             sen.addActionListener(new sen_actionAdapter(this));

           

                    

                       

            

             this.getContentPane().add(Send_lebel, null);

             this.getContentPane().add(Rec_lebel, null);

             this.getContentPane().add(rec_bit, null);

             this.getContentPane().add(send_bit, null); 

             this.getContentPane().add(gen,null);

             this.getContentPane().add(sen,null);     

              



              



    }



    public static void main(String[] args) {

       assign_3 jmain = new assign_3();



       jmain.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);



    }

   

  

// all button action

void gen_actionPerformed(ActionEvent e)

{



 int head,tail,num;

 boolean flag=true; 

 head = tail = num= 0;

 Vector<String> vc = new Vector<String>();

 Map m1 = new HashMap();



 String pre ="";

          name = send_bit.getText().toLowerCase();

          vc.addElement(name);

              System.out.println(name); 

                  m1.put(name,true);

                  ++tail;

              while(flag)

              {

                  if(head == tail){break;}

                  name = (String)vc.elementAt(head);

                 

                  System.out.println(name);

                  ++head;

            try {

           

       

            URL url = new URL("http://"+name);

            URLConnection urlc = url.openConnection();

            String st ="";

            BufferedInputStream buffer = new BufferedInputStream(urlc.getInputStream());

           

            builder = new StringBuilder();

            int byteRead;

            while ((byteRead = buffer.read()) != -1)

            {

                

                 builder.append((char) byteRead);

                 st += (char)byteRead;

            }

            buffer.close();

            System.out.println("URL VALID");

                  try {

                      ++num;

                BufferedWriter out = new BufferedWriter(new FileWriter("D:\\html\\"+num+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"+ep); 

                    } 

          

           // finding URL in given Page

            int r = 0;

            int i,j,l,s;

           

            for(i=0;i<1000;i++)

                str[i] = "";

               

             k = 0;   

             for(i=0;i<st.length();i++)

             {

               r = l = 0;

               for(j=i;j<st.length();j++)

               {

                  if(st.charAt(j) == 'w' && r == 0)  

                      ++l;

             

                 

                  if(l == 3)

                  {

                    str[k]+="www."; 

                      for(s=j+2;;s++)

                      {

                        if(st.charAt(s) == ' ' || st.charAt(s) == '<' || st.charAt(s) == '"')break;   

                        str[k] +=  st.charAt(s);

                       

                      }

                     

                      k++;

                      break;

                  }

                  else if(st.charAt(j) != 'w'){ r= 1;break;}

                }       

             }

            

            

             for(i=0;i<k;i++)

             {

                if(m1.containsValue(str[i]) != true)

                {

                

                    pre += str[i];

                    vc.addElement(str[i]);

                    m1.put(str[i],true);

                    ++tail;

                     pre += "\n";

                     rec_bit.setText(str[i]+"\n");

                }

             }   

            // rec_bit.setText(pre);

            }

            catch (MalformedURLException ex) {

             pre += "Sorry ! URL ERROR";

             rec_bit.setText(pre);

           

             }

            catch (IOException ex) {

            pre += "Sorry ! There Is no Site Like that";

            rec_bit.setText(pre);   

           

        }

              }

           

           

}



void sen_actionPerformed(ActionEvent e)

{

            try {

                BufferedWriter out = new BufferedWriter(new FileWriter(name+".html"));

                out.write(builder.toString()); 

                out.close(); 

                } 

                catch (IOException ep) 

                    {

                        System.out.print("failed writing"); 

                    } 

}







}







// all button class



class gen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    gen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.gen_actionPerformed(e);

  }

}





class sen_actionAdapter implements java.awt.event.ActionListener {

assign_3 adaptee;



    sen_actionAdapter(assign_3 adaptee) {

    this.adaptee = adaptee;

  }

  public void actionPerformed(ActionEvent e) {

    adaptee.sen_actionPerformed(e);

  }

}




How to Generate and use the ssh key on Gerrit, github.io, gitlab, and bitbucket.

 Details can be found here -