Programming a Spider in Java 源码帖

1Programming a Spider in Java 源码帖
2Listing 1: Finding the bad links (CheckLinks.java)
3import java.awt.*;
4import javax.swing.*;
5import java.net.*;
6import java.io.*;
7/**
8* This example uses a Java spider to scan a Web site
9* and check for broken links. Written by Jeff Heaton.
10* Jeff Heaton is the author of "Programming Spiders,
11* Bots, and Aggregators" by Sybex. Jeff can be contacted
12* through his Web site at http://www.jeffheaton.com.
13* 
14* @author Jeff Heaton(http://www.jeffheaton.com)
15* @version 1.0
16*/
17public class CheckLinks extends javax.swing.JFrame implements
18              Runnable,ISpiderReportable {
19   /**
20    * The constructor. Perform setup here.
21    */
22   public CheckLinks()
23   {
24     //{{INIT_CONTROLS
25     setTitle("Find Broken Links");
26     getContentPane().setLayout(null);
27     setSize(405,288);
28     setVisible(false);
29     label1.setText("Enter a URL:");
30     getContentPane().add(label1);
31     label1.setBounds(12,12,84,12);
32     begin.setText("Begin");
33     begin.setActionCommand("Begin");
34     getContentPane().add(begin);
35     begin.setBounds(12,36,84,24);
36     getContentPane().add(url);
37     url.setBounds(108,36,288,24);
38     errorScroll.setAutoscrolls(true);
39     errorScroll.setHorizontalScrollBarPolicy(javax.swing.
40                 ScrollPaneConstants.HORIZONTAL_SCROLLBAR_ALWAYS);
41     errorScroll.setVerticalScrollBarPolicy(javax.swing.
42                 ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS);
43     errorScroll.setOpaque(true);
44     getContentPane().add(errorScroll);
45     errorScroll.setBounds(12,120,384,156);
46     errors.setEditable(false);
47     errorScroll.getViewport().add(errors);
48     errors.setBounds(0,0,366,138);
49     current.setText("Currently Processing: ");
50     getContentPane().add(current);
51     current.setBounds(12,72,384,12);
52     goodLinksLabel.setText("Good Links: 0");
53     getContentPane().add(goodLinksLabel);
54     goodLinksLabel.setBounds(12,96,192,12);
55     badLinksLabel.setText("Bad Links: 0");
56     getContentPane().add(badLinksLabel);
57     badLinksLabel.setBounds(216,96,96,12);
58     //}}
59     //{{INIT_MENUS
60     //}}
61     //{{REGISTER_LISTENERS
62     SymAction lSymAction = new SymAction();
63     begin.addActionListener(lSymAction);
64     //}}
65   }
66   /**
67    * Main method for the application
68    * 
69    * @param args Not used
70    */
71   static public void main(String args[])
72   {
73     (new CheckLinks()).setVisible(true);
74   }
75   /**
76    * Add notifications.
77    */
78   public void addNotify()
79   {
80     // Record the size of the window prior to calling parent's
81     // addNotify.
82     Dimension size = getSize();
83     super.addNotify();
84     if ( frameSizeAdjusted )
85       return;
86     frameSizeAdjusted = true;
87     // Adjust size of frame according to the insets and menu bar
88     Insets insets = getInsets();
89     javax.swing.JMenuBar menuBar = getRootPane().getJMenuBar();
90     int menuBarHeight = 0;
91     if ( menuBar != null )
92       menuBarHeight = menuBar.getPreferredSize().height;
93     setSize(insets.left + insets.right + size.width, insets.top +
94                           insets.bottom + size.height + 
95                           menuBarHeight);
96   }
97   // Used by addNotify
98   boolean frameSizeAdjusted = false;
99   //{{DECLARE_CONTROLS
100   javax.swing.JLabel label1 = new javax.swing.JLabel();
101   /**
102    * The begin or cancel button
103    */
104   javax.swing.JButton begin = new javax.swing.JButton();
105   /**
106    * The URL being processed
107    */
108   javax.swing.JTextField url = new javax.swing.JTextField();
109   /**
110    * Scroll the errors.
111    */
112   javax.swing.JScrollPane errorScroll =
113         new javax.swing.JScrollPane();
114   /**
115    * A place to store the errors created
116    */
117   javax.swing.JTextArea errors = new javax.swing.JTextArea();
118   javax.swing.JLabel current = new javax.swing.JLabel();
119   javax.swing.JLabel goodLinksLabel = new javax.swing.JLabel();
120   javax.swing.JLabel badLinksLabel = new javax.swing.JLabel();
121   //}}
122   //{{DECLARE_MENUS
123   //}}
124   /**
125    * The background spider thread
126    */
127   protected Thread backgroundThread;
128   /**
129    * The spider object being used
130    */
131   protected Spider spider;
132   /**
133    * The URL that the spider began with
134    */
135   protected URL base;
136   /**
137    * How many bad links have been found
138    */
139   protected int badLinksCount = 0;
140   /**
141    * How many good links have been found
142    */
143   protected int goodLinksCount = 0;
144
145   /**
146    * Internal class used to dispatch events
147    * 
148    * @author Jeff Heaton
149    * @version 1.0
150    */
151   class SymAction implements java.awt.event.ActionListener {
152     public void actionPerformed(java.awt.event.ActionEvent event)
153     {
154       Object object = event.getSource();
155       if ( object == begin )
156         begin_actionPerformed(event);
157     }
158   }
159   /**
160    * Called when the begin or cancel buttons are clicked
161    * 
162    * @param event The event associated with the button.
163    */
164   void begin_actionPerformed(java.awt.event.ActionEvent event)
165   {
166     if ( backgroundThread==null ) {
167       begin.setLabel("Cancel");
168       backgroundThread = new Thread(this);
169       backgroundThread.start();
170       goodLinksCount=0;
171       badLinksCount=0;
172     } else {
173       spider.cancel();
174     }
175   }
176   /**
177    * Perform the background thread operation. This method
178    * actually starts the background thread.
179    */
180   public void run()
181   {
182     try {
183       errors.setText("");
184       spider = new Spider(this);
185       spider.clear();
186       base = new URL(url.getText());
187       spider.addURL(base);
188       spider.begin();
189       Runnable doLater = new Runnable()
190       {
191         public void run()
192         {
193           begin.setText("Begin");
194         }
195       };
196       SwingUtilities.invokeLater(doLater);
197       backgroundThread=null;
198     } catch ( MalformedURLException e ) {
199       UpdateErrors err = new UpdateErrors();
200       err.msg = "Bad address.";
201       SwingUtilities.invokeLater(err);
202     }
203   }
204   /**
205    * Called by the spider when a URL is found. It is here
206    * that links are validated.
207    * 
208    * @param base The page that the link was found on.
209    * @param url The actual link address.
210    */
211   public boolean spiderFoundURL(URL base,URL url)
212   {
213     UpdateCurrentStats cs = new UpdateCurrentStats();
214     cs.msg = url.toString();
215     SwingUtilities.invokeLater(cs);
216     if ( !checkLink(url) ) {
217       UpdateErrors err = new UpdateErrors();
218       err.msg = url+"(on page " + base + ")\n";
219       SwingUtilities.invokeLater(err);
220       badLinksCount++;
221       return false;
222     }
223     goodLinksCount++;
224     if ( !url.getHost().equalsIgnoreCase(base.getHost()) )
225       return false;
226     else
227       return true;
228   }
229   /**
230    * Called when a URL error is found
231    * 
232    * @param url The URL that resulted in an error.
233    */
234   public void spiderURLError(URL url)
235   {
236   }
237   /**
238    * Called internally to check whether a link is good
239    * 
240    * @param url The link that is being checked.
241    * @return True if the link was good, false otherwise.
242    */
243   protected boolean checkLink(URL url)
244   {
245     try {
246       URLConnection connection = url.openConnection();
247       connection.connect();
248       return true;
249     } catch ( IOException e ) {
250       return false;
251     }
252   }
253   /**
254    * Called when the spider finds an e-mail address
255    * 
256    * @param email The email address the spider found.
257    */
258   public void spiderFoundEMail(String email)
259   {
260   }
261   /**
262    * Internal class used to update the error information
263    * in a Thread-Safe way
264    * 
265    * @author Jeff Heaton
266    * @version 1.0
267    */
268   class UpdateErrors implements Runnable {
269     public String msg;
270     public void run()
271     {
272       errors.append(msg);
273     }
274   }
275   /**
276    * Used to update the current status information
277    * in a "Thread-Safe" way
278    * 
279    * @author Jeff Heaton
280    * @version 1.0
281    */
282   class UpdateCurrentStats implements Runnable {
283     public String msg;
284     public void run()
285     {
286       current.setText("Currently Processing: " + msg );
287       goodLinksLabel.setText("Good Links: " + goodLinksCount);
288       badLinksLabel.setText("Bad Links: " + badLinksCount);
289     }
290   }
291}
292Listing 2: Reporting spider events(ISpiderReportable.java)
293import java.net.*;
294interface ISpiderReportable {
295   public boolean spiderFoundURL(URL base,URL url);
296   public void spiderURLError(URL url);
297   public void spiderFoundEMail(String email);
298}
299Listing 3: A reusable spider (Spider.java)
300import java.util.*;
301import java.net.*;
302import java.io.*;
303import javax.swing.text.*;
304import javax.swing.text.html.*;
305/**
306* That class implements a reusable spider
307* 
308* @author Jeff Heaton(http://www.jeffheaton.com)
309* @version 1.0
310*/
311public class Spider {
312   /**
313    * A collection of URLs that resulted in an error
314    */
315   protected Collection workloadError = new ArrayList(3);
316   /**
317    * A collection of URLs that are waiting to be processed
318    */
319   protected Collection workloadWaiting = new ArrayList(3);
320   /**
321    * A collection of URLs that were processed
322    */
323   protected Collection workloadProcessed = new ArrayList(3);
324   /**
325    * The class that the spider should report its URLs to
326    */
327   protected ISpiderReportable report;
328   /**
329    * A flag that indicates whether this process
330    * should be canceled
331    */
332   protected boolean cancel = false;
333   /**
334    * The constructor
335    * 
336    * @param report A class that implements the ISpiderReportable
337    * interface, that will receive information that the
338    * spider finds.
339    */
340   public Spider(ISpiderReportable report)
341   {
342     this.report = report;
343   }
344   /**
345    * Get the URLs that resulted in an error.
346    * 
347    * @return A collection of URL's.
348    */
349   public Collection getWorkloadError()
350   {
351     return workloadError;
352   }
353   /**
354    * Get the URLs that were waiting to be processed.
355    * You should add one URL to this collection to
356    * begin the spider.
357    * 
358    * @return A collection of URLs.
359    */
360   public Collection getWorkloadWaiting()
361   {
362     return workloadWaiting;
363   }
364   /**
365    * Get the URLs that were processed by this spider.
366    * 
367    * @return A collection of URLs.
368    */
369   public Collection getWorkloadProcessed()
370   {
371     return workloadProcessed;
372   }    
373   /**
374    * Clear all of the workloads.
375    */
376   public void clear()
377   {
378     getWorkloadError().clear();
379     getWorkloadWaiting().clear();
380     getWorkloadProcessed().clear();
381   }
382   /**
383    * Set a flag that will cause the begin
384    * method to return before it is done.
385    */
386   public void cancel()
387   {
388     cancel = true;
389   }
390   /**
391    * Add a URL for processing.
392    * 
393    * @param url
394    */
395   public void addURL(URL url)
396   {
397     if ( getWorkloadWaiting().contains(url) )
398       return;
399     if ( getWorkloadError().contains(url) )
400       return;
401     if ( getWorkloadProcessed().contains(url) )
402       return;
403     log("Adding to workload: " + url );
404     getWorkloadWaiting().add(url);
405   }
406   /**
407    * Called internally to process a URL
408    * 
409    * @param url The URL to be processed.
410    */
411   public void processURL(URL url)
412   {
413     try {
414       log("Processing: " + url );
415       // get the URL's contents
416       URLConnection connection = url.openConnection();
417       if ( (connection.getContentType()!=null) &&
418            !connection.getContentType().toLowerCase().s
419                        tartsWith("text/") ) {
420         getWorkloadWaiting().remove(url);
421         getWorkloadProcessed().add(url);
422         log("Not processing because content type is: " +
423              connection.getContentType() );
424         return;
425       }
426      
427       // read the URL
428       InputStream is = connection.getInputStream();
429       Reader r = new InputStreamReader(is);
430       // parse the URL
431       HTMLEditorKit.Parser parse = new HTMLParse().getParser();
432       parse.parse(r,new Parser(url),true);
433     } catch ( IOException e ) {
434       getWorkloadWaiting().remove(url);
435       getWorkloadError().add(url);
436       log("Error: " + url );
437       report.spiderURLError(url);
438       return;
439     }
440     // mark URL as complete
441     getWorkloadWaiting().remove(url);
442     getWorkloadProcessed().add(url);
443     log("Complete: " + url );
444   }
445   /**
446    * Called to start the spider
447    */
448   public void begin()
449   {
450     cancel = false;
451     while ( !getWorkloadWaiting().isEmpty() && !cancel ) {
452       Object list[] = getWorkloadWaiting().toArray();
453       for ( int i=0;(i<list.length)&&!cancel;i++ )
454         processURL((URL)list[i]);
455     }
456   }
457/**
458* A HTML parser callback used by this class to detect links
459* 
460* @author Jeff Heaton
461* @version 1.0
462*/
463   protected class Parser
464   extends HTMLEditorKit.ParserCallback {
465     protected URL base;
466     public Parser(URL base)
467     {
468       this.base = base;
469     }
470     public void handleSimpleTag(HTML.Tag t,
471                                 MutableAttributeSet a,int pos)
472     {
473       String href = (String)a.getAttribute(HTML.Attribute.HREF);
474      
475       if( (href==null) && (t==HTML.Tag.FRAME) )
476         href = (String)a.getAttribute(HTML.Attribute.SRC);
477        
478       if ( href==null )
479         return;
480       int i = href.indexOf('#');
481       if ( i!=-1 )
482         href = href.substring(0,i);
483       if ( href.toLowerCase().startsWith("mailt") ) {
484         report.spiderFoundEMail(href);
485         return;
486       }
487       handleLink(base,href);
488     }
489     public void handleStartTag(HTML.Tag t,
490                                MutableAttributeSet a,int pos)
491     {
492       handleSimpleTag(t,a,pos);     // handle the same way
493     }
494     protected void handleLink(URL base,String str)
495     {
496       try {
497         URL url = new URL(base,str);
498         if ( report.spiderFoundURL(base,url) )
499           addURL(url);
500       } catch ( MalformedURLException e ) {
501         log("Found malformed URL: " + str );
502       }
503     }
504   }
505   /**
506    * Called internally to log information
507    * This basic method just writes the log
508    * out to the stdout.
509    * 
510    * @param entry The information to be written to the log.
511    */
512   public void log(String entry)
513   {
514     System.out.println( (new Date()) + ":" + entry );
515   }
516}
517Listing 4: Parsing HTML (HTMLParse.java)
518import javax.swing.text.html.*;
519public class HTMLParse extends HTMLEditorKit {
520   public HTMLEditorKit.Parser getParser()
521   {
522     return super.getParser();
523   }
524}