點(diǎn)擊(此處)折疊或打開

1. class WebView(webkit.WebView):
   
2.     def get_html(self):
   
3.         self.execute_script('oldtitle=document.title;document.title=document.documentElement.innerHTML;')
   
4.         html = self.get_main_frame().get_title()
   
5.         self.execute_script('document.title=oldtitle;')
   
6.         return html
   

點(diǎn)擊(此處)折疊或打開

1. web = WebView()
   
2. web.open(url)
   
3. html = web.get_html()
   
4. 
   
5. print html
   
6. print str(html)

點(diǎn)擊(此處)折疊或打開

1. #!/usr/bin/env python
   
2. import sys, threads # kudos to Nicholas Herriot (see comments)
   
3. import gtk
   
4. import webkit
   
5. import warnings
   
6. from time import sleep
   
7. from optparse import OptionParser
   
8.  
   
9. warnings.filterwarnings('ignore')
   
10.  
    
11. class WebView(webkit.WebView):
    
12.     def get_html(self):
    
13.         self.execute_script('oldtitle=document.title;document.title=document.documentElement.innerHTML;')
    
14.         html = self.get_main_frame().get_title()
    
15.         self.execute_script('document.title=oldtitle;')
    
16.         return html
    
17.  
    
18. class Crawler(gtk.Window):
    
19.     def __init__(self, url, file):
    
20.         gtk.gdk.threads_init() # suggested by Nicholas Herriot for Ubuntu Koala
    
21.         gtk.Window.__init__(self)
    
22.         self._url = url
    
23.         self._file = file
    
24.  
    
25.     def crawl(self):
    
26.         view = WebView()
    
27.         view.open(self._url)
    
28.         view.connect('load-finished', self._finished_loading)
    
29.         self.add(view)
    
30.         gtk.main()
    
31.  
    
32.     def _finished_loading(self, view, frame):
    
33.         with open(self._file, 'w') as f:
    
34.             f.write(view.get_html())
    
35.         gtk.main_quit()
    
36.  
    
37. def main():
    
38.     options = get_cmd_options()
    
39.     crawler = Crawler(options.url, options.file)
    
40.     crawler.crawl()
    
41.  
    
42. def get_cmd_options():
    
43.     """
    
44.         gets and validates the input from the command line
    
45.     """
    
46.     usage = "usage: %prog [options] args"
    
47.     parser = OptionParser(usage)
    
48.     parser.add_option('-u', '--url', dest = 'url', help = 'URL to fetch data from')
    
49.     parser.add_option('-f', '--file', dest = 'file', help = 'Local file path to save data to')
    
50.  
    
51.     (options,args) = parser.parse_args()
    
52.  
    
53.     if not options.url:
    
54.         print 'You must specify an URL.',sys.argv[0],'--help for more details'
    
55.         exit(1)
    
56.     if not options.file:
    
57.         print 'You must specify a destination file.',sys.argv[0],'--help for more details'
    
58.         exit(1)
    
59.  
    
60.     return options
    
61.  
    
62. if __name__ == '__main__':
    
63.     main()