Python-网页爬虫与Sqlite3

大司命

1. #!/usr/bin/python
   
2. # -*- coding: utf-8 -*-
   
3. 
   
4. import sqlite3
   
5. import requests
   
6. from bs4 import BeautifulSoup
   
7. from re import escape
   
8. 
   
9. if __name__ == '__main__':
   
10.     conn = sqlite3.connect('Python.db')
    
11.     c = conn.cursor()
    
12.     c.execute('''CREATE TABLE IF NOT EXISTS Python (
    
13.         Url VARCHAR,
    
14.         Title VARCHAR,
    
15.         Author VARCHAR
    
16.     )''')
    
17.     conn.commit()
    
18. 
    
19.     # --------------------Split Line--------------------
    
20.     headers = {
    
21.         "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
    
22.     }
    
23. 
    
24.     for i in range(1, 1046):
    
25.         url = "http://xxx/index_%s.html" % str(i)
    
26.         req = requests.get(url=url, headers=headers)
    
27.         req.encoding = "utf-8"
    
28.         html = BeautifulSoup(req.text, "lxml")
    
29. 
    
30.         # --------------------Split Line--------------------
    
31.         for div in html.find_all('div', class_='loop'):
    
32.             content_body = div.select('h2 > a')[0]
    
33.             content_infor = div.select('.content_infor > span:nth-child(3)')[0]
    
34. 
    
35.             # --------------------Split Line--------------------
    
36.             cursor = c.execute(
    
37.                 "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
    
38.             len = 0
    
39.             for row in cursor:
    
40.                 len = row[0]
    
41.             if len > 0:
    
42.                 continue
    
43. 
    
44.             # --------------------Split Line--------------------
    
45.             c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
    
46.                 "http://xxx" + content_body.get('href'),
    
47.                 escape(content_body.get('title').replace(""", """")),
    
48.                 content_infor.text.replace('xxx: ', '')))
    
49. 
    
50.         conn.commit()
    
51.         print("第%s页" % str(i))
    
52. 
    
53.     # --------------------Split Line--------------------
    
54.     conn.close()
    

复制代码

Python-网页爬虫与Sqlite3
https://bbs.266.la/forum.php?mod=viewthread&tid=540
(出处: 派生社区)