Python-xpath

大司命

1. #!/usr/bin/python
   
2. # -*- coding: utf-8 -*-
   
3. 
   
4. import requests
   
5. from lxml import etree
   
6. import sqlite3
   
7. 
   
8. 
   
9. def write_sql(c, text):
   
10.     html = etree.HTML(text)
    
11.     # 标题
    
12.     titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
    
13.     # 链接
    
14.     hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')
    
15.     # 日期
    
16.     ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
    
17. 
    
18.     number = 0
    
19.     for title, href, em in zip(titles, hrefs, ems):
    
20.         href = host + href
    
21.         cursor = c.execute(
    
22.             "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
    
23.         res = c.fetchall()
    
24.         # 判断该字段是否已存在
    
25.         if res[0][0] > 0:
    
26.             continue
    
27. 
    
28.         c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
    
29.             href,
    
30.             title.replace(""", """"),
    
31.             em))
    
32.         number += 1
    
33.         print(title, href, em)
    
34. 
    
35.     conn.commit()
    
36.     return number > 0
    
37. 
    
38. 
    
39. if __name__ == '__main__':
    
40. 
    
41.     conn = sqlite3.connect("Python-xxx.db")
    
42.     c = conn.cursor()
    
43.     c.execute('''CREATE TABLE IF NOT EXISTS Python (
    
44.         Url VARCHAR,
    
45.         Title VARCHAR,
    
46.         Author VARCHAR
    
47.     )''')
    
48.     conn.commit()
    
49. 
    
50.     host = "https://xxx"
    
51.     url = host + "/xxx"
    
52.     req = requests.get(url)
    
53.     req.encoding = 'utf-8'
    
54.     # print(req.text)
    
55. 
    
56.     html = etree.HTML(req.text)
    
57.     clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
    
58.     hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
    
59.     # print(clearfix, href)
    
60. 
    
61.     for clearfix, href in zip(clearfixs, hrefs):
    
62.         print(clearfix, host + href)
    
63. 
    
64.         page = 1
    
65.         while True:
    
66.             url = host + href + "/list_%s.html" % page
    
67.             req = requests.get(url)
    
68.             req.encoding = 'utf-8'
    
69. 
    
70.             if (not write_sql(c, req.text)):
    
71.                 break
    
72. 
    
73.             print("第%s页" % page)
    
74.             page += 1
    
75. 
    
76.     conn.close()
    

复制代码

xpath用着就是舒服~

Python-xpath
https://bbs.266.la/forum.php?mod=viewthread&tid=959
(出处: 派生社区)

yjtnihaoma1

挖2024坟啊啊啊啊啊啊啊啊啊啊

hanlang

学习了，谢旋