1 __author__ = 'minmin' 2 #coding:utf-8 3 import re,urllib,sgmllib,os 4  5 #根据当前的url获取html 6 def getHtml(url): 7     page = urllib.urlopen(url) 8     html = page.read() 9     page.close()10     return html11 12 #根据html获取想要的文章内容13 def func(str):14      result =   re.findall(r"
     
      ([^<>]*)
      
",getHtml(url),re.M)15      artical =''16 17      for j in result:18          if len(j)<>0:19              j = j.replace("
      ","    ")20              j = j.replace("","    ")21              j = j.replace("
      
","   ")22              j = j.replace(" "," ")23              j = j.replace("“"," ")24              j = j.replace("”"," ")25              j = j.replace("·"," ")26              artical = artical + j + '\n'27      return  artical28 29 #html链接的标签是“a”，链接的属性是“href”，也就是要获得html中所有tag=a，attrs=href 值。30 class URLPaser(sgmllib.SGMLParser):31     def reset(self):32         sgmllib.SGMLParser.reset(self)33         self.urls = []34 35     def start_a(self,attrs):36         href = [v for k,v in attrs if k == 'href']37         if href:38             self.urls.extend(href)39 40 IParser = URLPaser()41 socket = urllib.urlopen("http://economy.china.com/internet/")#打开这个网页42 43 #fout = file('qq_art_urls.txt','w')#要把这个链接写到这个文件中44 IParser.feed(socket.read())#分析啦45 46 reg = 'http://economy.china.com/internet/.*'#这个是用来匹配符合条件的链接，使用正则表达式匹配47 48 pattern = re.compile(reg)49 50 os.getcwd()#获得当前文件夹路径51 os.path.sep#当前系统路径分隔符52 53 #判断文件是否存在54 if os.path.exists('china_news_Technology')==False:55      os.makedirs('china_news_Technology')56 i = 057 url2 = []58 for url in IParser.urls:#链接都存在urls里59     url = "http://economy.china.com"+url60     if pattern.match(url):61         if url not in url2:62             url2.append(url)63             url = url.replace(".html","_all.html#page_2")#由于一篇文章被分成几页，找到显示全页的那一个页面64             artical = func(url)65             if len(artical)<>0:66                   print artical67                   print url68                   i = i + 169                   f = open("china_news_Technology/"+str(i) + '.txt','a+')70                   f.write(artical)71                   f.close()