# -*- coding: utf-8 -*-
# Created : 2018/8/26 18:33
# author ：GuoLi
 
import requests
import json
import time
from lxml import etree
import html
import re
from bs4 import BeautifulSoup
 
 
class Weibospider:
  def __init__(self):
    # 获取首页的相关信息：
    self.start_url = 'https://weibo.com/u/5644764907"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
      "accept-encoding": "gzip, deflate, br",
      "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
      "cache-control": "max-age=0",
      "cookie": 使用自己本机的cookie,
      "referer": "https://www.weibo.com/u/5644764907",
      "upgrade-insecure-requests": "1",
      "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
    }
    self.proxy = {
      'HTTP': 'HTTP://180.125.70.78:9999',
      'HTTP': 'HTTP://117.90.4.230:9999',
      'HTTP': 'HTTP://111.77.196.229:9999',
      'HTTP': 'HTTP://111.177.183.57:9999',
      'HTTP': 'HTTP://123.55.98.146:9999',
    }
 
  def parse_home_url(self, url): # 处理解析首页面的详细信息（不包括两个通过ajax获取到的页面）
    res = requests.get(url, headers=self.headers)
    response = res.content.decode().replace("\\", "")
    # every_url = re.compile('target="_blank" href="(/\d+/\w+\" rel="external nofollow" ', re.S).findall(response)
    every_id = re.compile('name=(\d+)', re.S).findall(response) # 获取次级页面需要的id
    home_url = []
    for id in every_id:
      base_url = 'https://weibo.com/aj/v6/comment/big"//div[@class='list_li S_line1 clearfix']/div[@class='WB_face W_fl']/a/img/@alt") # 评论人的姓名
    info = html.xpath("//div[@node-type='replywrap']/div[@class='WB_text']/text()") # 评论信息
    info = "".join(info).replace(" ", "").split("\n")
    info.pop(0)
    comment_time = html.xpath("//div[@class='WB_from S_txt2']/text()") # 评论时间
    name_url = html.xpath("//div[@class='WB_face W_fl']/a/@href") # 评论人的url
    name_url = ["https:" + i for i in name_url]
    comment_info_list = []
    for i in range(len(name)):
      item = {}
      item["name"] = name[i] # 存储评论人的网名
      item["comment_info"] = info[i] # 存储评论的信息
      item["comment_time"] = comment_time[i] # 存储评论时间
      item["comment_url"] = name_url[i] # 存储评论人的相关主页
      comment_info_list.append(item)
    return count, comment_info_list
 
  def write_file(self, path_name, content_list):
    for content in content_list:
      with open(path_name, "a", encoding="UTF-8") as f:
        f.write(json.dumps(content, ensure_ascii=False))
        f.write("\n")
 
  def run(self):
    start_url = 'https://weibo.com/u/5644764907"第{}条微博相关评论.txt".format(i * 45 + j + 1)
        all_count, comment_info_list = self.parse_comment_info(all_url[j])
        self.write_file(path_name, comment_info_list)
        for num in range(1, 10000):
          if num * 15 < int(all_count) + 15:
            comment_url = all_url[j] + "&page={}".format(num + 1)
            print(comment_url)
            try:
              count, comment_info_list = self.parse_comment_info(comment_url)
              self.write_file(path_name, comment_info_list)
            except Exception as e:
              print("Error:", e)
              time.sleep(60)
              count, comment_info_list = self.parse_comment_info(comment_url)
              self.write_file(path_name, comment_info_list)
            del count
            time.sleep(0.2)
 
        print("第{}微博信息获取完成！".format(i * 45 + j + 1))
 
 
if __name__ == '__main__':
  weibo = Weibospider()
  weibo.run()