import requests

import pandas as pd

import re

import json

import time

import random

import math

from bs4 import BeautifulSoup

def get_fundcode():

'''

url = 'http://fund.eastmoney.com/js/fundcode_search.js'

r = requests.get(url)

cont = re.findall('var r = (.*])', r.text)[0] # 提取list

ls = json.loads(cont) # 将字符串个事的list转化为list格式

fundcode = pd.DataFrame(ls, columns=['fundcode', 'fundsx', 'name', 'category', 'fundpy']) # list转为DataFrame

fundcode = fundcode.loc[0:100, ['fundcode', 'name', 'category']]

#fundcode.to_csv('./fundcode.csv', index=False, encoding = 'gbk')

return fundcode

def get_fundjbgk():

fund_jbgk = []

fund_list = get_fundcode()

for i in fund_list['fundcode']:

jbgk_addr = f'http://fundf10.eastmoney.com/jbgk_{i}.html'

g = requests.get(jbgk_addr)

g.encoding = g.apparent_encoding

s = BeautifulSoup(g.text, 'html.parser')

table = s.find('table', {'class': 'info w790'})

temp_jbgk = []

for row in table.findAll('tr'):

for col in row.findAll('td'):

temp_jbgk.append(col.get_text())

fund_jbgk.append(temp_jbgk)

time.sleep(random.randint(1, 3))

df_jbgk = pd.DataFrame(fund_jbgk, columns=['基金全称', '基金简称', '基金代码', '基金类型', '发行日期', '成立日期/规模', '资产规模', '份额规模', '基金管理人', '基金托管人', '基金经理人', '成立来分红', '管理费率', '托管费率', '销售服务费率', '最高认购费率'])

df_jbgk.to_csv('./fund_info.csv', index=False, encoding = 'gbk')

def get_one_page(fundcode, pageIndex=1):

'''

url = 'http://api.fund.eastmoney.com/f10/lsjz'

cookie = 'EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; EMFUND5=null; EMFUND6=null; EMFUND7=null; EMFUND8=null; EMFUND0=null; EMFUND9=01-24 17:11:50@#$%u957F%u4FE1%u5229%u5E7F%u6DF7%u5408A@%23%24519961; st_pvi=27838598767214; st_si=11887649835514'

headers = {

'Cookie': cookie,

'Host': 'api.fund.eastmoney.com',

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',

'Referer': 'http://fundf10.eastmoney.com/jjjz_%s.html' % fundcode,

}

params = {

'callback': 'jQuery18307633215694564663_1548321266367',

'fundCode': fundcode,

'pageIndex': pageIndex,

'pageSize': 20,

}

try:

r = requests.get(url=url, headers=headers, params=params)

if r.status_code == 200:

return r.text

return None

except RequestException:

return None

def parse_one_page(html):

'''

if html is not None: # 判断内容是否为None

content = re.findall('\((.*?)\)', html)[0] # 提取网页文本内容中的数据部分

lsjz_list = json.loads(content)['Data']['LSJZList'] # 获取历史净值列表

total_count = json.loads(content)['TotalCount'] # 获取数据量

total_page = math.ceil(total_count / 20) #

lsjz = pd.DataFrame(lsjz_list)

info = {'lsjz': lsjz,

'total_page': total_page}

return info

return None

def main(fundcode):

'''

html = get_one_page(fundcode)

info = parse_one_page(html)

total_page = info['total_page']

lsjz = info['lsjz']

lsjz.to_csv('./%s_lsjz.csv' % fundcode, index=False, encoding = 'gbk') # 将基金历史净值以csv格式储存

page = 1

while page < total_page:

page += 1

print(lsjz)

html = get_one_page(fundcode, pageIndex=page)

info = parse_one_page(html)

if info is None:

break

lsjz = info['lsjz']

lsjz.to_csv('./%s_lsjz.csv' % fundcode, mode='a', index=False, header=False, encoding = 'gbk') # 追加存储

time.sleep(random.randint(3, 5))

if __name__=='__main__':

# 获取所有基金代码

get_fundjbgk()

# # fundcode = '519961'

# fundcodes = pd.read_csv('./fundcode.csv', converters={'fundcode': str})

# # 获取所有基金净值数据

# for fundcode in fundcodes['fundcode']:

# print(fundcode)

# main(fundcode)

# time.sleep(random.randint(5, 10))

proxies=['http://61.135.217.7:80']

import requests

import json

from time import time

from lxml import etree

import re

from random import sample

from time import sleep

np = 1

base = "http://fund.eastmoney.com/"

acount = 2

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4620.400 QQBrowser/9.7.13014.400'

}

url = "http://fund.eastmoney.com/data/rankhandler.aspx?op=ph&dt=kf&ft=all&rs=&gs=0&sc=zzf&st=desc&sd=2017-07-09&ed=2018-07-09&qdii=&tabSubtype=,,,,,&pi=1&pn=50&dx=1&v=0."

def IndexSpider(url, headers): # 爬取第一个页面信息

url = url + str(int(time())) # 这个主要是url最后一个值是v=数字，我就用时间戳来伪装了

rsp = requests.get(url, headers=headers).content

html = rsp.decode('utf-8')

url = url[:-10] # 请求完了之后减掉加的时间戳，方面爬取下一页的时候重复操作

return html

def ChangeUrl(url): # 改变url，pi代表的是页数，这样可以按顺序爬取相应页数的数据

global acount

url = url.replace("&pi=1", "&pi=" + str(acount))

acount = acount + 1

return url

def ChangeUrl_2(jijindaima): # 我要爬取相应基金点开之后的页面的数据，分析可知是对应基金代码前面加上域名，后面加上.html

global base

jijindaima = jijindaima.replace('\"', '')

url_2 = base + jijindaima + '.html'

return url_2

def DetailRequest(url): # 爬取点开那一页之后的数据

global np

url = url.replace('\"', '')

print(url)

print("正在爬取第{0}条记录".format(np))

np = np + 1

re_leixing = re.compile('基金类型(.*?)</a>')

re_jingli = re.compile('基金经理：<a href=(.*?)</a>')

re_chengliri = re.compile('<td><span class="letterSpace01">成 立 日</span>：(.*?)</td>')

rsp = requests.get(url, headers=headers).content

html = rsp.decode('utf-8')

leixing = re_leixing.findall(html)[0][-3:]

jingli = re_jingli.findall(html)[0][-2:]

chengliri = re_chengliri.findall(html)[0]

return jingli, leixing, chengliri

if __name__ == '__main__':

nw = 1

url2_detail = []

jijindaima_list = []

detail_url_list = []

with open('w.txt', 'a', encoding='utf-8') as f:

f.write("基金代码\t\t基金简称\t\t单位净值\t\t累计净值\t\t基金经理\t\t基金类型\t\t成立日\n")

for i in range(1, 32):

html = IndexSpider(url, headers=headers)

url = ChangeUrl(url)

right = html.find("]")

left = html.find("[")

html = html[left + 1:right]

lists = html.split("\",\"")

for list in lists:

l = list.split(",")

jijindaima_list.append(l[0])

for i in jijindaima_list:

detail_url_list.append(ChangeUrl_2(i))

for i in detail_url_list:

url2_detail.append(DetailRequest(i))

with open('w.txt', 'a', encoding='utf-8') as f:

for list, l2 in zip(lists, url2_detail):

l = list.split(",")

f.writelines(

l[0] + '\t\t' + l[1] + '\t\t' + l[4] + '\t\t' + l[5] + '\t\t' + l2[0] + '\t\t' + l2[1] + '\t\t' +

l2[2] + '\n')

print('正在写入第{0}条记录……'.format(nw))

nw = nw + 1

print("5秒后爬取下一页……")

sleep(5)