[原创]上海证券交易所定期报告年报爬取Shell脚本

需要分析年报中的表格内容，所以需要收集财报数据，为了方便一键获取，故编写本脚本。

脚本内容

Download_Dir: 需要下载的目录，请确保有足够的权限创建，预计全部下载需要180GB存储空间

#!/bin/bash
# 上海证券交易所年报抓取
# Author: [email protected]
# Date: 2023-08-13
# Version: 0.0.1

# 最终要下载的目录
Download_Dir='/data/caibao'

Curl_Init(){
    curl -s -H 'Referer: http://www.sse.com.cn/' "${@}"
}

STOCK_TYPE_LIST=(1 2 8)
for STOCK_TYPE in ${STOCK_TYPE_LIST[@]};do
    # 查询股票代码 1 2 8
    SECURITY_CODE_List=`Curl_Init "http://query.sse.com.cn/sseQuery/commonQuery.do?jsonCallBack=jsonpCallback25180210&STOCK_TYPE=${STOCK_TYPE}&REG_PROVINCE=&CSRC_CODE=&STOCK_CODE=&sqlId=COMMON_SSE_CP_GPJCTPZ_GPLB_GP_L&COMPANY_STATUS=2%2C4%2C5%2C7%2C8&type=inParams&isPagination=true&pageHelp.cacheSize=1&pageHelp.beginPage=1&pageHelp.pageSize=3000&pageHelp.pageNo=1&pageHelp.endPage=1&_=1691554195145"|sed -r 's/^jsonpCallback[0-9]{1,99}\(|\)$//g'|jq -rc '.result[]|{"A_STOCK_CODE":.A_STOCK_CODE,"B_STOCK_CODE":.B_STOCK_CODE,"SEC_NAME_CN":.SEC_NAME_CN}'`
    IFS=$'\n'
    #echo "${SECURITY_CODE_List}"
    for SECURITY_Info in ${SECURITY_CODE_List};do
        SECURITY_CODE=`echo "${SECURITY_Info}"|jq -r .A_STOCK_CODE`
        SECURITY_NAME_CN=`echo "${SECURITY_Info}"|jq -r .SEC_NAME_CN`
        Query_Bulletin_List=`Curl_Init "http://query.sse.com.cn/security/stock/queryCompanyBulletin.do?jsonCallBack=jsonpCallback24653133&isPagination=true&pageHelp.pageSize=3000&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.cacheSize=1&pageHelp.endPage=1&productId=${SECURITY_CODE}&securityType=0101%2C120100%2C020100%2C020200%2C120200&reportType2=DQBG&reportType=ALL&beginDate=1990-01-01&endDate=2023-08-10&_=1691554852277" |sed -r 's/^jsonpCallback[0-9]{1,99}\(|\)$//g'|jq -rc '.result[]'`
        for Query_Bulletin in ${Query_Bulletin_List};do
            # 证券简称
            Bulletin_SECURITY_NAME=`echo "${Query_Bulletin}"|jq -r '.SECURITY_NAME'`
            # 公告标题
            Bulletin_TITLE=`echo "${Query_Bulletin}"|jq -r '.TITLE'`
            # 文件下载路径
            Bulletin_URL=`echo "${Query_Bulletin}"|jq -r '.URL'`
            # PDF最终下载地址
            Bulletin_Download_Url="http://www.sse.com.cn${Bulletin_URL}"
            # 公告分类
            Bulletin_BULLETIN_TYPE=`echo "${Query_Bulletin}"|jq -r '.BULLETIN_TYPE'`
            # 证券代码
            Bulletin_SECURITY_CODE=`echo "${Query_Bulletin}"|jq -r '.SECURITY_CODE'`
            # 年份
            Bulletin_BULLETIN_YEAR=`echo "${Query_Bulletin}"|jq -r '.BULLETIN_YEAR'`
            # 下载的文件路径及名称
            Bulletin_Download_Name="${Download_Dir}/${Bulletin_SECURITY_CODE}_${Bulletin_BULLETIN_YEAR}_${SECURITY_NAME_CN}_${Bulletin_BULLETIN_TYPE}_${Bulletin_TITLE}.pdf"
            if [ -z "${Bulletin_URL}" ];then
                echo "null ${Bulletin_BULLETIN_YEAR} ${SECURITY_NAME_CN} ${Bulletin_TITLE} ${Bulletin_Download_Url}"
            else
                echo "${Bulletin_BULLETIN_YEAR} ${SECURITY_NAME_CN} ${Bulletin_TITLE} ${Bulletin_Download_Url}"
            fi
            #exit
            if [ ! -d "${Download_Dir}" ];then
                mkdir -p "${Download_Dir}" ||exit 2
            elif [ -f "${Bulletin_Download_Name}" ];then
                continue
            fi
            wget -q -O "${Bulletin_Download_Name}" "${Bulletin_Download_Url}" &
        done
        wait
        sleep 3
    done
done

END

[原创]上海证券交易所定期报告年报爬取Shell脚本