Files
Quant_Code/5.课程代码/1.投资组合策略7天入门/原始代码/archive/[Lession3] Download Data.ipynb

637 lines
21 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#88\n",
"# 888\n",
"# 889\n",
"# 99, IF99"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Download data "
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime\n",
"from typing import List, Dict, Tuple\n",
"\n",
"from vnpy.trader.engine import BaseEngine, MainEngine, EventEngine\n",
"from vnpy.trader.constant import Interval, Exchange\n",
"from vnpy.trader.object import BarData, HistoryRequest\n",
"from vnpy.trader.database import database_manager\n",
"from vnpy.trader.rqdata import rqdata_client"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rqdata_client.init()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def download_data(symbol, exchange):\n",
" start = datetime(2006,1,1)\n",
" req = HistoryRequest(\n",
" symbol=symbol,\n",
" exchange=exchange,\n",
" interval=Interval.DAILY,\n",
" start=start,\n",
" end=datetime.now()\n",
" )\n",
"\n",
" data = rqdata_client.query_history(req)\n",
" database_manager.save_bar_data(data)\n",
" \n",
" print(\"finish! symbol=\",symbol, \"count=\", len(data))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"download_data(\"IC99\", Exchange.CFFEX)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"database_manager.save_bar_data(data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# GET ALL instuments"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>order_book_id</th>\n",
" <th>underlying_symbol</th>\n",
" <th>market_tplus</th>\n",
" <th>symbol</th>\n",
" <th>margin_rate</th>\n",
" <th>maturity_date</th>\n",
" <th>type</th>\n",
" <th>trading_code</th>\n",
" <th>exchange</th>\n",
" <th>product</th>\n",
" <th>contract_multiplier</th>\n",
" <th>round_lot</th>\n",
" <th>trading_hours</th>\n",
" <th>listed_date</th>\n",
" <th>industry_name</th>\n",
" <th>de_listed_date</th>\n",
" <th>underlying_order_book_id</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>A0303</td>\n",
" <td>A</td>\n",
" <td>0.0</td>\n",
" <td>豆一0303</td>\n",
" <td>0.05</td>\n",
" <td>2003-03-14</td>\n",
" <td>Future</td>\n",
" <td>a0303</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2002-03-15</td>\n",
" <td>油脂</td>\n",
" <td>2003-03-14</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>A0305</td>\n",
" <td>A</td>\n",
" <td>0.0</td>\n",
" <td>豆一0305</td>\n",
" <td>0.05</td>\n",
" <td>2003-05-23</td>\n",
" <td>Future</td>\n",
" <td>a0305</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2002-03-15</td>\n",
" <td>油脂</td>\n",
" <td>2003-05-23</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>A0307</td>\n",
" <td>A</td>\n",
" <td>0.0</td>\n",
" <td>豆一0307</td>\n",
" <td>0.05</td>\n",
" <td>2003-07-14</td>\n",
" <td>Future</td>\n",
" <td>a0307</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2002-03-15</td>\n",
" <td>油脂</td>\n",
" <td>2003-07-14</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>A0309</td>\n",
" <td>A</td>\n",
" <td>0.0</td>\n",
" <td>豆一0309</td>\n",
" <td>0.05</td>\n",
" <td>2003-09-12</td>\n",
" <td>Future</td>\n",
" <td>a0309</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2002-05-22</td>\n",
" <td>油脂</td>\n",
" <td>2003-09-12</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>A0311</td>\n",
" <td>A</td>\n",
" <td>0.0</td>\n",
" <td>豆一0311</td>\n",
" <td>0.05</td>\n",
" <td>2003-11-14</td>\n",
" <td>Future</td>\n",
" <td>a0311</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2002-05-22</td>\n",
" <td>油脂</td>\n",
" <td>2003-11-14</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6609</th>\n",
" <td>PB2201</td>\n",
" <td>PB</td>\n",
" <td>0.0</td>\n",
" <td>铅2201</td>\n",
" <td>0.08</td>\n",
" <td>2022-01-17</td>\n",
" <td>Future</td>\n",
" <td>pb2201</td>\n",
" <td>SHFE</td>\n",
" <td>Commodity</td>\n",
" <td>5.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-01:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2021-01-18</td>\n",
" <td>有色</td>\n",
" <td>2022-01-17</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6610</th>\n",
" <td>BC2201</td>\n",
" <td>BC</td>\n",
" <td>0.0</td>\n",
" <td>国际铜2201</td>\n",
" <td>0.08</td>\n",
" <td>2022-01-17</td>\n",
" <td>Future</td>\n",
" <td>bc2201</td>\n",
" <td>INE</td>\n",
" <td>Commodity</td>\n",
" <td>5.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-01:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2021-01-18</td>\n",
" <td>未知</td>\n",
" <td>2022-01-17</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6611</th>\n",
" <td>BB2201</td>\n",
" <td>BB</td>\n",
" <td>0.0</td>\n",
" <td>细木工板2201</td>\n",
" <td>0.40</td>\n",
" <td>2022-01-17</td>\n",
" <td>Future</td>\n",
" <td>bb2201</td>\n",
" <td>DCE</td>\n",
" <td>Commodity</td>\n",
" <td>500.0</td>\n",
" <td>1.0</td>\n",
" <td>09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2021-01-18</td>\n",
" <td>建材</td>\n",
" <td>2022-01-17</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6612</th>\n",
" <td>SR2201</td>\n",
" <td>SR</td>\n",
" <td>0.0</td>\n",
" <td>白砂糖2201</td>\n",
" <td>0.07</td>\n",
" <td>2022-01-14</td>\n",
" <td>Future</td>\n",
" <td>SR201</td>\n",
" <td>CZCE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2021-01-18</td>\n",
" <td>软商品</td>\n",
" <td>2022-01-14</td>\n",
" <td>null</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6613</th>\n",
" <td>WR2201</td>\n",
" <td>WR</td>\n",
" <td>0.0</td>\n",
" <td>线材2201</td>\n",
" <td>0.07</td>\n",
" <td>2022-01-17</td>\n",
" <td>Future</td>\n",
" <td>wr2201</td>\n",
" <td>SHFE</td>\n",
" <td>Commodity</td>\n",
" <td>10.0</td>\n",
" <td>1.0</td>\n",
" <td>09:01-10:15,10:31-11:30,13:31-15:00</td>\n",
" <td>2021-01-18</td>\n",
" <td>焦煤钢矿</td>\n",
" <td>2022-01-17</td>\n",
" <td>null</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>6614 rows × 17 columns</p>\n",
"</div>"
],
"text/plain": [
" order_book_id underlying_symbol market_tplus symbol margin_rate \\\n",
"0 A0303 A 0.0 豆一0303 0.05 \n",
"1 A0305 A 0.0 豆一0305 0.05 \n",
"2 A0307 A 0.0 豆一0307 0.05 \n",
"3 A0309 A 0.0 豆一0309 0.05 \n",
"4 A0311 A 0.0 豆一0311 0.05 \n",
"... ... ... ... ... ... \n",
"6609 PB2201 PB 0.0 铅2201 0.08 \n",
"6610 BC2201 BC 0.0 国际铜2201 0.08 \n",
"6611 BB2201 BB 0.0 细木工板2201 0.40 \n",
"6612 SR2201 SR 0.0 白砂糖2201 0.07 \n",
"6613 WR2201 WR 0.0 线材2201 0.07 \n",
"\n",
" maturity_date type trading_code exchange product \\\n",
"0 2003-03-14 Future a0303 DCE Commodity \n",
"1 2003-05-23 Future a0305 DCE Commodity \n",
"2 2003-07-14 Future a0307 DCE Commodity \n",
"3 2003-09-12 Future a0309 DCE Commodity \n",
"4 2003-11-14 Future a0311 DCE Commodity \n",
"... ... ... ... ... ... \n",
"6609 2022-01-17 Future pb2201 SHFE Commodity \n",
"6610 2022-01-17 Future bc2201 INE Commodity \n",
"6611 2022-01-17 Future bb2201 DCE Commodity \n",
"6612 2022-01-14 Future SR201 CZCE Commodity \n",
"6613 2022-01-17 Future wr2201 SHFE Commodity \n",
"\n",
" contract_multiplier round_lot \\\n",
"0 10.0 1.0 \n",
"1 10.0 1.0 \n",
"2 10.0 1.0 \n",
"3 10.0 1.0 \n",
"4 10.0 1.0 \n",
"... ... ... \n",
"6609 5.0 1.0 \n",
"6610 5.0 1.0 \n",
"6611 500.0 1.0 \n",
"6612 10.0 1.0 \n",
"6613 10.0 1.0 \n",
"\n",
" trading_hours listed_date \\\n",
"0 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2002-03-15 \n",
"1 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2002-03-15 \n",
"2 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2002-03-15 \n",
"3 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2002-05-22 \n",
"4 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2002-05-22 \n",
"... ... ... \n",
"6609 21:01-01:00,09:01-10:15,10:31-11:30,13:31-15:00 2021-01-18 \n",
"6610 21:01-01:00,09:01-10:15,10:31-11:30,13:31-15:00 2021-01-18 \n",
"6611 09:01-10:15,10:31-11:30,13:31-15:00 2021-01-18 \n",
"6612 21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00 2021-01-18 \n",
"6613 09:01-10:15,10:31-11:30,13:31-15:00 2021-01-18 \n",
"\n",
" industry_name de_listed_date underlying_order_book_id \n",
"0 油脂 2003-03-14 null \n",
"1 油脂 2003-05-23 null \n",
"2 油脂 2003-07-14 null \n",
"3 油脂 2003-09-12 null \n",
"4 油脂 2003-11-14 null \n",
"... ... ... ... \n",
"6609 有色 2022-01-17 null \n",
"6610 未知 2022-01-17 null \n",
"6611 建材 2022-01-17 null \n",
"6612 软商品 2022-01-14 null \n",
"6613 焦煤钢矿 2022-01-17 null \n",
"\n",
"[6614 rows x 17 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df=rqdata_client.get_all_futures()\n",
"df\n",
"# underlying_symbol\n",
"# exchange\n",
"# listed_date"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"data_dict = df.transpose().to_dict()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'order_book_id': 'A0303',\n",
" 'underlying_symbol': 'A',\n",
" 'market_tplus': 0.0,\n",
" 'symbol': '豆一0303',\n",
" 'margin_rate': 0.05,\n",
" 'maturity_date': '2003-03-14',\n",
" 'type': 'Future',\n",
" 'trading_code': 'a0303',\n",
" 'exchange': 'DCE',\n",
" 'product': 'Commodity',\n",
" 'contract_multiplier': 10.0,\n",
" 'round_lot': 1.0,\n",
" 'trading_hours': '21:01-23:00,09:01-10:15,10:31-11:30,13:31-15:00',\n",
" 'listed_date': '2002-03-15',\n",
" 'industry_name': '油脂',\n",
" 'de_listed_date': '2003-03-14',\n",
" 'underlying_order_book_id': 'null'}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_dict[0]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"underlying_symbols = []\n",
"vt_symbols = [] # IF99.CFFEX\n",
"for data in data_dict.values():\n",
" # 1) Filter underlying symbol \n",
" underlying = data['underlying_symbol']\n",
" if underlying not in underlying_symbols:\n",
" underlying_symbols.append(underlying)\n",
" \n",
" # 2) Filter listed date\n",
" dt_str = data[\"listed_date\"]\n",
" dt = datetime.strptime(dt_str, \"%Y-%m-%d\")\n",
" \n",
" if dt <= datetime(2016,1,1):\n",
" \n",
" vt_symbol = underlying + \"99\" + \".\" + data[\"exchange\"]\n",
" \n",
" vt_symbols.append(vt_symbol)\n",
" \n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"finish! symbol= A99 count= 3661\n",
"finish! symbol= AG99 count= 2121\n",
"finish! symbol= AL99 count= 3661\n",
"finish! symbol= AU99 count= 3173\n",
"finish! symbol= B99 count= 3661\n",
"finish! symbol= BB99 count= 1739\n",
"finish! symbol= BU99 count= 1781\n",
"finish! symbol= C99 count= 3661\n",
"finish! symbol= CF99 count= 3661\n",
"finish! symbol= CS99 count= 1485\n",
"finish! symbol= CU99 count= 3661\n",
"finish! symbol= ER99 count= 994\n",
"finish! symbol= FB99 count= 1739\n",
"finish! symbol= FG99 count= 1980\n",
"finish! symbol= FU99 count= 3650\n",
"finish! symbol= HC99 count= 1670\n",
"finish! symbol= I99 count= 1774\n",
"finish! symbol= IC99 count= 1409\n",
"finish! symbol= IF99 count= 2620\n",
"finish! symbol= IH99 count= 1409\n",
"finish! symbol= J99 count= 2380\n",
"finish! symbol= JD99 count= 1759\n",
"finish! symbol= JM99 count= 1909\n",
"finish! symbol= JR99 count= 1753\n",
"finish! symbol= L99 count= 3282\n",
"finish! symbol= LR99 count= 1597\n",
"finish! symbol= M99 count= 3661\n",
"finish! symbol= MA99 count= 1612\n",
"finish! symbol= ME99 count= 860\n",
"finish! symbol= NI99 count= 1422\n",
"finish! symbol= OI99 count= 2075\n",
"finish! symbol= P99 count= 3223\n",
"finish! symbol= PB99 count= 2394\n",
"finish! symbol= PM99 count= 2193\n",
"finish! symbol= PP99 count= 1685\n",
"finish! symbol= RB99 count= 2878\n",
"finish! symbol= RI99 count= 2069\n",
"finish! symbol= RM99 count= 1961\n",
"finish! symbol= RO99 count= 1444\n",
"finish! symbol= RS99 count= 1961\n",
"finish! symbol= RU99 count= 3661\n",
"finish! symbol= S99 count= 0\n",
"finish! symbol= SF99 count= 1574\n",
"finish! symbol= SM99 count= 1574\n",
"finish! symbol= SN99 count= 1422\n",
"finish! symbol= SR99 count= 3659\n",
"finish! symbol= T99 count= 1427\n",
"finish! symbol= TA99 count= 3430\n",
"finish! symbol= TC99 count= 617\n",
"finish! symbol= TF99 count= 1797\n",
"finish! symbol= V99 count= 2839\n",
"finish! symbol= WH99 count= 2069\n",
"finish! symbol= WR99 count= 2853\n",
"finish! symbol= WS99 count= 1792\n",
"finish! symbol= WT99 count= 1675\n",
"finish! symbol= Y99 count= 3658\n",
"finish! symbol= ZC99 count= 1388\n",
"finish! symbol= ZN99 count= 3368\n"
]
}
],
"source": [
"for vt_symbol in vt_symbols:\n",
" symbol, exchange =tuple(vt_symbol.split(\".\"))\n",
" download_data(symbol, Exchange(exchange))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}