From 4d481aabfbf3b472814dd59775b80f0e0525fee3 Mon Sep 17 00:00:00 2001 From: Dmitry Maylarov Date: Mon, 31 Jan 2022 16:21:35 +0300 Subject: [PATCH] wip add cache --- compare.py | 16 +++++++-- poetry.lock | 93 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/compare.py b/compare.py index a9682b5..fffd824 100644 --- a/compare.py +++ b/compare.py @@ -3,14 +3,24 @@ import math import numpy as np import pandas as pd import matplotlib.pyplot as plt -from pandas_datareader import data as pdr import backtrader as bt import strategies as st # import data def get_data(stocks, start, end): - stockData = pdr.get_data_yahoo(stocks, start, end) + from pandas_datareader.yahoo.headers import DEFAULT_HEADERS + from pandas_datareader import data as pdr + import requests_cache + + expire_after = datetime.timedelta(days=1) + session = requests_cache.CachedSession( + cache_name="cache", backend="sqlite", expire_after=expire_after + ) + session.headers = DEFAULT_HEADERS + start = datetime.datetime.fromisoformat("1900-01-01") + end = datetime.datetime.now() + stockData = pdr.get_data_yahoo(stocks, start, end, session=session) return stockData @@ -69,7 +79,7 @@ if __name__ == "__main__": # startDate = datetime.datetime.fromisoformat("2000-01-01") # endDate = datetime.datetime.fromisoformat("2022-01-01") print(stockList[0]) - for period_years in (1, 2, 5): + for period_years in (20, 50): endDate = datetime.datetime.now() startDate = endDate - datetime.timedelta(days=period_years * 365) stockData = get_data(stockList[0], startDate, endDate) diff --git a/poetry.lock b/poetry.lock index 54a3530..13aafdb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,25 @@ +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "attrs" +version = "21.4.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] + [[package]] name = "backtrader" version = "1.9.76.123" @@ -37,6 +59,17 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "cattrs" +version = "1.10.0" +description = "Composable complex class support for attrs and dataclasses." +category = "main" +optional = false +python-versions = ">=3.7,<4.0" + +[package.dependencies] +attrs = ">=20" + [[package]] name = "certifi" version = "2021.10.8" @@ -345,6 +378,33 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +[[package]] +name = "requests-cache" +version = "0.9.1" +description = "A transparent persistent cache for the requests library" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" + +[package.dependencies] +appdirs = ">=1.4.4,<2.0.0" +attrs = ">=21.2,<22.0" +cattrs = ">=1.8,<2.0" +requests = ">=2.22,<3.0" +url-normalize = ">=1.4,<2.0" +urllib3 = ">=1.25.5,<2.0.0" + +[package.extras] +dynamodb = ["boto3 (>=1.15,<2.0)", "botocore (>=1.18,<2.0)"] +all = ["boto3 (>=1.15,<2.0)", "botocore (>=1.18,<2.0)", "pymongo (>=3,<5)", "redis (>=3,<5)", "itsdangerous (>=2.0,<3.0)", "pyyaml (>=5.4)", "ujson (>=4.0)"] +mongodb = ["pymongo (>=3,<5)"] +redis = ["redis (>=3,<5)"] +bson = ["bson (>=0.5)"] +security = ["itsdangerous (>=2.0,<3.0)"] +yaml = ["pyyaml (>=5.4)"] +json = ["ujson (>=4.0)"] +docs = ["furo (>=2021.9.8)", "linkify-it-py (>=1.0.1,<2.0.0)", "myst-parser (>=0.15.1,<0.16.0)", "sphinx (==4.3.0)", "sphinx-autodoc-typehints (>=1.11,<2.0)", "sphinx-automodapi (>=0.13,<0.15)", "sphinx-copybutton (>=0.3,<0.5)", "sphinx-inline-tabs (>=2022.1.2b11)", "sphinx-notfound-page", "sphinx-panels (>=0.6,<0.7)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + [[package]] name = "setuptools-scm" version = "6.4.2" @@ -377,6 +437,17 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +six = "*" + [[package]] name = "urllib3" version = "1.26.8" @@ -393,9 +464,17 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "5c046d6151a6f00e7232d2f03f01f263b272bd933555657ee3b7db084d0c19d2" +content-hash = "af9bd6a522875cfde52f5013c7482651fcb72ed606769bd0d05854e98e963136" [metadata.files] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] backtrader = [] black = [ {file = "black-22.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1297c63b9e1b96a3d0da2d85d11cd9bf8664251fd69ddac068b98dc4f34f73b6"}, @@ -422,6 +501,10 @@ black = [ {file = "black-22.1.0-py3-none-any.whl", hash = "sha256:3524739d76b6b3ed1132422bf9d82123cd1705086723bc3e235ca39fd21c667d"}, {file = "black-22.1.0.tar.gz", hash = "sha256:a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5"}, ] +cattrs = [ + {file = "cattrs-1.10.0-py3-none-any.whl", hash = "sha256:35dd9063244263e63bd0bd24ea61e3015b00272cead084b2c40d788b0f857c46"}, + {file = "cattrs-1.10.0.tar.gz", hash = "sha256:211800f725cdecedcbcf4c753bbd22d248312b37d130f06045434acb7d9b34e1"}, +] certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, @@ -810,6 +893,10 @@ requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] +requests-cache = [ + {file = "requests-cache-0.9.1.tar.gz", hash = "sha256:7737f83f0f48481a904bb9a9402233db5090931e46f9644c502646a573848d35"}, + {file = "requests_cache-0.9.1-py3-none-any.whl", hash = "sha256:3e3384c48dca231ee4c49e7ba53162bc0d99e16721baf085bfba1552d065d151"}, +] setuptools-scm = [ {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, @@ -822,6 +909,10 @@ tomli = [ {file = "tomli-2.0.0-py3-none-any.whl", hash = "sha256:b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224"}, {file = "tomli-2.0.0.tar.gz", hash = "sha256:c292c34f58502a1eb2bbb9f5bbc9a5ebc37bee10ffb8c2d6bbdfa8eb13cc14e1"}, ] +url-normalize = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] urllib3 = [ {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, diff --git a/pyproject.toml b/pyproject.toml index 3c18b68..9d83685 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ pandas = "^1.4.0" matplotlib = "^3.5.1" pandas-datareader = "^0.10.0" backtrader = { git = "https://github.com/mementum/backtrader"} +requests-cache = "^0.9.1" [tool.poetry.dev-dependencies] neovim = "^0.3.1"