Results/Analysis¶
Models are trained using three types of training data:¶
$\textbf{LOB , } \text{using only } \textit{Limit Order Book}$
$\textbf{LIQ , } \text{using only } \textit{Liquidity Measures}$
$\textbf{LOB+LIQ , } \text{using } \textit{Limit Order Book} \textbf{ and } \textit{Liquidity Measures } $
The effect of the rolling window size on the performance is also investigated. The results are shown below.
In [1]:
from IPython.display import HTML
def hide_code(): return HTML('''<script>code_show=true; function code_toggle() {if (code_show){$('div.input').hide();} else {$('div.input').show();}code_show = !code_show} $( document ).ready(code_toggle);</script><form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')
hide_code()
Out[1]:
In [2]:
import pandas as pd
import numpy as np
from utils.df_handler import get_significant_figures
errors={f'{i}':[] for i in range(10,61,10)}
for window_size in range(10,61,10):
for error_type in ['se','ape']:
for data_type in [f'LOB_{window_size}',f'LIQ_{window_size}',f'LOB+LIQ_{window_size}']:
with open('./eval/errors/' + f'{data_type}.npy', 'rb') as input:
nn_errors = np.load(input,allow_pickle='TRUE').item()
with open('./eval/errors/' + f'LinReg_{data_type}.npy', 'rb') as input:
linreg_errors = np.load(input,allow_pickle='TRUE').item()
for model_errors in [nn_errors,linreg_errors]:
errors[f'{window_size}'].append(np.ravel([[i,k] for i,k in \
zip(np.mean(model_errors[error_type+'_train'],0),np.mean(model_errors[error_type+'_val'],0))]).tolist())
indices = pd.MultiIndex.from_tuples([(f'{l}',i,j,k) for l in range(10,61,10)
for i in ['MSE','MAPE (%)'] \
for j in ['LOB','LIQ','LOB+LIQ'] \
for k in ['NN','LinReg']])
columns = pd.MultiIndex.from_tuples([(i,k)
for i in ['mid price','bid price expectation','ask price expectation','bid price variance','ask price variance'] \
for k in ['train','val']])
errors_df = pd.DataFrame(np.concatenate([i for i in np.array([*errors.values()])],axis=0),index=indices,columns=columns)
Performance table¶
The training and validation performances of each model for each window size (10, 20, 30, 40, 50, 60) are shown below with $\textbf{Mean Squared Error (MSE)}$ and $\textbf{Mean Absolute Percentage Error (MAPE)}$ as the chosen performance criteria. Results obtained with linear regression $\textbf{(LinReg)}$ are also added for comparison. Each value is rounded with a relative error less than 1%.
In [3]:
%%html
<html>
<style>
</style>
<head>
</head>
<body>
<label for="windows">Select window size:</label>
<select name="windows" id="windows" style="width: 50px;">
<! -- <option value="NONE" width = "1000">None</option> -->
<option value="10" width = "1000" selected="selected">10</option>
<option value="20" width = "1000">20</option>
<option value="30" width = "1000">30</option>
<option value="40" width = "1000">40</option>
<option value="50" width = "1000">50</option>
<option value="60" width = "1000">60</option>
</select>
<h3 id="table_title"></h3>
<table id="10" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.697•E-4</td> <td>0.661•E-3</td> <td>0.677•E-4</td> <td>0.655•E-3</td> <td>0.664•E-4</td> <td>0.657•E-3</td> <td>0.986•E-9</td> <td>0.13•E-8</td> <td>0.969•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.998•E-2</td> <td>0.552•E-2</td> <td>0.997•E-2</td> <td>0.552•E-2</td> <td>0.997•E-2</td> <td>0.553•E-2</td> <td>0.583•E-9</td> <td>0.106•E-8</td> <td>0.58•E-9</td> <td>0.13•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.538•E-4</td> <td>0.74•E-3</td> <td>0.532•E-4</td> <td>0.749•E-3</td> <td>0.509•E-4</td> <td>0.707•E-3</td> <td>0.987•E-9</td> <td>0.131•E-8</td> <td>0.968•E-9</td> <td>0.149•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.399•E-2</td> <td>0.394•E-2</td> <td>0.398•E-2</td> <td>0.399•E-2</td> <td>0.398•E-2</td> <td>0.392•E-2</td> <td>0.825•E-9</td> <td>0.136•E-8</td> <td>0.803•E-9</td> <td>0.164•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.497•E-4</td> <td>0.654•E-3</td> <td>0.487•E-4</td> <td>0.648•E-3</td> <td>0.465•E-4</td> <td>0.648•E-3</td> <td>0.987•E-9</td> <td>0.13•E-8</td> <td>0.969•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.428•E-2</td> <td>0.487•E-2</td> <td>0.427•E-2</td> <td>0.488•E-2</td> <td>0.428•E-2</td> <td>0.486•E-2</td> <td>0.572•E-9</td> <td>0.105•E-8</td> <td>0.571•E-9</td> <td>0.128•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.649•E-1</td> <td>0.121</td> <td>0.635•E-1</td> <td>0.119</td> <td>0.628•E-1</td> <td>0.118</td> <td>14.86</td> <td>18.09</td> <td>14.84</td> <td>22.53</td> </tr> <tr> <th>LinReg</th> <td>0.941</td> <td>0.549</td> <td>0.944</td> <td>0.55</td> <td>0.939</td> <td>0.548</td> <td>10.42</td> <td>14.96</td> <td>10.51</td> <td>18.86</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.535•E-1</td> <td>0.136</td> <td>0.53•E-1</td> <td>0.139</td> <td>0.516•E-1</td> <td>0.134</td> <td>14.87</td> <td>18.2</td> <td>14.83</td> <td>22.82</td> </tr> <tr> <th>LinReg</th> <td>0.503</td> <td>0.432</td> <td>0.503</td> <td>0.434</td> <td>0.501</td> <td>0.431</td> <td>13.37</td> <td>18.36</td> <td>13.3</td> <td>21.62</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.527•E-1</td> <td>0.128</td> <td>0.519•E-1</td> <td>0.128</td> <td>0.504•E-1</td> <td>0.127</td> <td>14.86</td> <td>18.08</td> <td>14.84</td> <td>22.45</td> </tr> <tr> <th>LinReg</th> <td>0.517</td> <td>0.616</td> <td>0.518</td> <td>0.618</td> <td>0.515</td> <td>0.614</td> <td>10.34</td> <td>14.99</td> <td>10.45</td> <td>18.41</td> </tr> </tbody></table>
<table id="20" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.517•E-4</td> <td>0.389•E-3</td> <td>0.501•E-4</td> <td>0.392•E-3</td> <td>0.484•E-4</td> <td>0.389•E-3</td> <td>0.979•E-9</td> <td>0.128•E-8</td> <td>0.966•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.986•E-2</td> <td>0.538•E-2</td> <td>0.985•E-2</td> <td>0.537•E-2</td> <td>0.985•E-2</td> <td>0.537•E-2</td> <td>0.572•E-9</td> <td>0.105•E-8</td> <td>0.57•E-9</td> <td>0.129•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.529•E-4</td> <td>0.113•E-2</td> <td>0.519•E-4</td> <td>0.111•E-2</td> <td>0.498•E-4</td> <td>0.111•E-2</td> <td>0.98•E-9</td> <td>0.128•E-8</td> <td>0.965•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.369•E-2</td> <td>0.359•E-2</td> <td>0.368•E-2</td> <td>0.362•E-2</td> <td>0.369•E-2</td> <td>0.357•E-2</td> <td>0.815•E-9</td> <td>0.132•E-8</td> <td>0.795•E-9</td> <td>0.163•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.435•E-4</td> <td>0.112•E-2</td> <td>0.422•E-4</td> <td>0.104•E-2</td> <td>0.403•E-4</td> <td>0.107•E-2</td> <td>0.98•E-9</td> <td>0.129•E-8</td> <td>0.966•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.354•E-2</td> <td>0.382•E-2</td> <td>0.354•E-2</td> <td>0.382•E-2</td> <td>0.354•E-2</td> <td>0.382•E-2</td> <td>0.559•E-9</td> <td>0.104•E-8</td> <td>0.559•E-9</td> <td>0.13•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.562•E-1</td> <td>0.112</td> <td>0.551•E-1</td> <td>0.113</td> <td>0.54•E-1</td> <td>0.111</td> <td>14.79</td> <td>17.96</td> <td>14.81</td> <td>22.56</td> </tr> <tr> <th>LinReg</th> <td>0.933</td> <td>0.557</td> <td>0.936</td> <td>0.558</td> <td>0.93</td> <td>0.555</td> <td>10.3</td> <td>14.82</td> <td>10.42</td> <td>18.84</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.545•E-1</td> <td>0.163</td> <td>0.539•E-1</td> <td>0.162</td> <td>0.523•E-1</td> <td>0.162</td> <td>14.79</td> <td>17.97</td> <td>14.81</td> <td>22.63</td> </tr> <tr> <th>LinReg</th> <td>0.482</td> <td>0.426</td> <td>0.482</td> <td>0.429</td> <td>0.48</td> <td>0.425</td> <td>13.29</td> <td>18.14</td> <td>13.23</td> <td>21.58</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.499•E-1</td> <td>0.157</td> <td>0.49•E-1</td> <td>0.154</td> <td>0.475•E-1</td> <td>0.154</td> <td>14.8</td> <td>18.05</td> <td>14.81</td> <td>22.57</td> </tr> <tr> <th>LinReg</th> <td>0.466</td> <td>0.429</td> <td>0.467</td> <td>0.429</td> <td>0.464</td> <td>0.427</td> <td>10.21</td> <td>14.71</td> <td>10.33</td> <td>18.33</td> </tr> </tbody></table>
<table id="30" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.803•E-4</td> <td>0.367•E-3</td> <td>0.778•E-4</td> <td>0.372•E-3</td> <td>0.766•E-4</td> <td>0.37•E-3</td> <td>0.974•E-9</td> <td>0.127•E-8</td> <td>0.963•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.976•E-2</td> <td>0.548•E-2</td> <td>0.975•E-2</td> <td>0.548•E-2</td> <td>0.976•E-2</td> <td>0.548•E-2</td> <td>0.566•E-9</td> <td>0.104•E-8</td> <td>0.564•E-9</td> <td>0.129•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.522•E-4</td> <td>0.136•E-2</td> <td>0.514•E-4</td> <td>0.133•E-2</td> <td>0.491•E-4</td> <td>0.135•E-2</td> <td>0.975•E-9</td> <td>0.129•E-8</td> <td>0.963•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.351•E-2</td> <td>0.326•E-2</td> <td>0.35•E-2</td> <td>0.328•E-2</td> <td>0.351•E-2</td> <td>0.326•E-2</td> <td>0.807•E-9</td> <td>0.13•E-8</td> <td>0.791•E-9</td> <td>0.163•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.521•E-4</td> <td>0.13•E-2</td> <td>0.508•E-4</td> <td>0.13•E-2</td> <td>0.495•E-4</td> <td>0.128•E-2</td> <td>0.975•E-9</td> <td>0.128•E-8</td> <td>0.963•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.335•E-2</td> <td>0.397•E-2</td> <td>0.334•E-2</td> <td>0.396•E-2</td> <td>0.334•E-2</td> <td>0.397•E-2</td> <td>0.552•E-9</td> <td>0.103•E-8</td> <td>0.552•E-9</td> <td>0.129•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.735•E-1</td> <td>0.125</td> <td>0.721•E-1</td> <td>0.125</td> <td>0.713•E-1</td> <td>0.124</td> <td>14.74</td> <td>17.92</td> <td>14.78</td> <td>22.66</td> </tr> <tr> <th>LinReg</th> <td>0.926</td> <td>0.57</td> <td>0.929</td> <td>0.571</td> <td>0.923</td> <td>0.568</td> <td>10.23</td> <td>14.73</td> <td>10.34</td> <td>18.86</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.554•E-1</td> <td>0.174</td> <td>0.547•E-1</td> <td>0.175</td> <td>0.53•E-1</td> <td>0.174</td> <td>14.75</td> <td>18.23</td> <td>14.77</td> <td>22.7</td> </tr> <tr> <th>LinReg</th> <td>0.468</td> <td>0.421</td> <td>0.468</td> <td>0.423</td> <td>0.466</td> <td>0.419</td> <td>13.2</td> <td>17.93</td> <td>13.2</td> <td>21.61</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.556•E-1</td> <td>0.19</td> <td>0.547•E-1</td> <td>0.19</td> <td>0.536•E-1</td> <td>0.188</td> <td>14.75</td> <td>18.09</td> <td>14.77</td> <td>22.63</td> </tr> <tr> <th>LinReg</th> <td>0.45</td> <td>0.427</td> <td>0.452</td> <td>0.429</td> <td>0.45</td> <td>0.427</td> <td>10.13</td> <td>14.62</td> <td>10.26</td> <td>18.41</td> </tr> </tbody></table>
<table id="40" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.368•E-4</td> <td>0.415•E-3</td> <td>0.353•E-4</td> <td>0.419•E-3</td> <td>0.345•E-4</td> <td>0.423•E-3</td> <td>0.974•E-9</td> <td>0.127•E-8</td> <td>0.961•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.968•E-2</td> <td>0.563•E-2</td> <td>0.967•E-2</td> <td>0.563•E-2</td> <td>0.968•E-2</td> <td>0.563•E-2</td> <td>0.563•E-9</td> <td>0.103•E-8</td> <td>0.56•E-9</td> <td>0.13•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.722•E-4</td> <td>0.18•E-2</td> <td>0.708•E-4</td> <td>0.193•E-2</td> <td>0.684•E-4</td> <td>0.181•E-2</td> <td>0.976•E-9</td> <td>0.128•E-8</td> <td>0.963•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.34•E-2</td> <td>0.315•E-2</td> <td>0.339•E-2</td> <td>0.316•E-2</td> <td>0.339•E-2</td> <td>0.315•E-2</td> <td>0.807•E-9</td> <td>0.13•E-8</td> <td>0.789•E-9</td> <td>0.162•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.429•E-4</td> <td>0.163•E-2</td> <td>0.424•E-4</td> <td>0.16•E-2</td> <td>0.405•E-4</td> <td>0.159•E-2</td> <td>0.976•E-9</td> <td>0.127•E-8</td> <td>0.962•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.319•E-2</td> <td>0.353•E-2</td> <td>0.319•E-2</td> <td>0.353•E-2</td> <td>0.318•E-2</td> <td>0.353•E-2</td> <td>0.549•E-9</td> <td>0.104•E-8</td> <td>0.547•E-9</td> <td>0.131•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.479•E-1</td> <td>0.137</td> <td>0.466•E-1</td> <td>0.138</td> <td>0.459•E-1</td> <td>0.138</td> <td>14.74</td> <td>17.87</td> <td>14.76</td> <td>22.63</td> </tr> <tr> <th>LinReg</th> <td>0.92</td> <td>0.582</td> <td>0.922</td> <td>0.583</td> <td>0.917</td> <td>0.58</td> <td>10.2</td> <td>14.72</td> <td>10.31</td> <td>18.93</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.659•E-1</td> <td>0.206</td> <td>0.648•E-1</td> <td>0.213</td> <td>0.636•E-1</td> <td>0.2</td> <td>14.75</td> <td>18.11</td> <td>14.78</td> <td>22.7</td> </tr> <tr> <th>LinReg</th> <td>0.459</td> <td>0.42</td> <td>0.459</td> <td>0.422</td> <td>0.457</td> <td>0.418</td> <td>13.19</td> <td>17.94</td> <td>13.17</td> <td>21.33</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.505•E-1</td> <td>0.198</td> <td>0.5•E-1</td> <td>0.197</td> <td>0.484•E-1</td> <td>0.195</td> <td>14.75</td> <td>17.93</td> <td>14.77</td> <td>22.69</td> </tr> <tr> <th>LinReg</th> <td>0.439</td> <td>0.423</td> <td>0.44</td> <td>0.424</td> <td>0.437</td> <td>0.421</td> <td>10.09</td> <td>14.65</td> <td>10.22</td> <td>18.45</td> </tr> </tbody></table>
<table id="50" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.281•E-4</td> <td>0.474•E-3</td> <td>0.268•E-4</td> <td>0.47•E-3</td> <td>0.259•E-4</td> <td>0.471•E-3</td> <td>0.973•E-9</td> <td>0.126•E-8</td> <td>0.957•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.961•E-2</td> <td>0.578•E-2</td> <td>0.961•E-2</td> <td>0.578•E-2</td> <td>0.961•E-2</td> <td>0.578•E-2</td> <td>0.561•E-9</td> <td>0.103•E-8</td> <td>0.556•E-9</td> <td>0.13•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.435•E-4</td> <td>0.113•E-2</td> <td>0.427•E-4</td> <td>0.11•E-2</td> <td>0.412•E-4</td> <td>0.109•E-2</td> <td>0.976•E-9</td> <td>0.127•E-8</td> <td>0.96•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.336•E-2</td> <td>0.308•E-2</td> <td>0.338•E-2</td> <td>0.311•E-2</td> <td>0.335•E-2</td> <td>0.309•E-2</td> <td>0.847•E-9</td> <td>0.131•E-8</td> <td>0.877•E-9</td> <td>0.165•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.33•E-4</td> <td>0.151•E-2</td> <td>0.326•E-4</td> <td>0.151•E-2</td> <td>0.313•E-4</td> <td>0.158•E-2</td> <td>0.976•E-9</td> <td>0.127•E-8</td> <td>0.961•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.307•E-2</td> <td>0.502•E-2</td> <td>0.306•E-2</td> <td>0.507•E-2</td> <td>0.306•E-2</td> <td>0.504•E-2</td> <td>0.546•E-9</td> <td>0.106•E-8</td> <td>0.542•E-9</td> <td>0.142•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.408•E-1</td> <td>0.142</td> <td>0.397•E-1</td> <td>0.141</td> <td>0.389•E-1</td> <td>0.14</td> <td>14.72</td> <td>17.79</td> <td>14.73</td> <td>22.7</td> </tr> <tr> <th>LinReg</th> <td>0.915</td> <td>0.592</td> <td>0.917</td> <td>0.593</td> <td>0.912</td> <td>0.59</td> <td>10.18</td> <td>14.68</td> <td>10.26</td> <td>19</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.514•E-1</td> <td>0.179</td> <td>0.507•E-1</td> <td>0.174</td> <td>0.494•E-1</td> <td>0.174</td> <td>14.74</td> <td>17.85</td> <td>14.77</td> <td>22.78</td> </tr> <tr> <th>LinReg</th> <td>0.456</td> <td>0.418</td> <td>0.457</td> <td>0.42</td> <td>0.453</td> <td>0.416</td> <td>13.51</td> <td>17.86</td> <td>13.87</td> <td>21.6</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.445•E-1</td> <td>0.217</td> <td>0.439•E-1</td> <td>0.217</td> <td>0.427•E-1</td> <td>0.219</td> <td>14.74</td> <td>17.91</td> <td>14.77</td> <td>22.77</td> </tr> <tr> <th>LinReg</th> <td>0.43</td> <td>0.424</td> <td>0.431</td> <td>0.425</td> <td>0.429</td> <td>0.423</td> <td>10.07</td> <td>14.67</td> <td>10.18</td> <td>18.65</td> </tr> </tbody></table>
<table id="60" border="1" class="dataframe"> <thead> <tr> <th></th> <th></th> <th></th> <th colspan="2" halign="left">mid price</th> <th colspan="2" halign="left">bid price expectation</th> <th colspan="2" halign="left">ask price expectation</th> <th colspan="2" halign="left">bid price variance</th> <th colspan="2" halign="left">ask price variance</th> </tr> <tr> <th></th> <th></th> <th></th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> <th>train</th> <th>val</th> </tr> </thead> <tbody> <tr> <th rowspan="6" valign="top">MSE</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.36•E-4</td> <td>0.812•E-3</td> <td>0.348•E-4</td> <td>0.804•E-3</td> <td>0.342•E-4</td> <td>0.82•E-3</td> <td>0.972•E-9</td> <td>0.126•E-8</td> <td>0.957•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.956•E-2</td> <td>0.589•E-2</td> <td>0.956•E-2</td> <td>0.588•E-2</td> <td>0.956•E-2</td> <td>0.588•E-2</td> <td>0.559•E-9</td> <td>0.102•E-8</td> <td>0.553•E-9</td> <td>0.131•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.5•E-4</td> <td>0.181•E-2</td> <td>0.494•E-4</td> <td>0.171•E-2</td> <td>0.48•E-4</td> <td>0.178•E-2</td> <td>0.975•E-9</td> <td>0.127•E-8</td> <td>0.959•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.322•E-2</td> <td>0.294•E-2</td> <td>0.323•E-2</td> <td>0.295•E-2</td> <td>0.32•E-2</td> <td>0.295•E-2</td> <td>0.11•E-8</td> <td>0.142•E-8</td> <td>0.942•E-9</td> <td>0.165•E-8</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.42•E-4</td> <td>0.241•E-2</td> <td>0.414•E-4</td> <td>0.249•E-2</td> <td>0.397•E-4</td> <td>0.241•E-2</td> <td>0.975•E-9</td> <td>0.126•E-8</td> <td>0.959•E-9</td> <td>0.148•E-8</td> </tr> <tr> <th>LinReg</th> <td>0.296•E-2</td> <td>0.364•E-2</td> <td>0.296•E-2</td> <td>0.365•E-2</td> <td>0.296•E-2</td> <td>0.365•E-2</td> <td>0.542•E-9</td> <td>0.103•E-8</td> <td>0.537•E-9</td> <td>0.136•E-8</td> </tr> <tr> <th rowspan="6" valign="top">MAPE (%)</th> <th rowspan="2" valign="top">LOB</th> <th>NN</th> <td>0.466•E-1</td> <td>0.185</td> <td>0.459•E-1</td> <td>0.184</td> <td>0.451•E-1</td> <td>0.185</td> <td>14.73</td> <td>17.89</td> <td>14.74</td> <td>22.77</td> </tr> <tr> <th>LinReg</th> <td>0.911</td> <td>0.599</td> <td>0.913</td> <td>0.6</td> <td>0.908</td> <td>0.597</td> <td>10.17</td> <td>14.62</td> <td>10.23</td> <td>19.08</td> </tr> <tr> <th rowspan="2" valign="top">LIQ</th> <th>NN</th> <td>0.559•E-1</td> <td>0.24</td> <td>0.554•E-1</td> <td>0.233</td> <td>0.541•E-1</td> <td>0.237</td> <td>14.74</td> <td>17.97</td> <td>14.77</td> <td>22.91</td> </tr> <tr> <th>LinReg</th> <td>0.445</td> <td>0.415</td> <td>0.447</td> <td>0.416</td> <td>0.443</td> <td>0.413</td> <td>15.12</td> <td>18.52</td> <td>14.35</td> <td>21.89</td> </tr> <tr> <th rowspan="2" valign="top">LOB+LIQ</th> <th>NN</th> <td>0.509•E-1</td> <td>0.284</td> <td>0.5•E-1</td> <td>0.289</td> <td>0.488•E-1</td> <td>0.281</td> <td>14.76</td> <td>17.86</td> <td>14.76</td> <td>22.84</td> </tr> <tr> <th>LinReg</th> <td>0.422</td> <td>0.418</td> <td>0.423</td> <td>0.419</td> <td>0.421</td> <td>0.417</td> <td>10.05</td> <td>14.64</td> <td>10.14</td> <td>18.59</td> </tr> </tbody></table>
<script>
var sel = document.getElementById("windows");
for (var i=0, n=sel.options.length;i<n;i++) { // looping over the options
if (sel.options[i].value != "NONE") {
document.getElementById(sel.options[i].value).style.display="none";
};
};
if (sel.value != "NONE") {
document.getElementById(sel.value).style.display="block";
document.getElementById("table_title").innerHTML = "Errors for Window Size " + sel.value;
};
sel.onchange = function (){
for (var i=0, n=sel.options.length;i<n;i++) { // looping over the options
if (sel.options[i].value != "NONE") {
document.getElementById(sel.options[i].value).style.display="none";
};
};
if (sel.value != "NONE") {
document.getElementById(sel.value).style.display="block";
document.getElementById("table_title").innerHTML = "Errors for Window Size " + sel.value;
}else {document.getElementById("table_title").innerHTML = ""};
};
</script>
</body>
</html>
In [4]:
from utils.plotter import plotter
from utils.df_handler import force_zehnerpot
def get_axis(obs,err_type):
axis = []
obs_dict = {1:'mid price',3:'bid price expectation',5:'ask price expectation',7:'bid price variance',9:'ask price variance'}
err_type_dict={0:'MSE',6:'MAPE (%)'}
model_dict = {0:'LOB',2:'LIQ',4:'LOB+LIQ'}
color_dict = []
axis += [[list(errors.keys()),[errors[i][err_type + model][obs] for i in errors],'-X',dict(color=color,label=model_dict[model],markersize=15,alpha=0.5)] for model,color in zip([0,2,4],['blue','green','red'])]
axis += [[list(errors.keys()),[np.array(errors[i])[err_type+1:err_type+6:2,obs].min() for i in errors],'-X',dict(color='black',label='LinReg',markersize=15,alpha=0.5)]]
axis +=[ \
[dict(axis='both',labelsize=12)] \
,['Window Size',dict(fontsize=15)], [err_type_dict[err_type],dict(fontsize=15)], [[-0.125, 1.2, bool(err_type)*obs_dict[obs].title()],dict(fontsize=20,horizontalalignment='center',verticalalignment='top')] \
,[dict(ncol=1,shadow=1,labelspacing=0.3,fontsize=15,loc='best')],[dict(b=True,axis='x',alpha=0.5)] \
]
return axis
args = [get_axis(obs,err_type) for obs in range(1,10,2) for err_type in [0,6]]
attrs = ['plot','plot','plot','plot'
,'tick_params'
,'set_xlabel','set_ylabel','text'
,'legend','grid']
fig = plotter(args,attrs,fig_title='Validation Performance of Models\n Stock: GARAN\n Year: 2017',dpi=300, ncols=2,xpad=0,ypad=12,hspace = 0.32,suptitle_y=.94)
Histograms and Autocorrelation Analysis of Errors¶
Below we have histograms and autocorrelation of errors for both feedforward and linear regression models.
For each forecast quantity we have three training cases (LOB, LIQ, LOB+LIQ) and six window sizes (10, 20, 30, 40, 50, 60), resulting in ninety plots in total.
Each plot consists of two coordinate systems, one belonging to the histograms, the other belonging to the autocorrelation graphs. The axes of histogram and autocorrelation graphs are given different colors for better distinguishability.¶
We show two types of errors per plot:
Squared Error shown above the x-axes of histogram and autocorrelation graphs
Relative Error shown below the x-axes of histogram and autocorrelation graphs
Then for each type of error we look at two cases:
Training Errors shown at the left side of the y-axes of histogram and autocorrelation graphs
Validation Errors shown at the right side of the y-axes of histogram and autocorrelation graphs
Below is a table summarizing what is shown in each quadrant:
| Squared Training Errors | Squared Validation Errors |
|---|---|
| Relative Training Errors | Relative Validation Errors |
Histograms¶
In histogram coordinate system we show the error bins on the x-axis and the count of errors that falls into the corresponding error value range on the y-axis.
The error values are divided by their mean value and the error counts are normalized by the total count. The mean values used as reference correspond to the MSE and MAPE values shown in the previous section.¶
Applying the described procedure,
Squared Error becomes: $\frac{(y_i - \hat{y}_i)^{2}} {\frac{1}{n} \sum^{n}_{k=1} (y_k - \hat{y}_k)^{2}} $¶
Relative Error becomes: $\frac {|{1 - \frac{\hat{y}_i}{y_i}}|} {\frac{1}{n} \sum^{n}_{k=1} |{1 - \frac{\hat{y}_k}{y_k}}|} $¶
where $y$: Ground Truth and $\hat{y}$: Prediction.
Autocorrelation¶
Autocorrelation coefficients of errors for time lags ranging from 1000 minutes to 18000 minutes can also be seen in the plots. The coefficients are calculated and shown every 1000 minutes.
The Pearson correlation coefficient formula is used to quantify the autocorrelation of the errors $(X_k)_{1\leq k\leq n}$ for a given lag $l$ with total minutes n:
$\;\;\;\;\;\;\;\;\;\;\; \rho_l = \frac{1}{n-l}\sum^{n-l}_{i=1} \tilde{X}^{l}_{i} \cdot \tilde{Y}^{l}_{i}$
, where $\tilde{X}^{l} := (\tilde{X}_1,...,\tilde{X}_{n-l})$ and $\tilde{Y}^{l}$ is the delayed version of $\tilde{X} = (\tilde{X}_1,...,\tilde{X}_n)$ by $l$ minutes, $\tilde{Y}^{l}_{k} := \tilde{X}_{l+k}$
and $\tilde{Z} = (\tilde{Z}_k)_{1\leq k\leq N}$ is the standardized version of a random vector $Z = (Z_k)_{1\leq k\leq N}$,
$\tilde{Z}_{k} := \frac{Z_k-\mu_Z}{\sqrt{\frac{1}{N}\sum^{N}_{i=1}Z^{2}_{i}-\mu^{2}_Z}}$ with $\mu_Z := \frac{1}{N}\sum^{N}_{i=1}Z_i$.
The time lags are given on the x-axis in minutes and on the y-axis, the absolute value of the correlation coeeficient $|\rho|$ is given.
The sign of the correlation coefficient can be read off the tuples $(\text{symbol of error type , sign})$ shown next to the y-axis.¶
In [5]:
# from utils.plotter import plotter
# import numpy as np
# def acf(x, maxlags,step=1):
# return np.array([1]+[np.corrcoef(x[:-i], x[i:])[0,1] for i in range(step, maxlags,step)])
# def get_xlim(errors_h,errors,tol):
# for i,k in enumerate(np.where(errors_h>tol*errors_h.max(),errors_h,0)[::-1]):
# if k != 0:
# return errors[len(errors)-i-1]
# def get_axis_args(data_type,observable,bins,xlim_tol,maxlags,**kwargs):
# obs_dict = {'Mid Price':0,'Bid Price Expectation':1,'Ask Price Expectation':2,'Bid Price Variance':3,'Ask Price Variance':4}
# n = obs_dict[observable]
# data_type_dict = {'LOB':'a) ','LIQ':'b) ','LOB+LIQ':'c) '}
# acorr_step = kwargs.get('acorr_step',1)
# acorr_start = 1 #acorr_step in kac kati
# pf = (maxlags-acorr_start*acorr_step)//acorr_step//4
# ps = 6
# lw=1
# maxlags += 1
# with open('./eval/errors/' + f'{data_type}.npy', 'rb') as input:
# nn_errors = np.load(input,allow_pickle='TRUE').item()
# with open('./eval/errors/' + f'LinReg_{data_type}.npy', 'rb') as input:
# linreg_errors = np.load(input,allow_pickle='TRUE').item()
# data_type=data_type.split('_')[0]
# se_train = nn_errors['se_train'][:,n] ; se_train /= se_train.mean()
# se_train_reg = linreg_errors['se_train'][:,n] ; se_train_reg /= se_train_reg.mean()
# se_val = nn_errors['se_val'][:,n] ; se_val /= se_val.mean()
# se_val_reg = linreg_errors['se_val'][:,n] ; se_val_reg /= se_val_reg.mean()
# ape_train = nn_errors['ape_train'][:,n] ; ape_train /= ape_train.mean()
# ape_train_reg = linreg_errors['ape_train'][:,n] ; ape_train_reg /= ape_train_reg.mean()
# ape_val = nn_errors['ape_val'][:,n] ; ape_val /= ape_val.mean()
# ape_val_reg = linreg_errors['ape_val'][:,n] ; ape_val_reg /= ape_val_reg.mean()
# train_h , train= np.histogram(se_train,bins) ; train = train[:-1] * 100 ; train_h = train_h/train_h.sum() * 100
# train_h_reg , train_reg = np.histogram(se_train_reg,bins) ; train_reg = train_reg[:-1] * 100 ; train_h_reg = train_h_reg/train_h_reg.sum() * 100
# val_h , val= np.histogram(se_val,bins) ; val = val[:-1] * 100 ; val_h = val_h/val_h.sum() * 100
# val_h_reg , val_reg = np.histogram(se_val_reg,bins) ; val_reg = val_reg[:-1] * 100 ; val_h_reg = val_h_reg/val_h_reg.sum() * 100
# train_h_ape , train_ape= np.histogram(ape_train,bins) ; train_ape = train_ape[:-1] * 100 ; train_h_ape = train_h_ape/train_h_ape.sum() * 100
# train_h_reg_ape , train_reg_ape = np.histogram(ape_train_reg,bins) ; train_reg_ape = train_reg_ape[:-1] * 100 ; train_h_reg_ape = train_h_reg_ape/train_h_reg_ape.sum() * 100
# val_h_ape , val_ape = np.histogram(ape_val,bins) ; val_ape = val_ape[:-1] * 100 ; val_h_ape = val_h_ape/val_h_ape.sum() * 100
# val_h_reg_ape , val_reg_ape = np.histogram(ape_val_reg,bins) ; val_reg_ape = val_reg_ape[:-1] * 100 ; val_h_reg_ape = val_h_reg_ape/val_h_reg_ape.sum() * 100
# xlim= max([get_xlim(i,k,xlim_tol) for i,k in zip([train_h,train_h_reg,val_h,val_h_reg,train_h_ape,train_h_reg_ape,val_h_ape,val_h_reg_ape],[train,train_reg,val,val_reg,train_ape,train_reg_ape,val_ape,val_reg_ape])])
# ac_se_train = acf(se_train,maxlags,acorr_step)[acorr_start:] ; ac_se_train_reg = acf(se_train_reg,maxlags,acorr_step)[acorr_start:]
# ac_ape_train = acf(ape_train,maxlags,acorr_step)[acorr_start:] ; ac_ape_train_reg = acf(ape_train_reg,maxlags,acorr_step)[acorr_start:]
# ac_se_val = acf(se_val,maxlags,acorr_step)[acorr_start:] ; ac_se_val_reg = acf(se_val_reg,maxlags,acorr_step)[acorr_start:]
# ac_ape_val = acf(ape_val,maxlags,acorr_step)[acorr_start:] ; ac_ape_val_reg = acf(ape_val_reg,maxlags,acorr_step) [acorr_start:]
# acorr_range = [*range(acorr_start*acorr_step, maxlags,acorr_step)]
# acorr_range_neg = [*range(-acorr_start*acorr_step, -maxlags,-acorr_step)]
# ylim_2nd = max(ac_se_train.max(),ac_ape_train.max(),ac_se_val.max(),ac_ape_val.max(),ac_se_train_reg.max(),ac_ape_train_reg.max(),ac_se_val_reg.max(),ac_ape_val_reg.max())
# ylim_2nd = round(ylim_2nd,2)
# ylim_2nd += 0.2
# axis_args = [
# [
# [-train,train_h,-np.diff(train)[0],dict(align='edge',color='blue',alpha=0.8)] ,[val,val_h,np.diff(val)[0],dict(align='edge',color='tomato',alpha=0.8)] ,[-train_reg,train_h_reg,-np.diff(train_reg)[0],dict(align='edge',color='gold',alpha=0.8)] ,[val_reg,val_h_reg,np.diff(val_reg)[0],dict(align='edge',color='limegreen',alpha=0.8)]
# ,[-train_ape,-train_h_ape,-np.diff(train_ape)[0],dict(align='edge',color='blue',alpha=0.4)] ,[val_ape,-val_h_ape,np.diff(val_ape)[0],dict(align='edge',color='tomato',alpha=0.4)] ,[-train_reg_ape,-train_h_reg_ape,-np.diff(train_reg_ape)[0],dict(align='edge',color='gold',alpha=0.4)] ,[val_reg_ape,-val_h_reg_ape,np.diff(val_reg_ape)[0],dict(align='edge',color='limegreen',alpha=0.4)]
# ,[[],'-',dict(marker='o',color='blue',linewidth=2,alpha=0.5,ls='none',fillstyle='left')] ,[[],'-',dict(marker='o',color='tomato',linewidth=2,alpha=0.5,ls='none',fillstyle='right')] ,[[],'-',dict(marker='o',color='gold',linewidth=2,alpha=0.5,ls='none',fillstyle='left')] ,[[],'-',dict(marker='o',color='limegreen',linewidth=2,alpha=0.5,ls='none',fillstyle='right')]
# \
# ,[acorr_range_neg[::pf],ac_se_train[::pf] ,'o',dict(color='blue',markersize=ps)] ,[acorr_range[::pf],ac_se_val[::pf] ,'o',dict(color='crimson',markersize=ps)] ,[acorr_range_neg[::pf],ac_se_train_reg[::pf] ,'o',dict(color='gold',markersize=ps)] ,[acorr_range[::pf],ac_se_val_reg[::pf] ,'o',dict(color='limegreen',markersize=ps)]
# ,[acorr_range_neg,ac_se_train ,'-',dict(color='blue',alpha=0.5,linewidth=lw)] ,[acorr_range,ac_se_val ,'-',dict(color='crimson',alpha=0.5,linewidth=lw)] ,[acorr_range_neg,ac_se_train_reg ,'-',dict(color='gold',alpha=1,linewidth=lw)] ,[acorr_range,ac_se_val_reg ,'-',dict(color='limegreen',alpha=1,linewidth=lw)]
# ,[acorr_range_neg[::pf],-ac_ape_train[::pf],'D',dict(color='blue',markersize=ps)] ,[acorr_range[::pf],-ac_ape_val[::pf],'D',dict(color='crimson',markersize=ps)] ,[acorr_range_neg[::pf],-ac_ape_train_reg[::pf],'D',dict(color='gold',markersize=ps)] ,[acorr_range[::pf],-ac_ape_val_reg[::pf],'D',dict(color='limegreen',markersize=ps)]
# ,[acorr_range_neg,-ac_ape_train ,'--',dict(color='blue',alpha=0.5,linewidth=lw)] ,[acorr_range,-ac_ape_val ,'--',dict(color='crimson',alpha=0.5,linewidth=lw)] ,[acorr_range_neg,-ac_ape_train_reg ,'--',dict(color='gold',alpha=1,linewidth=lw)] ,[acorr_range,-ac_ape_val_reg ,'--',dict(color='limegreen',alpha=1,linewidth=lw)]
# ,[[],dict(marker='o',color='black',ls='solid',fillstyle='none')],[[],dict(marker='D',color='black',ls='dashed',fillstyle='none')]
# \
# ,[-xlim,xlim],[-ylim_2nd,ylim_2nd]
# ,[dict(color='chocolate', lw=1)],[dict(color='chocolate', lw=1)],[dict(color='magenta', lw=0.8)]
# ,[acorr_range_neg[::pf]+acorr_range[::pf]]
# ,dict(x='positive',y='positive'),dict(x='positive',y='positive')
# ,['Errors Relative to the Mean (%)',dict(fontsize=15,color='sienna')],['Normalized Error Counts (%)',dict(fontsize=15,color='sienna')],[data_type_dict[data_type]+data_type,dict(fontsize=20,pad=60)]
# ,['Time Lag in Minutes',dict(fontsize=15,color='darkmagenta',labelpad=5)],[r'Autocorr. Coeff. Magnitude',dict(fontsize=15,rotation=-90,color='darkmagenta',labelpad=15)]
# ,[[-0.06, 1, 'Squared Errors'],dict(fontsize=20,rotation=90,horizontalalignment='right',verticalalignment='top')],[[-0.06, 0, 'Relative Errors'],dict(fontsize=20,rotation=90,horizontalalignment='right',verticalalignment='bottom')],[[0.01, -0.05, 'Training Side'],dict(color='black',fontsize=20,horizontalalignment='left',verticalalignment='top')],[[0.99, -0.05, 'Validation Side'],dict(color='black',fontsize=20,horizontalalignment='right',verticalalignment='top')]
# ,[[1.08, 0.72, r'$(\circ,\plus) ; (\diamond,\minus)$'],dict(color='darkmagenta',fontsize=20,horizontalalignment='right',verticalalignment='bottom',rotation=-90)],[[1.08, 0.28, r'$(\circ,\minus) ; (\diamond,\plus)$'],dict(color='darkmagenta',fontsize=20,horizontalalignment='right',verticalalignment='top',rotation=-90)]
# ,[[0.5, 1.2, kwargs.get('ax_suptitle','')],dict(color='black',fontsize=25,horizontalalignment='center',verticalalignment='bottom')]
# ,dict(color='sienna'),dict(color='darkmagenta'),[dict(axis='x',colors='darkmagenta',rotation=90,direction='in')],[dict(axis='y',colors='darkmagenta',direction='in')],[dict(axis='x',colors='sienna',direction='in')],[dict(axis='y',colors='sienna',direction='in')]
# ,[dict(line_order = [[0, 1], [2, 3],[4],[5]],labels=['NN','LinReg','Squared Err. Autocorr.','Rel. Err. Autocorr.'] ,ncol=2,shadow=1,columnspacing=0.5,labelspacing=1,fontsize=12,loc='upper right')],[dict(b=True,axis='both',alpha=0.5)]
# ]
# ]
# return axis_args
# args = []
# window_size=60
# for obs in ['Mid Price','Bid Price Expectation','Ask Price Expectation','Bid Price Variance','Ask Price Variance']:#
# for i,data_type in enumerate([f'LOB_{window_size}',f'LIQ_{window_size}',f'LOB+LIQ_{window_size}']):#
# args += get_axis_args(data_type,obs,'fd',0.1,maxlags=18000,acorr_step=2000,ax_suptitle=obs*int(i%3==1))
# attrs = [
# 'bar','bar','bar','bar'
# ,'bar','bar','bar','bar'
# ,'plot','plot','plot','plot'#just for legend
# \
# ,'2nd_plot','2nd_plot','2nd_plot','2nd_plot' #yuvarlak
# ,'2nd_plot','2nd_plot','2nd_plot','2nd_plot' #yuvarlagin cizgisi
# ,'2nd_plot','2nd_plot','2nd_plot','2nd_plot' #diamond
# ,'2nd_plot','2nd_plot','2nd_plot','2nd_plot' #diamond cizgisi
# ,'plot','plot'#just for legend
# \
# ,'set_xlim','2nd_set_ylim'
# ,'axvline','axhline','2nd_axhline'
# ,'2nd_set_xticks'
# ,'ticks','2nd_ticks'
# ,'set_xlabel','set_ylabel','set_title'
# ,'2nd_set_xlabel','2nd_set_ylabel'
# ,'text','text','text','text'
# ,'text','text'
# ,'text' # baslik icin
# ,'color_ax','2nd_color_ax','2nd_tick_params','2nd_tick_params','tick_params','tick_params'
# ,'legend','grid'
# ]
# fig = plotter(args,attrs,second_plot=1,fig_title=f'Distributions and Autocorrelation of Errors\n Stock: GARAN\n Year: 2017\n Window Size: {window_size}' \
# ,dpi=300, ncols=3,xpad=10,ypad=25, hspace = 0.45 ,suptitle_y=0.95,suptitle_x=0.51,save_path=f'./results/autocorr_{window_size}.png')
In [6]:
%%html
<label for="dropdown">Select window size:</label>
<select name="dropdown" id="dropdown" style="width: 50px;">
<option value="10" width = "1000">10</option>
<option value="20" width = "1000">20</option>
<option value="30" width = "1000">30</option>
<option value="40" width = "1000">40</option>
<option value="50" width = "1000">50</option>
<option value="60" width = "1000">60</option>
</select>
<img src = "./results/autocorr_10.png" id="plots" width="100%">
<script>
var dropdown = document.getElementById("dropdown");
dropdown.onchange = function (){
document.getElementById("plots").src = "./results/autocorr_" + dropdown.value + ".png";
}
</script>
Best Results¶
The three tables below show the best performing model for each forecast variable and criterion:
Table 1) Best Models¶
Shows the best model in each category.
Table 2) Best Neural Models¶
Shows the best neural model in each category.
Table 3) Best Linear Models¶
Shows the best linear model in each category.
Feedforward Model trained with Limit Order Book data is the best in predicting Mid Price, Bid Price Expectation and Ask Price Expectation. It is also the best performer among neural models which are not trained with LOB. Lowest validation errors are achieved by window sizes 20 and 30 in terms of Mean Absolute Percentage and Mean Squared Error, respectively.
Neural Network models are outperformed by the linear models in variance predictions. For each case, the difference is ~3-4% in Mean Absolute Percentage Error.
On the other hand, mid price and expectation predictions of the neural model are ~8 times and ~4 times better in terms of MSE and MAPE, respectively.
In [7]:
best_results = {}
for err_type in ['MSE','MAPE (%)']:
best_results[err_type]={}
df = errors_df[[bool(i.count(err_type)) for i in errors_df.index]] \
[errors_df.columns[[bool(i.count('val')) for i in errors_df.columns]]]
for column in df.columns: #(midprice,val),(ask price,val),(bid price,val)...
temp_list = [*df[column][df[column]==df[column].min()].index[0]][::-1]
temp_list= [temp_list[0]+f' ({temp_list[1]})',temp_list[-1],'~'+force_zehnerpot(df[column].min(),2,0.05)]
best_results[err_type][column[0].title()]= temp_list
results_df = pd.DataFrame.from_dict(best_results)
pd.DataFrame(data=np.array([np.concatenate(i) for i in results_df.values]) , \
index=results_df.index,columns=pd.MultiIndex.from_tuples( \
[(i,k) for i in results_df.columns for k in ['Model (trained with)','Window Size','Validation Error']])). \
style.set_caption('Best Models')
Out[7]:
In [8]:
best_results = {}
for err_type in ['MSE','MAPE (%)']:
best_results[err_type]={}
df = errors_df[[bool(i.count(err_type) and not i.count('LinReg')) for i in errors_df.index]] \
[errors_df.columns[[bool(i.count('val')) for i in errors_df.columns]]]
for column in df.columns: #(midprice,val),(ask price,val),(bid price,val)...
temp_list = [*df[column][df[column]==df[column].min()].index[0]][::-1]
temp_list= [temp_list[0]+f' ({temp_list[1]})',temp_list[-1],'~'+force_zehnerpot(df[column].min(),2,0.05)]
best_results[err_type][column[0].title()]= temp_list
results_df_ = pd.DataFrame.from_dict(best_results)
pd.DataFrame(data=np.array([np.concatenate(i) for i in results_df_.values]) , \
index=results_df_.index,columns=pd.MultiIndex.from_tuples( \
[(i,k) for i in results_df_.columns for k in ['Model (trained with)','Window Size','Validation Error']])).\
style.set_caption('Best Neural Models')
Out[8]:
In [9]:
best_results = {}
for err_type in ['MSE','MAPE (%)']:
best_results[err_type]={}
df = errors_df[[bool(i.count(err_type) and i.count('LinReg')) for i in errors_df.index]] \
[errors_df.columns[[bool(i.count('val')) for i in errors_df.columns]]]
for column in df.columns: #(midprice,val),(ask price,val),(bid price,val)...
temp_list = [*df[column][df[column]==df[column].min()].index[0]][::-1]
temp_list= [temp_list[0]+f' ({temp_list[1]})',temp_list[-1],'~'+force_zehnerpot(df[column].min(),2,0.05)]
best_results[err_type][column[0].title()]= temp_list
results_df__ = pd.DataFrame.from_dict(best_results)
pd.DataFrame(data=np.array([np.concatenate(i) for i in results_df__.values]) , \
index=results_df__.index,columns=pd.MultiIndex.from_tuples( \
[(i,k) for i in results_df__.columns for k in ['Model (trained with)','Window Size','Validation Error']])). \
style.set_caption('Best Linear Models')
Out[9]: