This lab required some knowledge.
{
/*
~/ml4/public/class04/linr10/linr11a.scala
This script should download prices and predict daily direction of GSPC.
It should generate independent features from slopes of moving averages of prices.
It should create a Linear Regression model from many years of features.
Demo:
spark-shell -i linr11a.scala
*/
import org.apache.spark.sql.SQLContext
import org.apache.spark.ml.regression.LinearRegression
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.sql.Row
import sys.process._
// I should declare training_period, test_period
val training_period = " WHERE Date BETWEEN'1986-01-01'AND'2016-01-01' "
val test_period = " WHERE Date BETWEEN'2016-01-01'AND'2017-01-01' "
// I should get prices:
"/usr/bin/curl -L ml4.herokuapp.com/csv/GSPC.csv -o /tmp/gspc.csv"!
val sqlContext = new SQLContext(sc)
val dp10df = sqlContext
.read
.format("com.databricks.spark.csv")
.option("header","true")
.option("inferSchema","true")
.load("/tmp/gspc.csv")
dp10df.createOrReplaceTempView("tab")
spark.sql("SELECT COUNT(Date),MIN(Date),MAX(Date),MIN(Close),MAX(Close)FROM tab").show
// I should declare a dependent variable: "pctlead".
// If it truely is dependent that makes it predictable.
var sqls="SELECT Date,Close,LEAD(Close,1)OVER(ORDER BY Date) leadp FROM tab ORDER BY Date"
val dp11df=spark.sql(sqls);dp11df.createOrReplaceTempView("tab")
sqls="SELECT Date,Close,100*(leadp-Close)/Close pctlead FROM tab ORDER BY Date"
val dp12df=spark.sql(sqls);dp12df.createOrReplaceTempView("tab")
// I should now collect features AKA independent variables.
sqls = "SELECT Date, Close, pctlead"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS mavg2"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 3 PRECEDING AND CURRENT ROW) AS mavg3"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS mavg4"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS mavg5"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS mavg6"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 7 PRECEDING AND CURRENT ROW) AS mavg7"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 8 PRECEDING AND CURRENT ROW) AS mavg8"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 9 PRECEDING AND CURRENT ROW) AS mavg9"
sqls=sqls++" FROM tab ORDER BY Date"
val dp13df=spark.sql(sqls);dp13df.createOrReplaceTempView("tab")
// In Linear Regression, I should use pctlead as the label:
sqls = "SELECT Date, Close, pctlead, pctlead label "
sqls=sqls++",(mavg2-LAG(mavg2,1)OVER(ORDER BY Date))/mavg2 AS slp2 "
sqls=sqls++",(mavg3-LAG(mavg3,1)OVER(ORDER BY Date))/mavg3 AS slp3 "
sqls=sqls++",(mavg4-LAG(mavg4,1)OVER(ORDER BY Date))/mavg4 AS slp4 "
sqls=sqls++",(mavg5-LAG(mavg5,1)OVER(ORDER BY Date))/mavg5 AS slp5 "
sqls=sqls++",(mavg6-LAG(mavg6,1)OVER(ORDER BY Date))/mavg6 AS slp6 "
sqls=sqls++",(mavg7-LAG(mavg7,1)OVER(ORDER BY Date))/mavg7 AS slp7 "
sqls=sqls++",(mavg8-LAG(mavg8,1)OVER(ORDER BY Date))/mavg8 AS slp8 "
sqls=sqls++",(mavg9-LAG(mavg9,1)OVER(ORDER BY Date))/mavg9 AS slp9 "
sqls=sqls++" FROM tab ORDER BY Date"
val dp15df=spark.sql(sqls)
// I should copy slp-values into Vectors.dense():
val fill_vec = udf((
slp2:Float
,slp3:Float
,slp4:Float
,slp5:Float
,slp6:Float
,slp7:Float
,slp8:Float
,slp9:Float
)=> {Vectors.dense(
slp2
,slp3
,slp4
,slp5
,slp6
,slp7
,slp8
,slp9
)
}
)
val dp16df = dp15df.withColumn("features"
,fill_vec(
col("slp2")
,col("slp3")
,col("slp4")
,col("slp5")
,col("slp6")
,col("slp7")
,col("slp8")
,col("slp9")
)
)
// I should gather observations to learn from:
dp16df.createOrReplaceTempView("tab")
// I should create a LinearRegression instance. This instance is an 'Estimator'.
val lr = new LinearRegression()
lr.setMaxIter(1234).setRegParam(0.01)
// I should gather observations to learn from:
dp16df.createOrReplaceTempView("tab")
val train_df = spark.sql("SELECT * FROM tab"++training_period)
/*I should fit a LinearRegression model to observations.
This uses the parameters stored in lr.*/
val model1 = lr.fit(train_df)
val test_df = spark.sql("SELECT * FROM tab"++test_period)
/* I should predict. It is convenient that predictions_df will contain a copy of test_df.*/
val predictions_df = model1.transform(test_df)
predictions_df.createOrReplaceTempView("tab")
// eff logic
val eff = udf((pctlead:Float,prediction:Double)=> {if (prediction>0) pctlead else -pctlead})
// prediction report:
val p2df = predictions_df.withColumn("effectiveness",eff(col("pctlead"),col("prediction")))
println("Predictions Report:")
p2df.select("Date","Close","pctlead","effectiveness","label","prediction").show(255)
// I should report in a way which is consistent with the python script linr11a.py:
// I should get Linear Regression Accuracy:
sqls = "SELECT COUNT(Date) pc FROM tab WHERE (prediction>0 AND pctlead>0)OR(prediction<=0 AND pctlead<=0)"
val pc_df = spark.sql(sqls)
val poscount_i = pc_df.first()(0).asInstanceOf[Long]
sqls = "SELECT COUNT(Date) rcount FROM tab"
val rc_df = spark.sql(sqls)
val rcount_i = rc_df.first()(0).asInstanceOf[Long]
println("Linear Regression Accuracy:")
println(poscount_i.toFloat/rcount_i)
// I should get Long-Only Accuracy:
sqls = "SELECT COUNT(Date) upcount FROM tab WHERE pctlead>0"
val upcount_df = spark.sql(sqls)
val upcount_i = upcount_df.first()(0).asInstanceOf[Long]
println("Long Only Accuracy:")
println(upcount_i.toFloat/rcount_i)
val tp_df = spark.sql("SELECT COUNT(Date) tpc FROM tab WHERE prediction>0 AND pctlead>0")
println("True Positive Count:")
tp_df.show
val fp_df = spark.sql("SELECT COUNT(Date) fpc FROM tab WHERE prediction>0 AND pctlead<0")
println("False Positive Count:")
fp_df.show
val tn_df = spark.sql("SELECT COUNT(Date) tnc FROM tab WHERE prediction<=0 AND pctlead<=0")
println("True Negative Count:")
tn_df.show
val fn_df = spark.sql("SELECT COUNT(Date) fnc FROM tab WHERE prediction<=0 AND pctlead>0")
println("False Negative Count:")
fn_df.show
println("Effectiveness of negative predictions:")
spark.sql("SELECT -SUM(pctlead) eff_np FROM tab WHERE prediction<=0").show
println("Effectiveness of positive predictions:")
spark.sql("SELECT SUM(pctlead) eff_pp FROM tab WHERE prediction>0").show
println("Effectiveness of Long-Only-Model:")
spark.sql("SELECT SUM(pctlead) eff_lo FROM tab").show
}
I saw something like this:
dan@h80:~/ml4/public/class04/linr10 $
dan@h80:~/ml4/public/class04/linr10 $ spark-shell -i linr11a.scala
Spark context Web UI available at http://192.168.1.80:4042
Spark context available as 'sc' (master = local[*], app id = local-1515896821987).
Spark session available as 'spark'.
Loading linr11a.scala...
warning: there was one deprecation warning; re-run with -deprecation for details
warning: there was one feature warning; re-run with -feature for details
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 1252k 100 1252k 0 0 420k 0 0:00:02 0:00:02 --:--:-- 829k
+-----------+-------------------+-------------------+----------+----------+
|count(Date)| min(Date)| max(Date)|min(Close)|max(Close)|
+-----------+-------------------+-------------------+----------+----------+
| 17116|1950-01-03 00:00:00|2018-01-09 00:00:00| 16.66|2753.52002|
+-----------+-------------------+-------------------+----------+----------+
Predictions Report:
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+
| Date| Close| pctlead|effectiveness| label| prediction|
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+
|2016-01-04 00:00:00|2012.660034| 0.2012226074739071| 0.20122261| 0.2012226074739071| 0.08703354200252766|
|2016-01-05 00:00:00|2016.709961| -1.31153966170151| -1.3115396| -1.31153966170151| 0.13291345762180573|
|2016-01-06 00:00:00| 1990.26001| -2.3700443039098174| -2.3700442| -2.3700443039098174| 0.19395261892507978|
|2016-01-07 00:00:00|1943.089966| -1.083837463447639| -1.0838375| -1.083837463447639| 0.23275401465193407|
|2016-01-08 00:00:00|1922.030029| 0.08532723085774163| 0.08532723| 0.08532723085774163| 0.2907372292887592|
|2016-01-11 00:00:00|1923.670044| 0.7802798638371875| 0.7802799| 0.7802798638371875| 0.2194794015832985|
|2016-01-12 00:00:00|1938.680054| -2.4965452602732516| -2.4965453| -2.4965452602732516| 0.20729411675587486|
|2016-01-13 00:00:00|1890.280029| 1.669590564139637| 1.6695906| 1.669590564139637| 0.18928009706982082|
|2016-01-14 00:00:00|1921.839966| -2.159909812178397| -2.1599097| -2.159909812178397| 0.11450307354324421|
|2016-01-15 00:00:00|1880.329956|0.053182155440808176| 0.053182155|0.053182155440808176| 0.1831658929511503|
|2016-01-19 00:00:00|1881.329956| -1.1693855152753438| -1.1693856| -1.1693855152753438| 0.14274114868059404|
|2016-01-20 00:00:00|1859.329956| 0.5195438264643328| 0.5195438| 0.5195438264643328| 0.1952913927423723|
|2016-01-21 00:00:00| 1868.98999| 2.0283700930896904| 2.0283701| 2.0283700930896904| 0.03788587874095083|
|2016-01-22 00:00:00|1906.900024| -1.5637981868314241| -1.5637982| -1.5637981868314241| 0.09534460272886333|
|2016-01-25 00:00:00|1877.079956| 1.414433568220359| -1.4144336| 1.414433568220359|-0.01898306117525...|
|2016-01-26 00:00:00|1903.630005| -1.0863483946818713| -1.0863484| -1.0863483946818713| 0.082263005497833|
|2016-01-27 00:00:00|1882.949951| 0.5528577110863419| -0.5528577| 0.5528577110863419| -0.0228316850334181|
|2016-01-28 00:00:00|1893.359985| 2.4760217481832956| 2.4760218| 2.4760217481832956| 0.06731711604837584|
|2016-01-29 00:00:00| 1940.23999|-0.04432364060283344| 0.04432364|-0.04432364060283344|-0.07761119452146861|
|2016-02-01 00:00:00|1939.380005| -1.8743091042644822| 1.8743091| -1.8743091042644822| -0.1212303038400652|
|2016-02-02 00:00:00|1903.030029| 0.4992038935398218| 0.4992039| 0.4992038935398218| 0.05981993907765358|
|2016-02-03 00:00:00|1912.530029| 0.1526732629409628| -0.15267326| 0.1526732629409628|-0.02389321961047...|
|2016-02-04 00:00:00|1915.449951| -1.8481246133065947| -1.8481246| -1.8481246133065947| 0.10026073424988755|
|2016-02-05 00:00:00|1880.050049| -1.4153935962584503| -1.4153936| -1.4153935962584503| 0.08356500153135463|
|2016-02-08 00:00:00|1853.439941|-0.06636201005447463| -0.06636201|-0.06636201005447463| 0.16589068577997923|
|2016-02-09 00:00:00|1852.209961|-0.01889505009523...| -0.01889505|-0.01889505009523...| 0.1486704972373507|
|2016-02-10 00:00:00|1851.859985| -1.2301161634528224| -1.2301161| -1.2301161634528224| 0.18057699151112724|
|2016-02-11 00:00:00|1829.079956| 1.951804943402921| 1.951805| 1.951804943402921| 0.19405673841146112|
|2016-02-12 00:00:00|1864.780029| 1.651665425466653| 1.6516654| 1.651665425466653| 0.07078914712905633|
|2016-02-16 00:00:00|1895.579956| 1.648043908731858| -1.6480439| 1.648043908731858|-0.02464514699336...|
|2016-02-17 00:00:00|1926.819946| -0.4665713586089291| 0.46657136| -0.4665713586089291|-0.09270533419288834|
|2016-02-18 00:00:00|1917.829956|-0.00260330692217...| 0.002603307|-0.00260330692217...|-0.06664756778047445|
|2016-02-19 00:00:00|1917.780029| 1.4454197343192787| -1.4454198| 1.4454197343192787|-0.10222544831330452|
|2016-02-22 00:00:00| 1945.5| -1.2454371626831184| 1.2454371| -1.2454371626831184|-0.06850050231012314|
|2016-02-23 00:00:00| 1921.27002| 0.443978665736949| -0.44397867| 0.443978665736949|-0.05135699687537886|
|2016-02-24 00:00:00|1929.800049| 1.134827518081388| -1.1348275| 1.134827518081388|-0.00246842982022...|
|2016-02-25 00:00:00|1951.699951|-0.18701143063153722| 0.18701144|-0.18701143063153722|-0.07676985643311593|
|2016-02-26 00:00:00|1948.050049| -0.8120976670040306| -0.81209767| -0.8120976670040306|0.008288441173942403|
|2016-02-29 00:00:00| 1932.22998| 2.3868792264572924| 2.3868792| 2.3868792264572924| 0.06201598172965149|
|2016-03-01 00:00:00|1978.349976| 0.40943084379727995| -0.40943083| 0.40943084379727995|-0.06099063734154...|
|2016-03-02 00:00:00|1986.449951| 0.3498740552965472| -0.34987405| 0.3498740552965472| -0.0303992159107861|
|2016-03-03 00:00:00|1993.400024| 0.3305892405266673| -0.33058923| 0.3305892405266673|-0.08424891647150909|
|2016-03-04 00:00:00| 1999.98999| 0.08850144294971853| -0.088501446| 0.08850144294971853|-0.03694079013555097|
|2016-03-07 00:00:00| 2001.76001| -1.124010864818905| 1.1240109| -1.124010864818905|-0.06936345770279465|
|2016-03-08 00:00:00| 1979.26001| 0.5052393293188397| 0.5052393| 0.5052393293188397| 0.06339013711755134|
|2016-03-09 00:00:00| 1989.26001|0.015580467030054408| -0.015580467|0.015580467030054408|-0.00160050231127...|
|2016-03-10 00:00:00|1989.569946| 1.6395500477669567| 1.6395501| 1.6395500477669567| 0.05327296438135176|
|2016-03-11 00:00:00|2022.189941|-0.12609725467921024| 0.12609726|-0.12609725467921024|-0.05676898986071...|
|2016-03-14 00:00:00|2019.640015| -0.1836941718546818| -0.18369417| -0.1836941718546818|0.017847126028275276|
|2016-03-15 00:00:00|2015.930054| 0.5600351548705103| -0.56003517| 0.5600351548705103|-0.01988184037645028|
|2016-03-16 00:00:00|2027.219971| 0.6595236427847927| 0.65952367| 0.6595236427847927|0.022653883473985583|
|2016-03-17 00:00:00|2040.589966| 0.44056435392665033| -0.44056436| 0.44056435392665033|-0.04902630293293073|
|2016-03-18 00:00:00|2049.580078| 0.09855774954502321| -0.09855775| 0.09855774954502321|-0.00723418141760...|
|2016-03-21 00:00:00|2051.600098|-0.08773878504659462| 0.08773878|-0.08773878504659462|-0.07562468053115229|
|2016-03-22 00:00:00|2049.800049| -0.6386031655324603| -0.63860315| -0.6386031655324603| 0.00732724451524365|
|2016-03-23 00:00:00|2036.709961|-0.03780705229240...| -0.03780705|-0.03780705229240...|0.029076751899825726|
|2016-03-24 00:00:00|2035.939941| 0.05452557698998718| 0.054525577| 0.05452557698998718| 0.06466875013814474|
|2016-03-28 00:00:00|2037.050049| 0.8816651809226127| 0.88166517| 0.8816651809226127| 0.04922052322862414|
|2016-03-29 00:00:00| 2055.01001| 0.43503150624556275| 0.4350315| 0.43503150624556275|0.007130135296180666|
|2016-03-30 00:00:00|2063.949951|-0.20397592480187138| -0.20397593|-0.20397592480187138|0.005694311725338394|
|2016-03-31 00:00:00| 2059.73999| 0.6330915097686665| 0.6330915| 0.6330915097686665|0.006856114882150...|
|2016-04-01 00:00:00|2072.780029| -0.3208322111829817| -0.32083222| -0.3208322111829817|0.007013387843252...|
|2016-04-04 00:00:00|2066.129883| -1.014449245057461| -1.0144492| -1.014449245057461|0.006543372923626847|
|2016-04-05 00:00:00|2045.170044| 1.0507619189438957| 1.0507619| 1.0507619189438957| 0.06516495716196352|
|2016-04-06 00:00:00|2066.659912| -1.1975786560861146| -1.1975787| -1.1975786560861146| 0.01141639075874383|
|2016-04-07 00:00:00|2041.910034| 0.278657820631483| 0.27865782| 0.278657820631483| 0.06997303231483816|
|2016-04-08 00:00:00|2047.599976| -0.2739786123146514| -0.27397862| -0.2739786123146514| 0.06929975770371326|
|2016-04-11 00:00:00| 2041.98999| 0.9662134044055697| 0.9662134| 0.9662134044055697| 0.08119287920938206|
|2016-04-12 00:00:00|2061.719971| 1.0040137017230288| 1.0040137| 1.0040137017230288| 0.02643084427519642|
|2016-04-13 00:00:00|2082.419922|0.017292717774910475| 0.017292717|0.017292717774910475| 0.01580658626332606|
|2016-04-14 00:00:00|2082.780029|-0.09842849323767663| 0.098428495|-0.09842849323767663|-0.05232175241872806|
|2016-04-15 00:00:00| 2080.72998| 0.6541025568343978| 0.65410256| 0.6541025568343978|0.027500149099175995|
|2016-04-18 00:00:00|2094.340088| 0.308448519751584| -0.30844852| 0.308448519751584|-0.06127076307281...|
|2016-04-19 00:00:00|2100.800049| 0.07615446318947447| 0.07615446| 0.07615446318947447|0.012110192941894679|
|2016-04-20 00:00:00|2102.399902| -0.5194027068595269| 0.5194027| -0.5194027068595269|-0.04266109733010199|
|2016-04-21 00:00:00| 2091.47998|0.004785988914887...| 0.004785989|0.004785988914887...|0.015459482221444672|
|2016-04-22 00:00:00|2091.580078| -0.1812045849864888| -0.18120459| -0.1812045849864888| 0.0251550143477604|
|2016-04-25 00:00:00|2087.790039| 0.1872751534858758| 0.18727516| 0.1872751534858758| 0.06522087697392098|
|2016-04-26 00:00:00|2091.699951| 0.16493527182762052| 0.16493528| 0.16493527182762052| 0.04758615086083459|
|2016-04-27 00:00:00|2095.149902| -0.9230768157227635| -0.9230768| -0.9230768157227635| 0.04279456522837545|
|2016-04-28 00:00:00|2075.810059| -0.5063088481738572| -0.50630885| -0.5063088481738572|0.061796365290561595|
|2016-04-29 00:00:00|2065.300049| 0.780994655367873| 0.78099465| 0.780994655367873| 0.11235555391027519|
|2016-05-02 00:00:00|2081.429932| -0.8676638460102664| -0.86766386| -0.8676638460102664| 0.06193097310800989|
|2016-05-03 00:00:00|2063.370117| -0.5936889314753995| -0.5936889| -0.5936889314753995| 0.09474511762234014|
|2016-05-04 00:00:00|2051.120117|-0.02390079429950192| -0.023900794|-0.02390079429950192| 0.09313484314705445|
|2016-05-05 00:00:00|2050.629883| 0.31746391945074204| 0.3174639| 0.31746391945074204| 0.09431871215401959|
|2016-05-06 00:00:00|2057.139893| 0.07534966412709886| 0.075349666| 0.07534966412709886| 0.07792367359452629|
|2016-05-09 00:00:00|2058.689941| 1.2483643839788858| 1.2483643| 1.2483643839788858| 0.0762661313245832|
|2016-05-10 00:00:00|2084.389893| -0.9561518249023676| 0.95615184| -0.9561518249023676|-0.01624907708340706|
|2016-05-11 00:00:00|2064.459961|-0.01694651417848691| -0.016946515|-0.01694651417848691|0.039152202008452264|
|2016-05-12 00:00:00|2064.110107| -0.8478288992748931| -0.8478289| -0.8478288992748931| 0.0224273861108146|
|2016-05-13 00:00:00|2046.609985| 0.9796652584981903| 0.9796653| 0.9796652584981903| 0.09870618490565738|
|2016-05-16 00:00:00|2066.659912| -0.9411297372666149| -0.94112974| -0.9411297372666149|0.014763494256562149|
|2016-05-17 00:00:00|2047.209961|0.020517875938566795| 0.020517876|0.020517875938566795| 0.08849830166929842|
|2016-05-18 00:00:00|2047.630005|-0.37067077457677733| -0.37067077|-0.37067077457677733|0.027330816371681752|
|2016-05-19 00:00:00|2040.040039| 0.601950391425627| 0.6019504| 0.601950391425627| 0.12325470735819105|
|2016-05-20 00:00:00|2052.320068|-0.20854588262009888| -0.20854588|-0.20854588262009888|0.020875122918486963|
|2016-05-23 00:00:00|2048.040039| 1.3681382915580749| 1.3681383| 1.3681382915580749| 0.11132294743621687|
|2016-05-24 00:00:00|2076.060059| 0.6974740416216479| -0.69747406| 0.6974740416216479|-0.05698175868220819|
|2016-05-25 00:00:00|2090.540039|-0.02104437091817...| 0.021044372|-0.02104437091817...|-0.00182163803926...|
|2016-05-26 00:00:00|2090.100098| 0.4286857365622697| -0.42868572| 0.4286857365622697|-0.06943683886469904|
|2016-05-27 00:00:00|2099.060059|-0.10052632800821898| -0.100526325|-0.10052632800821898|2.239626658884635...|
|2016-05-31 00:00:00|2096.949951| 0.11350423498972209| -0.11350424| 0.11350423498972209|-0.05039564083661529|
|2016-06-01 00:00:00|2099.330078| 0.2824678244809108| 0.2824678| 0.2824678244809108|0.014864126374432143|
|2016-06-02 00:00:00| 2105.26001| -0.2911814678890852| 0.29118147| -0.2911814678890852|-0.02693365243091201|
|2016-06-03 00:00:00|2099.129883| 0.4897281051188776| 0.4897281| 0.4897281051188776| 0.02351315613798889|
|2016-06-06 00:00:00|2109.409912| 0.12894463918684698| 0.12894464| 0.12894463918684698|0.009835733345160483|
|2016-06-07 00:00:00|2112.129883| 0.3309566355867838| 0.33095664| 0.3309566355867838|0.019964682614502356|
|2016-06-08 00:00:00|2119.120117| -0.17177586918259| -0.17177586| -0.17177586918259|0.017393694108454903|
|2016-06-09 00:00:00| 2115.47998| -0.91751811331252| -0.91751814| -0.91751811331252|0.018536162926822274|
|2016-06-10 00:00:00|2096.070068| -0.8115191023280278| -0.8115191| -0.8115191023280278| 0.05633561630601992|
|2016-06-13 00:00:00|2079.060059|-0.17988855029992754| -0.17988855|-0.17988855029992754| 0.11097605197476765|
|2016-06-14 00:00:00|2075.320068| -0.1840712697237789| -0.18407127| -0.1840712697237789| 0.09879315490396504|
|2016-06-15 00:00:00| 2071.5| 0.31329905865315155| 0.31329906| 0.31329905865315155| 0.11935050114228699|
|2016-06-16 00:00:00| 2077.98999|-0.32579651646926594| -0.3257965|-0.32579651646926594| 0.07185903078496324|
|2016-06-17 00:00:00|2071.219971| 0.5808185112367291| 0.58081853| 0.5808185112367291| 0.08797421251272414|
|2016-06-20 00:00:00| 2083.25| 0.27120614424577516| 0.27120614| 0.27120614424577516|0.054005451213320836|
|2016-06-21 00:00:00|2088.899902|-0.16515635798043402| -0.16515636|-0.16515635798043402|0.044820264715194694|
|2016-06-22 00:00:00|2085.449951| 1.3364078570495475| 1.3364079| 1.3364078570495475| 0.02463802578115385|
|2016-06-23 00:00:00|2113.320068| -3.591979991551379| 3.59198| -3.591979991551379|-0.02182646537897...|
|2016-06-24 00:00:00|2037.410034| -1.809650212020111| -1.8096502| -1.809650212020111| 0.10095604965763462|
|2016-06-27 00:00:00|2000.540039| 1.7770165208875395| 1.7770165| 1.7770165208875395| 0.2094328464153043|
|2016-06-28 00:00:00|2036.089966| 1.7032672710494525| 1.7032672| 1.7032672710494525| 0.1400925193122321|
|2016-06-29 00:00:00| 2070.77002| 1.3565044272758033| 1.3565044| 1.3565044272758033| 0.03553169717026959|
|2016-06-30 00:00:00|2098.860107| 0.19486024753912218| -0.19486025| 0.19486024753912218|-0.01368343670679311|
|2016-07-01 00:00:00|2102.949951| -0.6847477275031026| 0.68474776| -0.6847477275031026|-0.10564751325612898|
|2016-07-05 00:00:00|2088.550049| 0.5352962934909359| 0.5352963| 0.5352962934909359| 0.03261622937548151|
|2016-07-06 00:00:00| 2099.72998|-0.08715777825870533| 0.08715778|-0.08715777825870533|-0.01321487882376...|
|2016-07-07 00:00:00|2097.899902| 1.5253349299217422| 1.525335| 1.5253349299217422| 0.00970844732109375|
|2016-07-08 00:00:00|2129.899902| 0.34086155847900335| -0.34086156| 0.34086155847900335|-0.10001499998621433|
|2016-07-11 00:00:00|2137.159912| 0.7009293462734553| -0.70092934| 0.7009293462734553|-0.09390854632526155|
|2016-07-12 00:00:00|2152.139893|0.013476772627251273| -0.013476772|0.013476772627251273|-0.06273252566241166|
|2016-07-13 00:00:00|2152.429932| 0.5259203949780415| -0.5259204| 0.5259203949780415|-0.00672463383634115|
|2016-07-14 00:00:00| 2163.75|-0.09289474292316421| 0.09289474|-0.09289474292316421|-0.05280834398602...|
|2016-07-15 00:00:00| 2161.73999| 0.23822952916738127| 0.23822953| 0.23822952916738127|0.007331306659500034|
|2016-07-18 00:00:00|2166.889893|-0.14351739837110478| 0.1435174|-0.14351739837110478|-0.05643722946827...|
|2016-07-19 00:00:00|2163.780029| 0.4270300527854589| 0.42703006| 0.4270300527854589|0.029135028432005737|
|2016-07-20 00:00:00| 2173.02002| -0.3612529073708161| 0.3612529| -0.3612529073708161|-0.02136851154742187|
|2016-07-21 00:00:00|2165.169922| 0.45539645178943006| 0.45539644| 0.45539645178943006| 0.05862839777815157|
|2016-07-22 00:00:00|2175.030029|-0.30114752038671483| -0.30114752|-0.30114752038671483|0.003872739063839385|
|2016-07-25 00:00:00| 2168.47998| 0.03227846263076587| 0.032278463| 0.03227846263076587| 0.05575364233210528|
|2016-07-26 00:00:00|2169.179932|-0.11985423438815265| -0.119854234|-0.11985423438815265|0.022278141407032374|
|2016-07-27 00:00:00|2166.580078| 0.16062092674702202| 0.16062093| 0.16062092674702202| 0.066877453017787|
|2016-07-28 00:00:00|2170.060059| 0.16313092282023237| 0.16313092| 0.16313092282023237| 0.02084907817433158|
|2016-07-29 00:00:00|2173.600098| -0.1269787392142437| -0.12697874| -0.1269787392142437|0.055164186406040205|
|2016-08-01 00:00:00|2170.840088| -0.6361619668044343| -0.636162| -0.6361619668044343| 0.01871067760577626|
|2016-08-02 00:00:00|2157.030029| 0.31339433893434987| 0.31339434| 0.31339433893434987| 0.0827891400460544|
|2016-08-03 00:00:00|2163.790039|0.021257191858254098| 0.021257192|0.021257191858254098| 0.0418932780383957|
|2016-08-04 00:00:00| 2164.25| 0.8603496361326065| 0.86034966| 0.8603496361326065| 0.06480785940636602|
|2016-08-05 00:00:00|2182.870117| -0.0907165288753599| -0.090716526| -0.0907165288753599|0.005081318066857778|
|2016-08-08 00:00:00|2180.889893|0.038979363549189745| 0.038979363|0.038979363549189745|0.017506053723946088|
|2016-08-09 00:00:00| 2181.73999|-0.28646859977113953| -0.2864686|-0.28646859977113953|0.003943602897653...|
|2016-08-10 00:00:00| 2175.48999| 0.4734588091577449| 0.4734588| 0.4734588091577449| 0.0506663638782674|
|2016-08-11 00:00:00|2185.790039|-0.07960462665462967| -0.079604626|-0.07960462665462967|0.007254849084550381|
|2016-08-12 00:00:00|2184.050049| 0.27929089824626846| 0.27929088| 0.27929089824626846| 0.04162200191308573|
|2016-08-15 00:00:00|2190.149902| -0.5479077020728967| 0.5479077| -0.5479077020728967|-0.01186933063353366|
|2016-08-16 00:00:00|2178.149902| 0.18685899424381655| 0.186859| 0.18685899424381655|0.059713797958216595|
|2016-08-17 00:00:00|2182.219971| 0.2199617391366979| 0.21996173| 0.2199617391366979| 0.02751569028115043|
|2016-08-18 00:00:00| 2187.02002|-0.14402716807320287| -0.14402717|-0.14402716807320287| 0.06191917075263412|
|2016-08-19 00:00:00|2183.870117| -0.0563322878235028| -0.056332286| -0.0563322878235028|0.023723971774663854|
|2016-08-22 00:00:00|2182.639893| 0.19517690543742305| 0.1951769| 0.19517690543742305|0.056532929973085906|
|2016-08-23 00:00:00|2186.899902| -0.52402768821378| -0.5240277| -0.52402768821378|0.017734810601854903|
|2016-08-24 00:00:00|2175.439941|-0.13652273013958158| -0.13652273|-0.13652273013958158| 0.07719527924246394|
|2016-08-25 00:00:00|2172.469971|-0.15788167596264593| -0.15788168|-0.15788167596264593| 0.05572356653909286|
|2016-08-26 00:00:00|2169.040039| 0.5228047337119758| 0.52280474| 0.5228047337119758| 0.07452695447099067|
|2016-08-29 00:00:00|2180.379883| -0.1953680655931885| -0.19536807| -0.1953680655931885| 0.03669524115740061|
|2016-08-30 00:00:00|2176.120117|-0.23758642547395195| -0.23758642|-0.23758642547395195|0.053912928496864576|
|2016-08-31 00:00:00|2170.949951|-0.00413846482083...| -0.004138465|-0.00413846482083...| 0.04602896318380502|
|2016-09-01 00:00:00|2170.860107| 0.4201041315648488| 0.42010415| 0.4201041315648488| 0.06396138233525586|
|2016-09-02 00:00:00| 2179.97998| 0.2981678758352634| 0.29816788| 0.2981678758352634|0.027061272029217553|
|2016-09-06 00:00:00| 2186.47998|-0.01463850585999...| -0.014638506|-0.01463850585999...| 0.03507840827130089|
|2016-09-07 00:00:00|2186.159912| -0.2223013501127694| -0.22230135| -0.2223013501127694|0.001104900755447...|
|2016-09-08 00:00:00|2181.300049| -2.4522068857295496| -2.4522069| -2.4522068857295496| 0.03586942869105701|
|2016-09-09 00:00:00|2127.810059| 1.467705252539182| 1.4677052| 1.467705252539182| 0.13003750766460198|
|2016-09-12 00:00:00|2159.040039| -1.4830674013266898| -1.4830674| -1.4830674013266898| 0.09179040546568898|
|2016-09-13 00:00:00| 2127.02002|-0.05876766500768526| -0.058767665|-0.05876766500768526| 0.13538445687768375|
|2016-09-14 00:00:00| 2125.77002| 1.0109273250546658| 1.0109273| 1.0109273250546658| 0.10299656562896597|
|2016-09-15 00:00:00| 2147.26001| -0.3772294907126729| -0.37722948| -0.3772294907126729| 0.08981405890727896|
|2016-09-16 00:00:00|2139.159912|-0.00186030973079...|-0.0018603097|-0.00186030973079...| 0.03297684927572516|
|2016-09-19 00:00:00|2139.120117|0.029913841439509457| 0.02991384|0.029913841439509457| 0.12522061779256152|
|2016-09-20 00:00:00| 2139.76001| 1.0917162154086604| -1.0917162| 1.0917162154086604|-0.00657471545100...|
|2016-09-21 00:00:00|2163.120117| 0.6499784681166678| 0.64997846| 0.6499784681166678| 0.04344795902070058|
|2016-09-22 00:00:00|2177.179932| -0.5736774814255415| 0.5736775| -0.5736774814255415| -0.0670728283114628|
|2016-09-23 00:00:00|2164.689941| -0.8587762454059558| -0.8587763| -0.8587762454059558|0.002278092915136469|
|2016-09-26 00:00:00|2146.100098| 0.6444170061260637| 0.644417| 0.6444170061260637| 0.03851714501551536|
|2016-09-27 00:00:00|2159.929932| 0.5296553758763286| 0.5296554| 0.5296553758763286| 0.01956469868906461|
|2016-09-28 00:00:00|2171.370117| -0.9321411325289909| -0.9321411| -0.9321411325289909| 0.0411228838312013|
|2016-09-29 00:00:00|2151.129883| 0.7967969361336731| 0.7967969| 0.7967969361336731| 0.03418411730268428|
|2016-09-30 00:00:00| 2168.27002| -0.3260695824222057| -0.3260696| -0.3260695824222057| 0.01894587817268246|
|2016-10-03 00:00:00|2161.199951|-0.49555623000289534| -0.49555624|-0.49555623000289534| 0.04060719379041892|
|2016-10-04 00:00:00| 2150.48999| 0.4296690541675125| 0.42966905| 0.4296690541675125| 0.08214547459166668|
|2016-10-05 00:00:00| 2159.72998| 0.04815601994837626| 0.04815602| 0.04815601994837626| 0.05618972707504136|
|2016-10-06 00:00:00| 2160.77002| -0.325348368171079| -0.32534838| -0.325348368171079|0.019473338329135687|
|2016-10-07 00:00:00| 2153.73999| 0.4605905098135844| 0.4605905| 0.4605905098135844| 0.06448828115009853|
|2016-10-10 00:00:00|2163.659912| -1.2446471763257407| -1.2446471| -1.2446471763257407| 0.01865659611688069|
|2016-10-11 00:00:00| 2136.72998| 0.11465894253984957| 0.114658944| 0.11465894253984957| 0.10060441412799451|
|2016-10-12 00:00:00|2139.179932|-0.30992638350910184| -0.3099264|-0.30992638350910184| 0.06813560495887139|
|2016-10-13 00:00:00|2132.550049|0.020160417815362805| 0.020160418|0.020160417815362805| 0.09740153124660542|
|2016-10-14 00:00:00| 2132.97998|-0.30379938212078617| -0.3037994|-0.30379938212078617| 0.06847460331341597|
|2016-10-17 00:00:00| 2126.5| 0.6160403479896491| 0.61604035| 0.6160403479896491| 0.09284634450633844|
|2016-10-18 00:00:00|2139.600098| 0.2191970828746938| 0.21919708| 0.2191970828746938|0.028563271274506367|
|2016-10-19 00:00:00|2144.290039| -0.1375723874264593| -0.1375724| -0.1375723874264593| 0.06667846133285449|
|2016-10-20 00:00:00|2141.340088|-0.00841417022030...| -0.00841417|-0.00841417022030...|0.008403406831473961|
|2016-10-21 00:00:00|2141.159912| 0.4749839534638121| 0.47498396| 0.4749839534638121|0.057633465109647175|
|2016-10-24 00:00:00|2151.330078|-0.37977277794560177| 0.37977278|-0.37977277794560177|-0.01019980342886...|
|2016-10-25 00:00:00|2143.159912|-0.17404114266579604| -0.17404114|-0.17404114266579604| 0.04775981301407148|
|2016-10-26 00:00:00|2139.429932| -0.2986726933387613| -0.2986727| -0.2986726933387613|0.027506605662294098|
|2016-10-27 00:00:00|2133.040039| -0.3108299365589125| -0.31082994| -0.3108299365589125| 0.0629669853941767|
|2016-10-28 00:00:00|2126.409912|-0.01222765180563...| -0.012227652|-0.01222765180563...| 0.06511662557856555|
|2016-10-31 00:00:00|2126.149902| -0.6786883176217423| -0.67868835| -0.6786883176217423| 0.08620427809391291|
|2016-11-01 00:00:00|2111.719971| -0.6525500629458174| -0.65255004| -0.6525500629458174| 0.08581020570495212|
|2016-11-02 00:00:00|2097.939941|-0.44234006982948293| -0.44234008|-0.44234006982948293| 0.12052389382225526|
|2016-11-03 00:00:00|2088.659912|-0.16661305078948002| -0.16661306|-0.16661305078948002| 0.12095496150285939|
|2016-11-04 00:00:00|2085.179932| 2.222354401596069| 2.2223544| 2.222354401596069| 0.1274469686705091|
|2016-11-07 00:00:00| 2131.52002| 0.3771974424148256| 0.37719744| 0.3771974424148256| 0.01727894118614634|
|2016-11-08 00:00:00|2139.560059| 1.1077020670818223| -1.107702| 1.1077020670818223|-0.02383686828285326|
|2016-11-09 00:00:00| 2163.26001| 0.1950745624886813| -0.19507456| 0.1950745624886813|-0.06903791587186517|
|2016-11-10 00:00:00| 2167.47998| -0.139795016699532| 0.13979502| -0.139795016699532|-0.06087901134116...|
|2016-11-11 00:00:00|2164.449951|-0.01155027862319...| 0.011550278|-0.01155027862319...|-0.05195444638695...|
|2016-11-14 00:00:00|2164.199951| 0.7480797692708188| -0.7480798| 0.7480797692708188|-7.79633808628769...|
|2016-11-15 00:00:00|2180.389893|-0.15822638011099693| 0.15822639|-0.15822638011099693|-0.07746632876706516|
|2016-11-16 00:00:00|2176.939941| 0.4676369709732771| 0.46763697| 0.4676369709732771|0.005814145745197...|
|2016-11-17 00:00:00|2187.120117|-0.23867984933357117| 0.23867986|-0.23867984933357117|-0.04491788988134...|
|2016-11-18 00:00:00|2181.899902| 0.7461400949272281| 0.7461401| 0.7461400949272281| 0.04385237610014097|
|2016-11-21 00:00:00|2198.179932| 0.21654319242507222| -0.2165432| 0.21654319242507222|-0.01178749539393...|
|2016-11-22 00:00:00|2202.939941| 0.0808024752228094| 0.08080248| 0.0808024752228094|0.017541231477313436|
|2016-11-23 00:00:00|2204.719971| 0.39143869124048053| -0.3914387| 0.39143869124048053|-0.02052102829776...|
|2016-11-25 00:00:00|2213.350098| -0.5254535651865005| -0.52545357| -0.5254535651865005|0.005803896627152853|
|2016-11-28 00:00:00|2201.719971| 0.13352928795321764| 0.13352929| 0.13352928795321764|0.001236510710497...|
|2016-11-29 00:00:00|2204.659912| -0.2653403805348535| -0.2653404| -0.2653403805348535|0.051704179975071575|
|2016-11-30 00:00:00|2198.810059| -0.3515529214704194| -0.35155293| -0.3515529214704194|0.029759298143040534|
|2016-12-01 00:00:00|2191.080078|0.039700648494513754| 0.03970065|0.039700648494513754| 0.06804129066701364|
|2016-12-02 00:00:00|2191.949951| 0.5821305360635018| 0.58213055| 0.5821305360635018| 0.0632553785160296|
|2016-12-05 00:00:00|2204.709961| 0.34108881136406527| 0.3410888| 0.34108881136406527| 0.03340206225641486|
|2016-12-06 00:00:00| 2212.22998| 1.3163241734930209| 1.3163241| 1.3163241734930209| 0.03660868441958645|
|2016-12-07 00:00:00|2241.350098| 0.2159342712376298| -0.21593428| 0.2159342712376298|-0.04433504115778635|
|2016-12-08 00:00:00|2246.189941| 0.5938984836723531| -0.5938985| 0.5938984836723531|-0.02892054885877942|
|2016-12-09 00:00:00|2259.530029|-0.11374347616603382| 0.11374348|-0.11374347616603382|-0.06386592131010393|
|2016-12-12 00:00:00|2256.959961| 0.6539774854251376| -0.6539775| 0.6539774854251376|-0.02497108743452...|
|2016-12-13 00:00:00|2271.719971| -0.8117172114256143| 0.8117172| -0.8117172114256143|-0.05741948077553...|
|2016-12-14 00:00:00|2253.280029| 0.38832279554189403| 0.3883228| 0.38832279554189403|0.017646025432132013|
|2016-12-15 00:00:00|2262.030029|-0.17506226483432866| 0.17506227|-0.17506226483432866|-0.01763929406028495|
|2016-12-16 00:00:00|2258.070068| 0.19751207295131734| 0.19751208| 0.19751207295131734|0.053183155041379666|
|2016-12-19 00:00:00|2262.530029| 0.36375123841505275| 0.36375123| 0.36375123841505275|0.007538061754018...|
|2016-12-20 00:00:00| 2270.76001|-0.24573614012164843| -0.24573614|-0.24573614012164843| 0.05659128849926827|
|2016-12-21 00:00:00|2265.179932|-0.18629738593322426| -0.18629739|-0.18629738593322426|0.005844312293819924|
|2016-12-22 00:00:00|2260.959961| 0.12517152222139497| 0.12517153| 0.12517152222139497| 0.07570360878826678|
|2016-12-23 00:00:00|2263.790039| 0.22483728227059685| 0.22483729| 0.22483728227059685|0.021483502619055662|
|2016-12-27 00:00:00|2268.879883| -0.8356529202828681| -0.83565295| -0.8356529202828681| 0.05333195679616798|
|2016-12-28 00:00:00|2249.919922|-0.02933046610003201| -0.029330466|-0.02933046610003201| 0.05191400429652269|
|2016-12-29 00:00:00| 2249.26001|-0.46370503870737506| -0.46370503|-0.46370503870737506| 0.06713886320955127|
|2016-12-30 00:00:00|2238.830078| 0.8486575281753026| 0.84865755| 0.8486575281753026| 0.08970054534422042|
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+
Linear Regression Accuracy:
0.5119048
Long Only Accuracy:
0.52380955
True Positive Count:
+---+
|tpc|
+---+
| 99|
+---+
False Positive Count:
+---+
|fpc|
+---+
| 90|
+---+
True Negative Count:
+---+
|tnc|
+---+
| 30|
+---+
False Negative Count:
+---+
|fnc|
+---+
| 33|
+---+
Effectiveness of negative predictions:
+------------------+
| eff_np|
+------------------+
|-2.467898511452154|
+------------------+
Effectiveness of positive predictions:
+-----------------+
| eff_pp|
+-----------------+
|9.875746989931645|
+-----------------+
Effectiveness of Long-Only-Model:
+------------------+
| eff_lo|
+------------------+
|12.343645501383794|
+------------------+
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.2.1
/_/
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_152)
Type in expressions to have them evaluated.
Type :help for more information.
scala>
scala>
For 2016 test data, I translate the above report into English: