Class04 Answer:

Copy logr12o.scala to linr11a.scala. Enhance it to use Linear Regression.

This lab required some knowledge.

{
/*
~/ml4/public/class04/linr10/linr11a.scala
This script should download prices and predict daily direction of GSPC.
It should generate independent features from slopes of moving averages of prices.
It should create a Linear Regression model from many years of features.
Demo:
spark-shell -i linr11a.scala
*/

import org.apache.spark.sql.SQLContext
import org.apache.spark.ml.regression.LinearRegression
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.sql.Row
import sys.process._

// I should declare training_period, test_period

val training_period = " WHERE Date BETWEEN'1986-01-01'AND'2016-01-01' "
val test_period     = " WHERE Date BETWEEN'2016-01-01'AND'2017-01-01' "

// I should get prices:
"/usr/bin/curl -L ml4.herokuapp.com/csv/GSPC.csv -o /tmp/gspc.csv"!

val sqlContext = new SQLContext(sc)
  
val dp10df = sqlContext
  .read
  .format("com.databricks.spark.csv")
  .option("header","true")
  .option("inferSchema","true")
  .load("/tmp/gspc.csv")

dp10df.createOrReplaceTempView("tab")

spark.sql("SELECT COUNT(Date),MIN(Date),MAX(Date),MIN(Close),MAX(Close)FROM tab").show

// I should declare a dependent variable: "pctlead".
// If it truely is dependent that makes it predictable.

var sqls="SELECT Date,Close,LEAD(Close,1)OVER(ORDER BY Date) leadp FROM tab ORDER BY Date"

val dp11df=spark.sql(sqls);dp11df.createOrReplaceTempView("tab")

sqls="SELECT Date,Close,100*(leadp-Close)/Close pctlead FROM tab ORDER BY Date"

val dp12df=spark.sql(sqls);dp12df.createOrReplaceTempView("tab")

// I should now collect features AKA independent variables.

sqls = "SELECT Date, Close, pctlead"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS mavg2"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 3 PRECEDING AND CURRENT ROW) AS mavg3"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS mavg4"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS mavg5"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS mavg6"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 7 PRECEDING AND CURRENT ROW) AS mavg7"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 8 PRECEDING AND CURRENT ROW) AS mavg8"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 9 PRECEDING AND CURRENT ROW) AS mavg9"
sqls=sqls++" FROM tab ORDER BY Date"

val dp13df=spark.sql(sqls);dp13df.createOrReplaceTempView("tab")

// In Linear Regression, I should use pctlead as the label:
  
sqls = "SELECT Date, Close, pctlead, pctlead label "
sqls=sqls++",(mavg2-LAG(mavg2,1)OVER(ORDER BY Date))/mavg2 AS slp2 "
sqls=sqls++",(mavg3-LAG(mavg3,1)OVER(ORDER BY Date))/mavg3 AS slp3 "
sqls=sqls++",(mavg4-LAG(mavg4,1)OVER(ORDER BY Date))/mavg4 AS slp4 "
sqls=sqls++",(mavg5-LAG(mavg5,1)OVER(ORDER BY Date))/mavg5 AS slp5 "
sqls=sqls++",(mavg6-LAG(mavg6,1)OVER(ORDER BY Date))/mavg6 AS slp6 "
sqls=sqls++",(mavg7-LAG(mavg7,1)OVER(ORDER BY Date))/mavg7 AS slp7 "
sqls=sqls++",(mavg8-LAG(mavg8,1)OVER(ORDER BY Date))/mavg8 AS slp8 "
sqls=sqls++",(mavg9-LAG(mavg9,1)OVER(ORDER BY Date))/mavg9 AS slp9 "
sqls=sqls++" FROM tab ORDER BY Date"

val dp15df=spark.sql(sqls)

// I should copy slp-values into Vectors.dense():

val fill_vec = udf((
  slp2:Float
  ,slp3:Float
  ,slp4:Float
  ,slp5:Float
  ,slp6:Float
  ,slp7:Float
  ,slp8:Float
  ,slp9:Float
  )=> {Vectors.dense(
  slp2
  ,slp3
  ,slp4
  ,slp5
  ,slp6
  ,slp7
  ,slp8
  ,slp9
  )
  }
)

val dp16df = dp15df.withColumn("features"
,fill_vec(
  col("slp2")
  ,col("slp3")
  ,col("slp4")
  ,col("slp5")
  ,col("slp6")
  ,col("slp7")
  ,col("slp8")
  ,col("slp9")
  )
)

// I should gather observations to learn from:
dp16df.createOrReplaceTempView("tab")

// I should create a LinearRegression instance. This instance is an 'Estimator'.

val lr = new LinearRegression()

lr.setMaxIter(1234).setRegParam(0.01)

// I should gather observations to learn from:
dp16df.createOrReplaceTempView("tab")

val train_df = spark.sql("SELECT * FROM tab"++training_period)

/*I should fit a LinearRegression model to observations.
This uses the parameters stored in lr.*/

val model1 = lr.fit(train_df)

val test_df = spark.sql("SELECT * FROM tab"++test_period)

/* I should predict. It is convenient that predictions_df will contain a copy of test_df.*/

val predictions_df = model1.transform(test_df)
predictions_df.createOrReplaceTempView("tab")

// eff logic
val eff = udf((pctlead:Float,prediction:Double)=> {if (prediction>0) pctlead else -pctlead})

// prediction report:
val p2df = predictions_df.withColumn("effectiveness",eff(col("pctlead"),col("prediction")))

println("Predictions Report:")
p2df.select("Date","Close","pctlead","effectiveness","label","prediction").show(255)

// I should report in a way which is consistent with the python script linr11a.py:

// I should get Linear Regression Accuracy:
sqls = "SELECT COUNT(Date) pc FROM tab WHERE (prediction>0 AND pctlead>0)OR(prediction<=0 AND pctlead<=0)"

val pc_df = spark.sql(sqls)

val poscount_i = pc_df.first()(0).asInstanceOf[Long]

sqls = "SELECT COUNT(Date) rcount FROM tab"

val rc_df = spark.sql(sqls)

val rcount_i = rc_df.first()(0).asInstanceOf[Long]

println("Linear Regression Accuracy:")
println(poscount_i.toFloat/rcount_i)

// I should get Long-Only Accuracy:
sqls = "SELECT COUNT(Date) upcount FROM tab WHERE pctlead>0"

val upcount_df = spark.sql(sqls)
val upcount_i  = upcount_df.first()(0).asInstanceOf[Long]

println("Long Only Accuracy:")
println(upcount_i.toFloat/rcount_i)

val tp_df = spark.sql("SELECT COUNT(Date) tpc FROM tab WHERE prediction>0 AND pctlead>0")
println("True Positive Count:")
tp_df.show

val fp_df = spark.sql("SELECT COUNT(Date) fpc FROM tab WHERE prediction>0 AND pctlead<0")
println("False Positive Count:")
fp_df.show

val tn_df = spark.sql("SELECT COUNT(Date) tnc FROM tab WHERE prediction<=0 AND pctlead<=0")
println("True Negative Count:")
tn_df.show

val fn_df = spark.sql("SELECT COUNT(Date) fnc FROM tab WHERE prediction<=0 AND pctlead>0")
println("False Negative Count:")
fn_df.show

println("Effectiveness of negative predictions:")
spark.sql("SELECT -SUM(pctlead) eff_np FROM tab WHERE prediction<=0").show

println("Effectiveness of positive predictions:")
spark.sql("SELECT SUM(pctlead) eff_pp FROM tab WHERE prediction>0").show

println("Effectiveness of Long-Only-Model:")
spark.sql("SELECT SUM(pctlead) eff_lo FROM tab").show

}

I saw something like this:


dan@h80:~/ml4/public/class04/linr10 $ 
dan@h80:~/ml4/public/class04/linr10 $ spark-shell -i linr11a.scala
Spark context Web UI available at http://192.168.1.80:4042
Spark context available as 'sc' (master = local[*], app id = local-1515896821987).
Spark session available as 'spark'.
Loading linr11a.scala...
warning: there was one deprecation warning; re-run with -deprecation for details
warning: there was one feature warning; re-run with -feature for details
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 1252k  100 1252k    0     0   420k      0  0:00:02  0:00:02 --:--:--  829k
+-----------+-------------------+-------------------+----------+----------+
|count(Date)|          min(Date)|          max(Date)|min(Close)|max(Close)|
+-----------+-------------------+-------------------+----------+----------+
|      17116|1950-01-03 00:00:00|2018-01-09 00:00:00|     16.66|2753.52002|
+-----------+-------------------+-------------------+----------+----------+

Predictions Report:
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+
|               Date|      Close|             pctlead|effectiveness|               label|          prediction|
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+
|2016-01-04 00:00:00|2012.660034|  0.2012226074739071|   0.20122261|  0.2012226074739071| 0.08703354200252766|
|2016-01-05 00:00:00|2016.709961|   -1.31153966170151|   -1.3115396|   -1.31153966170151| 0.13291345762180573|
|2016-01-06 00:00:00| 1990.26001| -2.3700443039098174|   -2.3700442| -2.3700443039098174| 0.19395261892507978|
|2016-01-07 00:00:00|1943.089966|  -1.083837463447639|   -1.0838375|  -1.083837463447639| 0.23275401465193407|
|2016-01-08 00:00:00|1922.030029| 0.08532723085774163|   0.08532723| 0.08532723085774163|  0.2907372292887592|
|2016-01-11 00:00:00|1923.670044|  0.7802798638371875|    0.7802799|  0.7802798638371875|  0.2194794015832985|
|2016-01-12 00:00:00|1938.680054| -2.4965452602732516|   -2.4965453| -2.4965452602732516| 0.20729411675587486|
|2016-01-13 00:00:00|1890.280029|   1.669590564139637|    1.6695906|   1.669590564139637| 0.18928009706982082|
|2016-01-14 00:00:00|1921.839966|  -2.159909812178397|   -2.1599097|  -2.159909812178397| 0.11450307354324421|
|2016-01-15 00:00:00|1880.329956|0.053182155440808176|  0.053182155|0.053182155440808176|  0.1831658929511503|
|2016-01-19 00:00:00|1881.329956| -1.1693855152753438|   -1.1693856| -1.1693855152753438| 0.14274114868059404|
|2016-01-20 00:00:00|1859.329956|  0.5195438264643328|    0.5195438|  0.5195438264643328|  0.1952913927423723|
|2016-01-21 00:00:00| 1868.98999|  2.0283700930896904|    2.0283701|  2.0283700930896904| 0.03788587874095083|
|2016-01-22 00:00:00|1906.900024| -1.5637981868314241|   -1.5637982| -1.5637981868314241| 0.09534460272886333|
|2016-01-25 00:00:00|1877.079956|   1.414433568220359|   -1.4144336|   1.414433568220359|-0.01898306117525...|
|2016-01-26 00:00:00|1903.630005| -1.0863483946818713|   -1.0863484| -1.0863483946818713|   0.082263005497833|
|2016-01-27 00:00:00|1882.949951|  0.5528577110863419|   -0.5528577|  0.5528577110863419| -0.0228316850334181|
|2016-01-28 00:00:00|1893.359985|  2.4760217481832956|    2.4760218|  2.4760217481832956| 0.06731711604837584|
|2016-01-29 00:00:00| 1940.23999|-0.04432364060283344|   0.04432364|-0.04432364060283344|-0.07761119452146861|
|2016-02-01 00:00:00|1939.380005| -1.8743091042644822|    1.8743091| -1.8743091042644822| -0.1212303038400652|
|2016-02-02 00:00:00|1903.030029|  0.4992038935398218|    0.4992039|  0.4992038935398218| 0.05981993907765358|
|2016-02-03 00:00:00|1912.530029|  0.1526732629409628|  -0.15267326|  0.1526732629409628|-0.02389321961047...|
|2016-02-04 00:00:00|1915.449951| -1.8481246133065947|   -1.8481246| -1.8481246133065947| 0.10026073424988755|
|2016-02-05 00:00:00|1880.050049| -1.4153935962584503|   -1.4153936| -1.4153935962584503| 0.08356500153135463|
|2016-02-08 00:00:00|1853.439941|-0.06636201005447463|  -0.06636201|-0.06636201005447463| 0.16589068577997923|
|2016-02-09 00:00:00|1852.209961|-0.01889505009523...|  -0.01889505|-0.01889505009523...|  0.1486704972373507|
|2016-02-10 00:00:00|1851.859985| -1.2301161634528224|   -1.2301161| -1.2301161634528224| 0.18057699151112724|
|2016-02-11 00:00:00|1829.079956|   1.951804943402921|     1.951805|   1.951804943402921| 0.19405673841146112|
|2016-02-12 00:00:00|1864.780029|   1.651665425466653|    1.6516654|   1.651665425466653| 0.07078914712905633|
|2016-02-16 00:00:00|1895.579956|   1.648043908731858|   -1.6480439|   1.648043908731858|-0.02464514699336...|
|2016-02-17 00:00:00|1926.819946| -0.4665713586089291|   0.46657136| -0.4665713586089291|-0.09270533419288834|
|2016-02-18 00:00:00|1917.829956|-0.00260330692217...|  0.002603307|-0.00260330692217...|-0.06664756778047445|
|2016-02-19 00:00:00|1917.780029|  1.4454197343192787|   -1.4454198|  1.4454197343192787|-0.10222544831330452|
|2016-02-22 00:00:00|     1945.5| -1.2454371626831184|    1.2454371| -1.2454371626831184|-0.06850050231012314|
|2016-02-23 00:00:00| 1921.27002|   0.443978665736949|  -0.44397867|   0.443978665736949|-0.05135699687537886|
|2016-02-24 00:00:00|1929.800049|   1.134827518081388|   -1.1348275|   1.134827518081388|-0.00246842982022...|
|2016-02-25 00:00:00|1951.699951|-0.18701143063153722|   0.18701144|-0.18701143063153722|-0.07676985643311593|
|2016-02-26 00:00:00|1948.050049| -0.8120976670040306|  -0.81209767| -0.8120976670040306|0.008288441173942403|
|2016-02-29 00:00:00| 1932.22998|  2.3868792264572924|    2.3868792|  2.3868792264572924| 0.06201598172965149|
|2016-03-01 00:00:00|1978.349976| 0.40943084379727995|  -0.40943083| 0.40943084379727995|-0.06099063734154...|
|2016-03-02 00:00:00|1986.449951|  0.3498740552965472|  -0.34987405|  0.3498740552965472| -0.0303992159107861|
|2016-03-03 00:00:00|1993.400024|  0.3305892405266673|  -0.33058923|  0.3305892405266673|-0.08424891647150909|
|2016-03-04 00:00:00| 1999.98999| 0.08850144294971853| -0.088501446| 0.08850144294971853|-0.03694079013555097|
|2016-03-07 00:00:00| 2001.76001|  -1.124010864818905|    1.1240109|  -1.124010864818905|-0.06936345770279465|
|2016-03-08 00:00:00| 1979.26001|  0.5052393293188397|    0.5052393|  0.5052393293188397| 0.06339013711755134|
|2016-03-09 00:00:00| 1989.26001|0.015580467030054408| -0.015580467|0.015580467030054408|-0.00160050231127...|
|2016-03-10 00:00:00|1989.569946|  1.6395500477669567|    1.6395501|  1.6395500477669567| 0.05327296438135176|
|2016-03-11 00:00:00|2022.189941|-0.12609725467921024|   0.12609726|-0.12609725467921024|-0.05676898986071...|
|2016-03-14 00:00:00|2019.640015| -0.1836941718546818|  -0.18369417| -0.1836941718546818|0.017847126028275276|
|2016-03-15 00:00:00|2015.930054|  0.5600351548705103|  -0.56003517|  0.5600351548705103|-0.01988184037645028|
|2016-03-16 00:00:00|2027.219971|  0.6595236427847927|   0.65952367|  0.6595236427847927|0.022653883473985583|
|2016-03-17 00:00:00|2040.589966| 0.44056435392665033|  -0.44056436| 0.44056435392665033|-0.04902630293293073|
|2016-03-18 00:00:00|2049.580078| 0.09855774954502321|  -0.09855775| 0.09855774954502321|-0.00723418141760...|
|2016-03-21 00:00:00|2051.600098|-0.08773878504659462|   0.08773878|-0.08773878504659462|-0.07562468053115229|
|2016-03-22 00:00:00|2049.800049| -0.6386031655324603|  -0.63860315| -0.6386031655324603| 0.00732724451524365|
|2016-03-23 00:00:00|2036.709961|-0.03780705229240...|  -0.03780705|-0.03780705229240...|0.029076751899825726|
|2016-03-24 00:00:00|2035.939941| 0.05452557698998718|  0.054525577| 0.05452557698998718| 0.06466875013814474|
|2016-03-28 00:00:00|2037.050049|  0.8816651809226127|   0.88166517|  0.8816651809226127| 0.04922052322862414|
|2016-03-29 00:00:00| 2055.01001| 0.43503150624556275|    0.4350315| 0.43503150624556275|0.007130135296180666|
|2016-03-30 00:00:00|2063.949951|-0.20397592480187138|  -0.20397593|-0.20397592480187138|0.005694311725338394|
|2016-03-31 00:00:00| 2059.73999|  0.6330915097686665|    0.6330915|  0.6330915097686665|0.006856114882150...|
|2016-04-01 00:00:00|2072.780029| -0.3208322111829817|  -0.32083222| -0.3208322111829817|0.007013387843252...|
|2016-04-04 00:00:00|2066.129883|  -1.014449245057461|   -1.0144492|  -1.014449245057461|0.006543372923626847|
|2016-04-05 00:00:00|2045.170044|  1.0507619189438957|    1.0507619|  1.0507619189438957| 0.06516495716196352|
|2016-04-06 00:00:00|2066.659912| -1.1975786560861146|   -1.1975787| -1.1975786560861146| 0.01141639075874383|
|2016-04-07 00:00:00|2041.910034|   0.278657820631483|   0.27865782|   0.278657820631483| 0.06997303231483816|
|2016-04-08 00:00:00|2047.599976| -0.2739786123146514|  -0.27397862| -0.2739786123146514| 0.06929975770371326|
|2016-04-11 00:00:00| 2041.98999|  0.9662134044055697|    0.9662134|  0.9662134044055697| 0.08119287920938206|
|2016-04-12 00:00:00|2061.719971|  1.0040137017230288|    1.0040137|  1.0040137017230288| 0.02643084427519642|
|2016-04-13 00:00:00|2082.419922|0.017292717774910475|  0.017292717|0.017292717774910475| 0.01580658626332606|
|2016-04-14 00:00:00|2082.780029|-0.09842849323767663|  0.098428495|-0.09842849323767663|-0.05232175241872806|
|2016-04-15 00:00:00| 2080.72998|  0.6541025568343978|   0.65410256|  0.6541025568343978|0.027500149099175995|
|2016-04-18 00:00:00|2094.340088|   0.308448519751584|  -0.30844852|   0.308448519751584|-0.06127076307281...|
|2016-04-19 00:00:00|2100.800049| 0.07615446318947447|   0.07615446| 0.07615446318947447|0.012110192941894679|
|2016-04-20 00:00:00|2102.399902| -0.5194027068595269|    0.5194027| -0.5194027068595269|-0.04266109733010199|
|2016-04-21 00:00:00| 2091.47998|0.004785988914887...|  0.004785989|0.004785988914887...|0.015459482221444672|
|2016-04-22 00:00:00|2091.580078| -0.1812045849864888|  -0.18120459| -0.1812045849864888|  0.0251550143477604|
|2016-04-25 00:00:00|2087.790039|  0.1872751534858758|   0.18727516|  0.1872751534858758| 0.06522087697392098|
|2016-04-26 00:00:00|2091.699951| 0.16493527182762052|   0.16493528| 0.16493527182762052| 0.04758615086083459|
|2016-04-27 00:00:00|2095.149902| -0.9230768157227635|   -0.9230768| -0.9230768157227635| 0.04279456522837545|
|2016-04-28 00:00:00|2075.810059| -0.5063088481738572|  -0.50630885| -0.5063088481738572|0.061796365290561595|
|2016-04-29 00:00:00|2065.300049|   0.780994655367873|   0.78099465|   0.780994655367873| 0.11235555391027519|
|2016-05-02 00:00:00|2081.429932| -0.8676638460102664|  -0.86766386| -0.8676638460102664| 0.06193097310800989|
|2016-05-03 00:00:00|2063.370117| -0.5936889314753995|   -0.5936889| -0.5936889314753995| 0.09474511762234014|
|2016-05-04 00:00:00|2051.120117|-0.02390079429950192| -0.023900794|-0.02390079429950192| 0.09313484314705445|
|2016-05-05 00:00:00|2050.629883| 0.31746391945074204|    0.3174639| 0.31746391945074204| 0.09431871215401959|
|2016-05-06 00:00:00|2057.139893| 0.07534966412709886|  0.075349666| 0.07534966412709886| 0.07792367359452629|
|2016-05-09 00:00:00|2058.689941|  1.2483643839788858|    1.2483643|  1.2483643839788858|  0.0762661313245832|
|2016-05-10 00:00:00|2084.389893| -0.9561518249023676|   0.95615184| -0.9561518249023676|-0.01624907708340706|
|2016-05-11 00:00:00|2064.459961|-0.01694651417848691| -0.016946515|-0.01694651417848691|0.039152202008452264|
|2016-05-12 00:00:00|2064.110107| -0.8478288992748931|   -0.8478289| -0.8478288992748931|  0.0224273861108146|
|2016-05-13 00:00:00|2046.609985|  0.9796652584981903|    0.9796653|  0.9796652584981903| 0.09870618490565738|
|2016-05-16 00:00:00|2066.659912| -0.9411297372666149|  -0.94112974| -0.9411297372666149|0.014763494256562149|
|2016-05-17 00:00:00|2047.209961|0.020517875938566795|  0.020517876|0.020517875938566795| 0.08849830166929842|
|2016-05-18 00:00:00|2047.630005|-0.37067077457677733|  -0.37067077|-0.37067077457677733|0.027330816371681752|
|2016-05-19 00:00:00|2040.040039|   0.601950391425627|    0.6019504|   0.601950391425627| 0.12325470735819105|
|2016-05-20 00:00:00|2052.320068|-0.20854588262009888|  -0.20854588|-0.20854588262009888|0.020875122918486963|
|2016-05-23 00:00:00|2048.040039|  1.3681382915580749|    1.3681383|  1.3681382915580749| 0.11132294743621687|
|2016-05-24 00:00:00|2076.060059|  0.6974740416216479|  -0.69747406|  0.6974740416216479|-0.05698175868220819|
|2016-05-25 00:00:00|2090.540039|-0.02104437091817...|  0.021044372|-0.02104437091817...|-0.00182163803926...|
|2016-05-26 00:00:00|2090.100098|  0.4286857365622697|  -0.42868572|  0.4286857365622697|-0.06943683886469904|
|2016-05-27 00:00:00|2099.060059|-0.10052632800821898| -0.100526325|-0.10052632800821898|2.239626658884635...|
|2016-05-31 00:00:00|2096.949951| 0.11350423498972209|  -0.11350424| 0.11350423498972209|-0.05039564083661529|
|2016-06-01 00:00:00|2099.330078|  0.2824678244809108|    0.2824678|  0.2824678244809108|0.014864126374432143|
|2016-06-02 00:00:00| 2105.26001| -0.2911814678890852|   0.29118147| -0.2911814678890852|-0.02693365243091201|
|2016-06-03 00:00:00|2099.129883|  0.4897281051188776|    0.4897281|  0.4897281051188776| 0.02351315613798889|
|2016-06-06 00:00:00|2109.409912| 0.12894463918684698|   0.12894464| 0.12894463918684698|0.009835733345160483|
|2016-06-07 00:00:00|2112.129883|  0.3309566355867838|   0.33095664|  0.3309566355867838|0.019964682614502356|
|2016-06-08 00:00:00|2119.120117|   -0.17177586918259|  -0.17177586|   -0.17177586918259|0.017393694108454903|
|2016-06-09 00:00:00| 2115.47998|   -0.91751811331252|  -0.91751814|   -0.91751811331252|0.018536162926822274|
|2016-06-10 00:00:00|2096.070068| -0.8115191023280278|   -0.8115191| -0.8115191023280278| 0.05633561630601992|
|2016-06-13 00:00:00|2079.060059|-0.17988855029992754|  -0.17988855|-0.17988855029992754| 0.11097605197476765|
|2016-06-14 00:00:00|2075.320068| -0.1840712697237789|  -0.18407127| -0.1840712697237789| 0.09879315490396504|
|2016-06-15 00:00:00|     2071.5| 0.31329905865315155|   0.31329906| 0.31329905865315155| 0.11935050114228699|
|2016-06-16 00:00:00| 2077.98999|-0.32579651646926594|   -0.3257965|-0.32579651646926594| 0.07185903078496324|
|2016-06-17 00:00:00|2071.219971|  0.5808185112367291|   0.58081853|  0.5808185112367291| 0.08797421251272414|
|2016-06-20 00:00:00|    2083.25| 0.27120614424577516|   0.27120614| 0.27120614424577516|0.054005451213320836|
|2016-06-21 00:00:00|2088.899902|-0.16515635798043402|  -0.16515636|-0.16515635798043402|0.044820264715194694|
|2016-06-22 00:00:00|2085.449951|  1.3364078570495475|    1.3364079|  1.3364078570495475| 0.02463802578115385|
|2016-06-23 00:00:00|2113.320068|  -3.591979991551379|      3.59198|  -3.591979991551379|-0.02182646537897...|
|2016-06-24 00:00:00|2037.410034|  -1.809650212020111|   -1.8096502|  -1.809650212020111| 0.10095604965763462|
|2016-06-27 00:00:00|2000.540039|  1.7770165208875395|    1.7770165|  1.7770165208875395|  0.2094328464153043|
|2016-06-28 00:00:00|2036.089966|  1.7032672710494525|    1.7032672|  1.7032672710494525|  0.1400925193122321|
|2016-06-29 00:00:00| 2070.77002|  1.3565044272758033|    1.3565044|  1.3565044272758033| 0.03553169717026959|
|2016-06-30 00:00:00|2098.860107| 0.19486024753912218|  -0.19486025| 0.19486024753912218|-0.01368343670679311|
|2016-07-01 00:00:00|2102.949951| -0.6847477275031026|   0.68474776| -0.6847477275031026|-0.10564751325612898|
|2016-07-05 00:00:00|2088.550049|  0.5352962934909359|    0.5352963|  0.5352962934909359| 0.03261622937548151|
|2016-07-06 00:00:00| 2099.72998|-0.08715777825870533|   0.08715778|-0.08715777825870533|-0.01321487882376...|
|2016-07-07 00:00:00|2097.899902|  1.5253349299217422|     1.525335|  1.5253349299217422| 0.00970844732109375|
|2016-07-08 00:00:00|2129.899902| 0.34086155847900335|  -0.34086156| 0.34086155847900335|-0.10001499998621433|
|2016-07-11 00:00:00|2137.159912|  0.7009293462734553|  -0.70092934|  0.7009293462734553|-0.09390854632526155|
|2016-07-12 00:00:00|2152.139893|0.013476772627251273| -0.013476772|0.013476772627251273|-0.06273252566241166|
|2016-07-13 00:00:00|2152.429932|  0.5259203949780415|   -0.5259204|  0.5259203949780415|-0.00672463383634115|
|2016-07-14 00:00:00|    2163.75|-0.09289474292316421|   0.09289474|-0.09289474292316421|-0.05280834398602...|
|2016-07-15 00:00:00| 2161.73999| 0.23822952916738127|   0.23822953| 0.23822952916738127|0.007331306659500034|
|2016-07-18 00:00:00|2166.889893|-0.14351739837110478|    0.1435174|-0.14351739837110478|-0.05643722946827...|
|2016-07-19 00:00:00|2163.780029|  0.4270300527854589|   0.42703006|  0.4270300527854589|0.029135028432005737|
|2016-07-20 00:00:00| 2173.02002| -0.3612529073708161|    0.3612529| -0.3612529073708161|-0.02136851154742187|
|2016-07-21 00:00:00|2165.169922| 0.45539645178943006|   0.45539644| 0.45539645178943006| 0.05862839777815157|
|2016-07-22 00:00:00|2175.030029|-0.30114752038671483|  -0.30114752|-0.30114752038671483|0.003872739063839385|
|2016-07-25 00:00:00| 2168.47998| 0.03227846263076587|  0.032278463| 0.03227846263076587| 0.05575364233210528|
|2016-07-26 00:00:00|2169.179932|-0.11985423438815265| -0.119854234|-0.11985423438815265|0.022278141407032374|
|2016-07-27 00:00:00|2166.580078| 0.16062092674702202|   0.16062093| 0.16062092674702202|   0.066877453017787|
|2016-07-28 00:00:00|2170.060059| 0.16313092282023237|   0.16313092| 0.16313092282023237| 0.02084907817433158|
|2016-07-29 00:00:00|2173.600098| -0.1269787392142437|  -0.12697874| -0.1269787392142437|0.055164186406040205|
|2016-08-01 00:00:00|2170.840088| -0.6361619668044343|    -0.636162| -0.6361619668044343| 0.01871067760577626|
|2016-08-02 00:00:00|2157.030029| 0.31339433893434987|   0.31339434| 0.31339433893434987|  0.0827891400460544|
|2016-08-03 00:00:00|2163.790039|0.021257191858254098|  0.021257192|0.021257191858254098|  0.0418932780383957|
|2016-08-04 00:00:00|    2164.25|  0.8603496361326065|   0.86034966|  0.8603496361326065| 0.06480785940636602|
|2016-08-05 00:00:00|2182.870117| -0.0907165288753599| -0.090716526| -0.0907165288753599|0.005081318066857778|
|2016-08-08 00:00:00|2180.889893|0.038979363549189745|  0.038979363|0.038979363549189745|0.017506053723946088|
|2016-08-09 00:00:00| 2181.73999|-0.28646859977113953|   -0.2864686|-0.28646859977113953|0.003943602897653...|
|2016-08-10 00:00:00| 2175.48999|  0.4734588091577449|    0.4734588|  0.4734588091577449|  0.0506663638782674|
|2016-08-11 00:00:00|2185.790039|-0.07960462665462967| -0.079604626|-0.07960462665462967|0.007254849084550381|
|2016-08-12 00:00:00|2184.050049| 0.27929089824626846|   0.27929088| 0.27929089824626846| 0.04162200191308573|
|2016-08-15 00:00:00|2190.149902| -0.5479077020728967|    0.5479077| -0.5479077020728967|-0.01186933063353366|
|2016-08-16 00:00:00|2178.149902| 0.18685899424381655|     0.186859| 0.18685899424381655|0.059713797958216595|
|2016-08-17 00:00:00|2182.219971|  0.2199617391366979|   0.21996173|  0.2199617391366979| 0.02751569028115043|
|2016-08-18 00:00:00| 2187.02002|-0.14402716807320287|  -0.14402717|-0.14402716807320287| 0.06191917075263412|
|2016-08-19 00:00:00|2183.870117| -0.0563322878235028| -0.056332286| -0.0563322878235028|0.023723971774663854|
|2016-08-22 00:00:00|2182.639893| 0.19517690543742305|    0.1951769| 0.19517690543742305|0.056532929973085906|
|2016-08-23 00:00:00|2186.899902|   -0.52402768821378|   -0.5240277|   -0.52402768821378|0.017734810601854903|
|2016-08-24 00:00:00|2175.439941|-0.13652273013958158|  -0.13652273|-0.13652273013958158| 0.07719527924246394|
|2016-08-25 00:00:00|2172.469971|-0.15788167596264593|  -0.15788168|-0.15788167596264593| 0.05572356653909286|
|2016-08-26 00:00:00|2169.040039|  0.5228047337119758|   0.52280474|  0.5228047337119758| 0.07452695447099067|
|2016-08-29 00:00:00|2180.379883| -0.1953680655931885|  -0.19536807| -0.1953680655931885| 0.03669524115740061|
|2016-08-30 00:00:00|2176.120117|-0.23758642547395195|  -0.23758642|-0.23758642547395195|0.053912928496864576|
|2016-08-31 00:00:00|2170.949951|-0.00413846482083...| -0.004138465|-0.00413846482083...| 0.04602896318380502|
|2016-09-01 00:00:00|2170.860107|  0.4201041315648488|   0.42010415|  0.4201041315648488| 0.06396138233525586|
|2016-09-02 00:00:00| 2179.97998|  0.2981678758352634|   0.29816788|  0.2981678758352634|0.027061272029217553|
|2016-09-06 00:00:00| 2186.47998|-0.01463850585999...| -0.014638506|-0.01463850585999...| 0.03507840827130089|
|2016-09-07 00:00:00|2186.159912| -0.2223013501127694|  -0.22230135| -0.2223013501127694|0.001104900755447...|
|2016-09-08 00:00:00|2181.300049| -2.4522068857295496|   -2.4522069| -2.4522068857295496| 0.03586942869105701|
|2016-09-09 00:00:00|2127.810059|   1.467705252539182|    1.4677052|   1.467705252539182| 0.13003750766460198|
|2016-09-12 00:00:00|2159.040039| -1.4830674013266898|   -1.4830674| -1.4830674013266898| 0.09179040546568898|
|2016-09-13 00:00:00| 2127.02002|-0.05876766500768526| -0.058767665|-0.05876766500768526| 0.13538445687768375|
|2016-09-14 00:00:00| 2125.77002|  1.0109273250546658|    1.0109273|  1.0109273250546658| 0.10299656562896597|
|2016-09-15 00:00:00| 2147.26001| -0.3772294907126729|  -0.37722948| -0.3772294907126729| 0.08981405890727896|
|2016-09-16 00:00:00|2139.159912|-0.00186030973079...|-0.0018603097|-0.00186030973079...| 0.03297684927572516|
|2016-09-19 00:00:00|2139.120117|0.029913841439509457|   0.02991384|0.029913841439509457| 0.12522061779256152|
|2016-09-20 00:00:00| 2139.76001|  1.0917162154086604|   -1.0917162|  1.0917162154086604|-0.00657471545100...|
|2016-09-21 00:00:00|2163.120117|  0.6499784681166678|   0.64997846|  0.6499784681166678| 0.04344795902070058|
|2016-09-22 00:00:00|2177.179932| -0.5736774814255415|    0.5736775| -0.5736774814255415| -0.0670728283114628|
|2016-09-23 00:00:00|2164.689941| -0.8587762454059558|   -0.8587763| -0.8587762454059558|0.002278092915136469|
|2016-09-26 00:00:00|2146.100098|  0.6444170061260637|     0.644417|  0.6444170061260637| 0.03851714501551536|
|2016-09-27 00:00:00|2159.929932|  0.5296553758763286|    0.5296554|  0.5296553758763286| 0.01956469868906461|
|2016-09-28 00:00:00|2171.370117| -0.9321411325289909|   -0.9321411| -0.9321411325289909|  0.0411228838312013|
|2016-09-29 00:00:00|2151.129883|  0.7967969361336731|    0.7967969|  0.7967969361336731| 0.03418411730268428|
|2016-09-30 00:00:00| 2168.27002| -0.3260695824222057|   -0.3260696| -0.3260695824222057| 0.01894587817268246|
|2016-10-03 00:00:00|2161.199951|-0.49555623000289534|  -0.49555624|-0.49555623000289534| 0.04060719379041892|
|2016-10-04 00:00:00| 2150.48999|  0.4296690541675125|   0.42966905|  0.4296690541675125| 0.08214547459166668|
|2016-10-05 00:00:00| 2159.72998| 0.04815601994837626|   0.04815602| 0.04815601994837626| 0.05618972707504136|
|2016-10-06 00:00:00| 2160.77002|  -0.325348368171079|  -0.32534838|  -0.325348368171079|0.019473338329135687|
|2016-10-07 00:00:00| 2153.73999|  0.4605905098135844|    0.4605905|  0.4605905098135844| 0.06448828115009853|
|2016-10-10 00:00:00|2163.659912| -1.2446471763257407|   -1.2446471| -1.2446471763257407| 0.01865659611688069|
|2016-10-11 00:00:00| 2136.72998| 0.11465894253984957|  0.114658944| 0.11465894253984957| 0.10060441412799451|
|2016-10-12 00:00:00|2139.179932|-0.30992638350910184|   -0.3099264|-0.30992638350910184| 0.06813560495887139|
|2016-10-13 00:00:00|2132.550049|0.020160417815362805|  0.020160418|0.020160417815362805| 0.09740153124660542|
|2016-10-14 00:00:00| 2132.97998|-0.30379938212078617|   -0.3037994|-0.30379938212078617| 0.06847460331341597|
|2016-10-17 00:00:00|     2126.5|  0.6160403479896491|   0.61604035|  0.6160403479896491| 0.09284634450633844|
|2016-10-18 00:00:00|2139.600098|  0.2191970828746938|   0.21919708|  0.2191970828746938|0.028563271274506367|
|2016-10-19 00:00:00|2144.290039| -0.1375723874264593|   -0.1375724| -0.1375723874264593| 0.06667846133285449|
|2016-10-20 00:00:00|2141.340088|-0.00841417022030...|  -0.00841417|-0.00841417022030...|0.008403406831473961|
|2016-10-21 00:00:00|2141.159912|  0.4749839534638121|   0.47498396|  0.4749839534638121|0.057633465109647175|
|2016-10-24 00:00:00|2151.330078|-0.37977277794560177|   0.37977278|-0.37977277794560177|-0.01019980342886...|
|2016-10-25 00:00:00|2143.159912|-0.17404114266579604|  -0.17404114|-0.17404114266579604| 0.04775981301407148|
|2016-10-26 00:00:00|2139.429932| -0.2986726933387613|   -0.2986727| -0.2986726933387613|0.027506605662294098|
|2016-10-27 00:00:00|2133.040039| -0.3108299365589125|  -0.31082994| -0.3108299365589125|  0.0629669853941767|
|2016-10-28 00:00:00|2126.409912|-0.01222765180563...| -0.012227652|-0.01222765180563...| 0.06511662557856555|
|2016-10-31 00:00:00|2126.149902| -0.6786883176217423|  -0.67868835| -0.6786883176217423| 0.08620427809391291|
|2016-11-01 00:00:00|2111.719971| -0.6525500629458174|  -0.65255004| -0.6525500629458174| 0.08581020570495212|
|2016-11-02 00:00:00|2097.939941|-0.44234006982948293|  -0.44234008|-0.44234006982948293| 0.12052389382225526|
|2016-11-03 00:00:00|2088.659912|-0.16661305078948002|  -0.16661306|-0.16661305078948002| 0.12095496150285939|
|2016-11-04 00:00:00|2085.179932|   2.222354401596069|    2.2223544|   2.222354401596069|  0.1274469686705091|
|2016-11-07 00:00:00| 2131.52002|  0.3771974424148256|   0.37719744|  0.3771974424148256| 0.01727894118614634|
|2016-11-08 00:00:00|2139.560059|  1.1077020670818223|    -1.107702|  1.1077020670818223|-0.02383686828285326|
|2016-11-09 00:00:00| 2163.26001|  0.1950745624886813|  -0.19507456|  0.1950745624886813|-0.06903791587186517|
|2016-11-10 00:00:00| 2167.47998|  -0.139795016699532|   0.13979502|  -0.139795016699532|-0.06087901134116...|
|2016-11-11 00:00:00|2164.449951|-0.01155027862319...|  0.011550278|-0.01155027862319...|-0.05195444638695...|
|2016-11-14 00:00:00|2164.199951|  0.7480797692708188|   -0.7480798|  0.7480797692708188|-7.79633808628769...|
|2016-11-15 00:00:00|2180.389893|-0.15822638011099693|   0.15822639|-0.15822638011099693|-0.07746632876706516|
|2016-11-16 00:00:00|2176.939941|  0.4676369709732771|   0.46763697|  0.4676369709732771|0.005814145745197...|
|2016-11-17 00:00:00|2187.120117|-0.23867984933357117|   0.23867986|-0.23867984933357117|-0.04491788988134...|
|2016-11-18 00:00:00|2181.899902|  0.7461400949272281|    0.7461401|  0.7461400949272281| 0.04385237610014097|
|2016-11-21 00:00:00|2198.179932| 0.21654319242507222|   -0.2165432| 0.21654319242507222|-0.01178749539393...|
|2016-11-22 00:00:00|2202.939941|  0.0808024752228094|   0.08080248|  0.0808024752228094|0.017541231477313436|
|2016-11-23 00:00:00|2204.719971| 0.39143869124048053|   -0.3914387| 0.39143869124048053|-0.02052102829776...|
|2016-11-25 00:00:00|2213.350098| -0.5254535651865005|  -0.52545357| -0.5254535651865005|0.005803896627152853|
|2016-11-28 00:00:00|2201.719971| 0.13352928795321764|   0.13352929| 0.13352928795321764|0.001236510710497...|
|2016-11-29 00:00:00|2204.659912| -0.2653403805348535|   -0.2653404| -0.2653403805348535|0.051704179975071575|
|2016-11-30 00:00:00|2198.810059| -0.3515529214704194|  -0.35155293| -0.3515529214704194|0.029759298143040534|
|2016-12-01 00:00:00|2191.080078|0.039700648494513754|   0.03970065|0.039700648494513754| 0.06804129066701364|
|2016-12-02 00:00:00|2191.949951|  0.5821305360635018|   0.58213055|  0.5821305360635018|  0.0632553785160296|
|2016-12-05 00:00:00|2204.709961| 0.34108881136406527|    0.3410888| 0.34108881136406527| 0.03340206225641486|
|2016-12-06 00:00:00| 2212.22998|  1.3163241734930209|    1.3163241|  1.3163241734930209| 0.03660868441958645|
|2016-12-07 00:00:00|2241.350098|  0.2159342712376298|  -0.21593428|  0.2159342712376298|-0.04433504115778635|
|2016-12-08 00:00:00|2246.189941|  0.5938984836723531|   -0.5938985|  0.5938984836723531|-0.02892054885877942|
|2016-12-09 00:00:00|2259.530029|-0.11374347616603382|   0.11374348|-0.11374347616603382|-0.06386592131010393|
|2016-12-12 00:00:00|2256.959961|  0.6539774854251376|   -0.6539775|  0.6539774854251376|-0.02497108743452...|
|2016-12-13 00:00:00|2271.719971| -0.8117172114256143|    0.8117172| -0.8117172114256143|-0.05741948077553...|
|2016-12-14 00:00:00|2253.280029| 0.38832279554189403|    0.3883228| 0.38832279554189403|0.017646025432132013|
|2016-12-15 00:00:00|2262.030029|-0.17506226483432866|   0.17506227|-0.17506226483432866|-0.01763929406028495|
|2016-12-16 00:00:00|2258.070068| 0.19751207295131734|   0.19751208| 0.19751207295131734|0.053183155041379666|
|2016-12-19 00:00:00|2262.530029| 0.36375123841505275|   0.36375123| 0.36375123841505275|0.007538061754018...|
|2016-12-20 00:00:00| 2270.76001|-0.24573614012164843|  -0.24573614|-0.24573614012164843| 0.05659128849926827|
|2016-12-21 00:00:00|2265.179932|-0.18629738593322426|  -0.18629739|-0.18629738593322426|0.005844312293819924|
|2016-12-22 00:00:00|2260.959961| 0.12517152222139497|   0.12517153| 0.12517152222139497| 0.07570360878826678|
|2016-12-23 00:00:00|2263.790039| 0.22483728227059685|   0.22483729| 0.22483728227059685|0.021483502619055662|
|2016-12-27 00:00:00|2268.879883| -0.8356529202828681|  -0.83565295| -0.8356529202828681| 0.05333195679616798|
|2016-12-28 00:00:00|2249.919922|-0.02933046610003201| -0.029330466|-0.02933046610003201| 0.05191400429652269|
|2016-12-29 00:00:00| 2249.26001|-0.46370503870737506|  -0.46370503|-0.46370503870737506| 0.06713886320955127|
|2016-12-30 00:00:00|2238.830078|  0.8486575281753026|   0.84865755|  0.8486575281753026| 0.08970054534422042|
+-------------------+-----------+--------------------+-------------+--------------------+--------------------+

Linear Regression Accuracy:
0.5119048
Long Only Accuracy:
0.52380955
True Positive Count:
+---+
|tpc|
+---+
| 99|
+---+

False Positive Count:
+---+
|fpc|
+---+
| 90|
+---+

True Negative Count:
+---+
|tnc|
+---+
| 30|
+---+

False Negative Count:
+---+
|fnc|
+---+
| 33|
+---+

Effectiveness of negative predictions:
+------------------+
|            eff_np|
+------------------+
|-2.467898511452154|
+------------------+

Effectiveness of positive predictions:
+-----------------+
|           eff_pp|
+-----------------+
|9.875746989931645|
+-----------------+

Effectiveness of Long-Only-Model:
+------------------+
|            eff_lo|
+------------------+
|12.343645501383794|
+------------------+


Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.2.1
      /_/
         
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_152)
Type in expressions to have them evaluated.
Type :help for more information.

scala>
scala>

For 2016 test data, I translate the above report into English:

Class04 Lab


learn4.us About Blog Contact Class01 Class02 Class03 Class04 Class05 Class06 Class07 Class08 Class09 Class10 dan101 Forum Google Hangout Vboxen