Class04 Answer:

Calculate predictions of test data and report effectiveness.

This lab required some knowledge.


{
/* ~/sparkapps/logr10/logr12o.scala
This script should download prices and predict daily direction of GSPC.
It should generate a label which I assume to be dependent on price calculations.
A label should classify an observation as down or up. Down is 0.0, up is 1.0.
It should generate independent features from slopes of moving averages of prices.
It should create a Logistic Regression model from many years of features.
Demo:
spark-shell -i logr12o.scala
*/

import org.apache.spark.sql.SQLContext
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.sql.Row
import sys.process._

// I should get prices:
"/usr/bin/curl -L ml4.herokuapp.com/csv/GSPC.csv -o /tmp/gspc.csv"!

val sqlContext = new SQLContext(sc)
  
val dp10df = sqlContext
  .read
  .format("com.databricks.spark.csv")
  .option("header","true")
  .option("inferSchema","true")
  .load("/tmp/gspc.csv")

dp10df.createOrReplaceTempView("tab")

spark.sql("SELECT COUNT(Date),MIN(Date),MAX(Date),MIN(Close),MAX(Close)FROM tab").show

// I should compute a label I can use to classify observations.

var sqls="SELECT Date,Close,LEAD(Close,1)OVER(ORDER BY Date) leadp FROM tab ORDER BY Date"

val dp11df=spark.sql(sqls);dp11df.createOrReplaceTempView("tab")

sqls="SELECT Date,Close,100*(leadp-Close)/Close pctlead FROM tab ORDER BY Date"

val dp12df=spark.sql(sqls);dp12df.createOrReplaceTempView("tab")

sqls = "SELECT Date, Close, pctlead"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS mavg2"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 3 PRECEDING AND CURRENT ROW) AS mavg3"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS mavg4"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS mavg5"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS mavg6"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 7 PRECEDING AND CURRENT ROW) AS mavg7"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 8 PRECEDING AND CURRENT ROW) AS mavg8"
sqls=sqls++",AVG(Close)OVER(ORDER BY Date ROWS BETWEEN 9 PRECEDING AND CURRENT ROW) AS mavg9"
sqls=sqls++" FROM tab ORDER BY Date"

val dp13df=spark.sql(sqls);dp13df.createOrReplaceTempView("tab")

sqls = "SELECT Date, Close, pctlead"
sqls=sqls++",(mavg2-LAG(mavg2,1)OVER(ORDER BY Date))/mavg2 AS slp2 "
sqls=sqls++",(mavg3-LAG(mavg3,1)OVER(ORDER BY Date))/mavg3 AS slp3 "
sqls=sqls++",(mavg4-LAG(mavg4,1)OVER(ORDER BY Date))/mavg4 AS slp4 "
sqls=sqls++",(mavg5-LAG(mavg5,1)OVER(ORDER BY Date))/mavg5 AS slp5 "
sqls=sqls++",(mavg6-LAG(mavg6,1)OVER(ORDER BY Date))/mavg6 AS slp6 "
sqls=sqls++",(mavg7-LAG(mavg7,1)OVER(ORDER BY Date))/mavg7 AS slp7 "
sqls=sqls++",(mavg8-LAG(mavg8,1)OVER(ORDER BY Date))/mavg8 AS slp8 "
sqls=sqls++",(mavg9-LAG(mavg9,1)OVER(ORDER BY Date))/mavg9 AS slp9 "
sqls=sqls++" FROM tab ORDER BY Date"

val dp14df=spark.sql(sqls);dp14df.createOrReplaceTempView("tab")

// For Class Boundry, I should get avg of pctlead over training period.

val training_period = " WHERE Date BETWEEN'1986-01-01'AND'2015-12-31' "

sqls = "SELECT AVG(pctlead) FROM tab"++training_period

val class_df = spark.sql(sqls)

val class_boundry = class_df.first()(0).asInstanceOf[Double]

// I should compute label from pctlead:

val pctlead2label = udf((pctlead:Float)=> {if (pctlead> class_boundry) 1.0 else 0.0}) 

// I should add the label to my DF of observations:

val dp15df = dp14df.withColumn("label",pctlead2label(col("pctlead")))

// I should copy slp-values into Vectors.dense():

val fill_vec = udf((
  slp2:Float
  ,slp3:Float
  ,slp4:Float
  ,slp5:Float
  ,slp6:Float
  ,slp7:Float
  ,slp8:Float
  ,slp9:Float
  )=> {Vectors.dense(
  slp2
  ,slp3
  ,slp4
  ,slp5
  ,slp6
  ,slp7
  ,slp8
  ,slp9
  )
  }
)

val dp16df = dp15df.withColumn("features"
,fill_vec(
  col("slp2")
  ,col("slp3")
  ,col("slp4")
  ,col("slp5")
  ,col("slp6")
  ,col("slp7")
  ,col("slp8")
  ,col("slp9")
  )
)

// I should create a LogisticRegression instance. This instance is an 'Estimator'.

val lr = new LogisticRegression()

lr.setMaxIter(1234).setRegParam(0.01)

// I should gather observations to learn from:
dp16df.createOrReplaceTempView("tab")

val train_df = spark.sql("SELECT * FROM tab"++training_period)

/*I should fit a LogisticRegression model to observations.
This uses the parameters stored in lr.*/
val model1 = lr.fit(train_df)
// Above line will fail with ugly error if train_df has any nulls.

val test_period = " WHERE Date BETWEEN'2016-01-01'AND'2017-01-01' "
val test_df = spark.sql("SELECT * FROM tab"++test_period)

/* I should predict. It is convenient that predictions_df will contain a copy of test_df.*/

val predictions_df = model1.transform(test_df)
predictions_df.createOrReplaceTempView("tab")

// eff logic
val eff = udf((pctlead:Float,prediction:Double)=> {if (prediction==1.0) pctlead else -pctlead})

// prediction report:
println("Logistic Regression Predictions:")
val p2df = predictions_df.withColumn("effectiveness",eff(col("pctlead"),col("prediction")))
p2df.select("Date","Close","pctlead","effectiveness","label","prediction").show(255)

println("Logistic Regression Accuracy:")
sqls = """
  SELECT
  tpc,tnc,allc
  ,100*(tpc+tnc)/allc accuracy
  FROM (SELECT 1 one1,COUNT(Date) tpc FROM tab WHERE prediction > 0 AND pctlead > 0)
      ,(SELECT 1 one2,COUNT(Date) tnc FROM tab WHERE prediction < 1 AND pctlead < 0)
      ,(SELECT 1 one3,COUNT(Date) allc  FROM tab)
  -- I should prevent Spark cartesian-product complaint:
  WHERE one1 = one2 AND one1 = one3
  """
spark.sql(sqls).show


println("Long Only Accuracy:")
sqls = """
  SELECT
  truec
  ,allc
  ,100*truec/allc accuracy
  FROM (SELECT 1 one1,COUNT(Date) truec FROM tab WHERE pctlead > 0)
      ,(SELECT 1 one2,COUNT(Date) allc  FROM tab)
  -- I should prevent Spark cartesian-product complaint:
  WHERE one1 = one2
  """
spark.sql(sqls).show

println("True Positive Count:")
spark.sql("SELECT COUNT(Date) tpc FROM tab WHERE prediction=1.0 AND pctlead > 0").show

println("False Positive Count:")
spark.sql("SELECT COUNT(Date) fpc FROM tab WHERE prediction=1.0 AND pctlead < 0").show

println("True Negative Count:")
spark.sql("SELECT COUNT(Date) tnc FROM tab WHERE prediction=0.0 AND pctlead < 0").show

println("False Negative Count:")
spark.sql("SELECT COUNT(Date) fnc FROM tab WHERE prediction=0.0 AND pctlead > 0").show

println("Effectiveness of Negative Predictions:")
spark.sql("SELECT -SUM(pctlead) eff_np FROM tab WHERE prediction = 0.0").show

println("Effectiveness of Positive Predictions:")
spark.sql("SELECT SUM(pctlead) eff_pp FROM tab WHERE prediction = 1.0").show

println("Effectiveness of Long-Only-Model:")
spark.sql("SELECT SUM(pctlead) eff_lo FROM tab").show

}

I saw something like this:


scala> :load logr12o.scala
Loading logr12o.scala...
warning: there was one deprecation warning; re-run with -deprecation for details
warning: there was one feature warning; re-run with -feature for details
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 1252k  100 1252k    0     0  1302k      0 --:--:-- --:--:-- --:--:-- 1782k
+-----------+-------------------+-------------------+----------+----------+
|count(Date)|          min(Date)|          max(Date)|min(Close)|max(Close)|
+-----------+-------------------+-------------------+----------+----------+
|      17116|1950-01-03 00:00:00|2018-01-09 00:00:00|     16.66|2753.52002|
+-----------+-------------------+-------------------+----------+----------+

Logistic Regression Predictions:
+-------------------+-----------+--------------------+-------------+-----+----------+
|               Date|      Close|             pctlead|effectiveness|label|prediction|
+-------------------+-----------+--------------------+-------------+-----+----------+
|2016-01-04 00:00:00|2012.660034|  0.2012226074739071|   0.20122261|  1.0|       1.0|
|2016-01-05 00:00:00|2016.709961|   -1.31153966170151|   -1.3115396|  0.0|       1.0|
|2016-01-06 00:00:00| 1990.26001| -2.3700443039098174|   -2.3700442|  0.0|       1.0|
|2016-01-07 00:00:00|1943.089966|  -1.083837463447639|   -1.0838375|  0.0|       1.0|
|2016-01-08 00:00:00|1922.030029| 0.08532723085774163|   0.08532723|  1.0|       1.0|
|2016-01-11 00:00:00|1923.670044|  0.7802798638371875|    0.7802799|  1.0|       1.0|
|2016-01-12 00:00:00|1938.680054| -2.4965452602732516|   -2.4965453|  0.0|       1.0|
|2016-01-13 00:00:00|1890.280029|   1.669590564139637|    1.6695906|  1.0|       1.0|
|2016-01-14 00:00:00|1921.839966|  -2.159909812178397|   -2.1599097|  0.0|       1.0|
|2016-01-15 00:00:00|1880.329956|0.053182155440808176|  0.053182155|  1.0|       1.0|
|2016-01-19 00:00:00|1881.329956| -1.1693855152753438|   -1.1693856|  0.0|       1.0|
|2016-01-20 00:00:00|1859.329956|  0.5195438264643328|    0.5195438|  1.0|       1.0|
|2016-01-21 00:00:00| 1868.98999|  2.0283700930896904|    2.0283701|  1.0|       1.0|
|2016-01-22 00:00:00|1906.900024| -1.5637981868314241|   -1.5637982|  0.0|       1.0|
|2016-01-25 00:00:00|1877.079956|   1.414433568220359|    1.4144336|  1.0|       1.0|
|2016-01-26 00:00:00|1903.630005| -1.0863483946818713|   -1.0863484|  0.0|       1.0|
|2016-01-27 00:00:00|1882.949951|  0.5528577110863419|    0.5528577|  1.0|       1.0|
|2016-01-28 00:00:00|1893.359985|  2.4760217481832956|    2.4760218|  1.0|       1.0|
|2016-01-29 00:00:00| 1940.23999|-0.04432364060283344|   0.04432364|  0.0|       0.0|
|2016-02-01 00:00:00|1939.380005| -1.8743091042644822|    1.8743091|  0.0|       0.0|
|2016-02-02 00:00:00|1903.030029|  0.4992038935398218|    0.4992039|  1.0|       1.0|
|2016-02-03 00:00:00|1912.530029|  0.1526732629409628|   0.15267326|  1.0|       1.0|
|2016-02-04 00:00:00|1915.449951| -1.8481246133065947|   -1.8481246|  0.0|       1.0|
|2016-02-05 00:00:00|1880.050049| -1.4153935962584503|   -1.4153936|  0.0|       1.0|
|2016-02-08 00:00:00|1853.439941|-0.06636201005447463|  -0.06636201|  0.0|       1.0|
|2016-02-09 00:00:00|1852.209961|-0.01889505009523...|  -0.01889505|  0.0|       1.0|
|2016-02-10 00:00:00|1851.859985| -1.2301161634528224|   -1.2301161|  0.0|       1.0|
|2016-02-11 00:00:00|1829.079956|   1.951804943402921|     1.951805|  1.0|       1.0|
|2016-02-12 00:00:00|1864.780029|   1.651665425466653|    1.6516654|  1.0|       1.0|
|2016-02-16 00:00:00|1895.579956|   1.648043908731858|    1.6480439|  1.0|       1.0|
|2016-02-17 00:00:00|1926.819946| -0.4665713586089291|   0.46657136|  0.0|       0.0|
|2016-02-18 00:00:00|1917.829956|-0.00260330692217...| -0.002603307|  0.0|       1.0|
|2016-02-19 00:00:00|1917.780029|  1.4454197343192787|    1.4454198|  1.0|       1.0|
|2016-02-22 00:00:00|     1945.5| -1.2454371626831184|    1.2454371|  0.0|       0.0|
|2016-02-23 00:00:00| 1921.27002|   0.443978665736949|  -0.44397867|  1.0|       0.0|
|2016-02-24 00:00:00|1929.800049|   1.134827518081388|   -1.1348275|  1.0|       0.0|
|2016-02-25 00:00:00|1951.699951|-0.18701143063153722|   0.18701144|  0.0|       0.0|
|2016-02-26 00:00:00|1948.050049| -0.8120976670040306|   0.81209767|  0.0|       0.0|
|2016-02-29 00:00:00| 1932.22998|  2.3868792264572924|    2.3868792|  1.0|       1.0|
|2016-03-01 00:00:00|1978.349976| 0.40943084379727995|  -0.40943083|  1.0|       0.0|
|2016-03-02 00:00:00|1986.449951|  0.3498740552965472|  -0.34987405|  1.0|       0.0|
|2016-03-03 00:00:00|1993.400024|  0.3305892405266673|  -0.33058923|  1.0|       0.0|
|2016-03-04 00:00:00| 1999.98999| 0.08850144294971853| -0.088501446|  1.0|       0.0|
|2016-03-07 00:00:00| 2001.76001|  -1.124010864818905|    1.1240109|  0.0|       0.0|
|2016-03-08 00:00:00| 1979.26001|  0.5052393293188397|    0.5052393|  1.0|       1.0|
|2016-03-09 00:00:00| 1989.26001|0.015580467030054408| -0.015580467|  0.0|       0.0|
|2016-03-10 00:00:00|1989.569946|  1.6395500477669567|    1.6395501|  1.0|       1.0|
|2016-03-11 00:00:00|2022.189941|-0.12609725467921024|   0.12609726|  0.0|       0.0|
|2016-03-14 00:00:00|2019.640015| -0.1836941718546818|   0.18369417|  0.0|       0.0|
|2016-03-15 00:00:00|2015.930054|  0.5600351548705103|   0.56003517|  1.0|       1.0|
|2016-03-16 00:00:00|2027.219971|  0.6595236427847927|   0.65952367|  1.0|       1.0|
|2016-03-17 00:00:00|2040.589966| 0.44056435392665033|  -0.44056436|  1.0|       0.0|
|2016-03-18 00:00:00|2049.580078| 0.09855774954502321|  -0.09855775|  1.0|       0.0|
|2016-03-21 00:00:00|2051.600098|-0.08773878504659462|   0.08773878|  0.0|       0.0|
|2016-03-22 00:00:00|2049.800049| -0.6386031655324603|   0.63860315|  0.0|       0.0|
|2016-03-23 00:00:00|2036.709961|-0.03780705229240...|  -0.03780705|  0.0|       1.0|
|2016-03-24 00:00:00|2035.939941| 0.05452557698998718|  0.054525577|  1.0|       1.0|
|2016-03-28 00:00:00|2037.050049|  0.8816651809226127|   0.88166517|  1.0|       1.0|
|2016-03-29 00:00:00| 2055.01001| 0.43503150624556275|   -0.4350315|  1.0|       0.0|
|2016-03-30 00:00:00|2063.949951|-0.20397592480187138|  -0.20397593|  0.0|       1.0|
|2016-03-31 00:00:00| 2059.73999|  0.6330915097686665|    0.6330915|  1.0|       1.0|
|2016-04-01 00:00:00|2072.780029| -0.3208322111829817|  -0.32083222|  0.0|       1.0|
|2016-04-04 00:00:00|2066.129883|  -1.014449245057461|   -1.0144492|  0.0|       1.0|
|2016-04-05 00:00:00|2045.170044|  1.0507619189438957|    1.0507619|  1.0|       1.0|
|2016-04-06 00:00:00|2066.659912| -1.1975786560861146|   -1.1975787|  0.0|       1.0|
|2016-04-07 00:00:00|2041.910034|   0.278657820631483|   0.27865782|  1.0|       1.0|
|2016-04-08 00:00:00|2047.599976| -0.2739786123146514|  -0.27397862|  0.0|       1.0|
|2016-04-11 00:00:00| 2041.98999|  0.9662134044055697|    0.9662134|  1.0|       1.0|
|2016-04-12 00:00:00|2061.719971|  1.0040137017230288|    1.0040137|  1.0|       1.0|
|2016-04-13 00:00:00|2082.419922|0.017292717774910475|  0.017292717|  0.0|       1.0|
|2016-04-14 00:00:00|2082.780029|-0.09842849323767663|  0.098428495|  0.0|       0.0|
|2016-04-15 00:00:00| 2080.72998|  0.6541025568343978|   0.65410256|  1.0|       1.0|
|2016-04-18 00:00:00|2094.340088|   0.308448519751584|   0.30844852|  1.0|       1.0|
|2016-04-19 00:00:00|2100.800049| 0.07615446318947447|  -0.07615446|  1.0|       0.0|
|2016-04-20 00:00:00|2102.399902| -0.5194027068595269|    0.5194027|  0.0|       0.0|
|2016-04-21 00:00:00| 2091.47998|0.004785988914887...| -0.004785989|  0.0|       0.0|
|2016-04-22 00:00:00|2091.580078| -0.1812045849864888|  -0.18120459|  0.0|       1.0|
|2016-04-25 00:00:00|2087.790039|  0.1872751534858758|   0.18727516|  1.0|       1.0|
|2016-04-26 00:00:00|2091.699951| 0.16493527182762052|   0.16493528|  1.0|       1.0|
|2016-04-27 00:00:00|2095.149902| -0.9230768157227635|   -0.9230768|  0.0|       1.0|
|2016-04-28 00:00:00|2075.810059| -0.5063088481738572|  -0.50630885|  0.0|       1.0|
|2016-04-29 00:00:00|2065.300049|   0.780994655367873|   0.78099465|  1.0|       1.0|
|2016-05-02 00:00:00|2081.429932| -0.8676638460102664|  -0.86766386|  0.0|       1.0|
|2016-05-03 00:00:00|2063.370117| -0.5936889314753995|   -0.5936889|  0.0|       1.0|
|2016-05-04 00:00:00|2051.120117|-0.02390079429950192| -0.023900794|  0.0|       1.0|
|2016-05-05 00:00:00|2050.629883| 0.31746391945074204|    0.3174639|  1.0|       1.0|
|2016-05-06 00:00:00|2057.139893| 0.07534966412709886|  0.075349666|  1.0|       1.0|
|2016-05-09 00:00:00|2058.689941|  1.2483643839788858|    1.2483643|  1.0|       1.0|
|2016-05-10 00:00:00|2084.389893| -0.9561518249023676|  -0.95615184|  0.0|       1.0|
|2016-05-11 00:00:00|2064.459961|-0.01694651417848691| -0.016946515|  0.0|       1.0|
|2016-05-12 00:00:00|2064.110107| -0.8478288992748931|   -0.8478289|  0.0|       1.0|
|2016-05-13 00:00:00|2046.609985|  0.9796652584981903|    0.9796653|  1.0|       1.0|
|2016-05-16 00:00:00|2066.659912| -0.9411297372666149|  -0.94112974|  0.0|       1.0|
|2016-05-17 00:00:00|2047.209961|0.020517875938566795|  0.020517876|  0.0|       1.0|
|2016-05-18 00:00:00|2047.630005|-0.37067077457677733|  -0.37067077|  0.0|       1.0|
|2016-05-19 00:00:00|2040.040039|   0.601950391425627|    0.6019504|  1.0|       1.0|
|2016-05-20 00:00:00|2052.320068|-0.20854588262009888|  -0.20854588|  0.0|       1.0|
|2016-05-23 00:00:00|2048.040039|  1.3681382915580749|    1.3681383|  1.0|       1.0|
|2016-05-24 00:00:00|2076.060059|  0.6974740416216479|  -0.69747406|  1.0|       0.0|
|2016-05-25 00:00:00|2090.540039|-0.02104437091817...| -0.021044372|  0.0|       1.0|
|2016-05-26 00:00:00|2090.100098|  0.4286857365622697|  -0.42868572|  1.0|       0.0|
|2016-05-27 00:00:00|2099.060059|-0.10052632800821898| -0.100526325|  0.0|       1.0|
|2016-05-31 00:00:00|2096.949951| 0.11350423498972209|  -0.11350424|  1.0|       0.0|
|2016-06-01 00:00:00|2099.330078|  0.2824678244809108|   -0.2824678|  1.0|       0.0|
|2016-06-02 00:00:00| 2105.26001| -0.2911814678890852|   0.29118147|  0.0|       0.0|
|2016-06-03 00:00:00|2099.129883|  0.4897281051188776|   -0.4897281|  1.0|       0.0|
|2016-06-06 00:00:00|2109.409912| 0.12894463918684698|  -0.12894464|  1.0|       0.0|
|2016-06-07 00:00:00|2112.129883|  0.3309566355867838|   0.33095664|  1.0|       1.0|
|2016-06-08 00:00:00|2119.120117|   -0.17177586918259|  -0.17177586|  0.0|       1.0|
|2016-06-09 00:00:00| 2115.47998|   -0.91751811331252|  -0.91751814|  0.0|       1.0|
|2016-06-10 00:00:00|2096.070068| -0.8115191023280278|   -0.8115191|  0.0|       1.0|
|2016-06-13 00:00:00|2079.060059|-0.17988855029992754|  -0.17988855|  0.0|       1.0|
|2016-06-14 00:00:00|2075.320068| -0.1840712697237789|  -0.18407127|  0.0|       1.0|
|2016-06-15 00:00:00|     2071.5| 0.31329905865315155|   0.31329906|  1.0|       1.0|
|2016-06-16 00:00:00| 2077.98999|-0.32579651646926594|   -0.3257965|  0.0|       1.0|
|2016-06-17 00:00:00|2071.219971|  0.5808185112367291|   0.58081853|  1.0|       1.0|
|2016-06-20 00:00:00|    2083.25| 0.27120614424577516|   0.27120614|  1.0|       1.0|
|2016-06-21 00:00:00|2088.899902|-0.16515635798043402|  -0.16515636|  0.0|       1.0|
|2016-06-22 00:00:00|2085.449951|  1.3364078570495475|    1.3364079|  1.0|       1.0|
|2016-06-23 00:00:00|2113.320068|  -3.591979991551379|     -3.59198|  0.0|       1.0|
|2016-06-24 00:00:00|2037.410034|  -1.809650212020111|   -1.8096502|  0.0|       1.0|
|2016-06-27 00:00:00|2000.540039|  1.7770165208875395|    1.7770165|  1.0|       1.0|
|2016-06-28 00:00:00|2036.089966|  1.7032672710494525|    1.7032672|  1.0|       1.0|
|2016-06-29 00:00:00| 2070.77002|  1.3565044272758033|   -1.3565044|  1.0|       0.0|
|2016-06-30 00:00:00|2098.860107| 0.19486024753912218|  -0.19486025|  1.0|       0.0|
|2016-07-01 00:00:00|2102.949951| -0.6847477275031026|   0.68474776|  0.0|       0.0|
|2016-07-05 00:00:00|2088.550049|  0.5352962934909359|    0.5352963|  1.0|       1.0|
|2016-07-06 00:00:00| 2099.72998|-0.08715777825870533|  -0.08715778|  0.0|       1.0|
|2016-07-07 00:00:00|2097.899902|  1.5253349299217422|    -1.525335|  1.0|       0.0|
|2016-07-08 00:00:00|2129.899902| 0.34086155847900335|  -0.34086156|  1.0|       0.0|
|2016-07-11 00:00:00|2137.159912|  0.7009293462734553|  -0.70092934|  1.0|       0.0|
|2016-07-12 00:00:00|2152.139893|0.013476772627251273| -0.013476772|  0.0|       0.0|
|2016-07-13 00:00:00|2152.429932|  0.5259203949780415|   -0.5259204|  1.0|       0.0|
|2016-07-14 00:00:00|    2163.75|-0.09289474292316421|   0.09289474|  0.0|       0.0|
|2016-07-15 00:00:00| 2161.73999| 0.23822952916738127|  -0.23822953|  1.0|       0.0|
|2016-07-18 00:00:00|2166.889893|-0.14351739837110478|    0.1435174|  0.0|       0.0|
|2016-07-19 00:00:00|2163.780029|  0.4270300527854589|  -0.42703006|  1.0|       0.0|
|2016-07-20 00:00:00| 2173.02002| -0.3612529073708161|    0.3612529|  0.0|       0.0|
|2016-07-21 00:00:00|2165.169922| 0.45539645178943006|   0.45539644|  1.0|       1.0|
|2016-07-22 00:00:00|2175.030029|-0.30114752038671483|  -0.30114752|  0.0|       1.0|
|2016-07-25 00:00:00| 2168.47998| 0.03227846263076587|  0.032278463|  0.0|       1.0|
|2016-07-26 00:00:00|2169.179932|-0.11985423438815265| -0.119854234|  0.0|       1.0|
|2016-07-27 00:00:00|2166.580078| 0.16062092674702202|   0.16062093|  1.0|       1.0|
|2016-07-28 00:00:00|2170.060059| 0.16313092282023237|   0.16313092|  1.0|       1.0|
|2016-07-29 00:00:00|2173.600098| -0.1269787392142437|  -0.12697874|  0.0|       1.0|
|2016-08-01 00:00:00|2170.840088| -0.6361619668044343|    -0.636162|  0.0|       1.0|
|2016-08-02 00:00:00|2157.030029| 0.31339433893434987|   0.31339434|  1.0|       1.0|
|2016-08-03 00:00:00|2163.790039|0.021257191858254098|  0.021257192|  0.0|       1.0|
|2016-08-04 00:00:00|    2164.25|  0.8603496361326065|   0.86034966|  1.0|       1.0|
|2016-08-05 00:00:00|2182.870117| -0.0907165288753599| -0.090716526|  0.0|       1.0|
|2016-08-08 00:00:00|2180.889893|0.038979363549189745|  0.038979363|  1.0|       1.0|
|2016-08-09 00:00:00| 2181.73999|-0.28646859977113953|   -0.2864686|  0.0|       1.0|
|2016-08-10 00:00:00| 2175.48999|  0.4734588091577449|    0.4734588|  1.0|       1.0|
|2016-08-11 00:00:00|2185.790039|-0.07960462665462967| -0.079604626|  0.0|       1.0|
|2016-08-12 00:00:00|2184.050049| 0.27929089824626846|   0.27929088|  1.0|       1.0|
|2016-08-15 00:00:00|2190.149902| -0.5479077020728967|    0.5479077|  0.0|       0.0|
|2016-08-16 00:00:00|2178.149902| 0.18685899424381655|     0.186859|  1.0|       1.0|
|2016-08-17 00:00:00|2182.219971|  0.2199617391366979|   0.21996173|  1.0|       1.0|
|2016-08-18 00:00:00| 2187.02002|-0.14402716807320287|  -0.14402717|  0.0|       1.0|
|2016-08-19 00:00:00|2183.870117| -0.0563322878235028| -0.056332286|  0.0|       1.0|
|2016-08-22 00:00:00|2182.639893| 0.19517690543742305|    0.1951769|  1.0|       1.0|
|2016-08-23 00:00:00|2186.899902|   -0.52402768821378|   -0.5240277|  0.0|       1.0|
|2016-08-24 00:00:00|2175.439941|-0.13652273013958158|  -0.13652273|  0.0|       1.0|
|2016-08-25 00:00:00|2172.469971|-0.15788167596264593|  -0.15788168|  0.0|       1.0|
|2016-08-26 00:00:00|2169.040039|  0.5228047337119758|   0.52280474|  1.0|       1.0|
|2016-08-29 00:00:00|2180.379883| -0.1953680655931885|  -0.19536807|  0.0|       1.0|
|2016-08-30 00:00:00|2176.120117|-0.23758642547395195|  -0.23758642|  0.0|       1.0|
|2016-08-31 00:00:00|2170.949951|-0.00413846482083...| -0.004138465|  0.0|       1.0|
|2016-09-01 00:00:00|2170.860107|  0.4201041315648488|   0.42010415|  1.0|       1.0|
|2016-09-02 00:00:00| 2179.97998|  0.2981678758352634|   0.29816788|  1.0|       1.0|
|2016-09-06 00:00:00| 2186.47998|-0.01463850585999...| -0.014638506|  0.0|       1.0|
|2016-09-07 00:00:00|2186.159912| -0.2223013501127694|  -0.22230135|  0.0|       1.0|
|2016-09-08 00:00:00|2181.300049| -2.4522068857295496|   -2.4522069|  0.0|       1.0|
|2016-09-09 00:00:00|2127.810059|   1.467705252539182|    1.4677052|  1.0|       1.0|
|2016-09-12 00:00:00|2159.040039| -1.4830674013266898|   -1.4830674|  0.0|       1.0|
|2016-09-13 00:00:00| 2127.02002|-0.05876766500768526| -0.058767665|  0.0|       1.0|
|2016-09-14 00:00:00| 2125.77002|  1.0109273250546658|    1.0109273|  1.0|       1.0|
|2016-09-15 00:00:00| 2147.26001| -0.3772294907126729|  -0.37722948|  0.0|       1.0|
|2016-09-16 00:00:00|2139.159912|-0.00186030973079...|-0.0018603097|  0.0|       1.0|
|2016-09-19 00:00:00|2139.120117|0.029913841439509457|   0.02991384|  0.0|       1.0|
|2016-09-20 00:00:00| 2139.76001|  1.0917162154086604|    1.0917162|  1.0|       1.0|
|2016-09-21 00:00:00|2163.120117|  0.6499784681166678|   0.64997846|  1.0|       1.0|
|2016-09-22 00:00:00|2177.179932| -0.5736774814255415|    0.5736775|  0.0|       0.0|
|2016-09-23 00:00:00|2164.689941| -0.8587762454059558|    0.8587763|  0.0|       0.0|
|2016-09-26 00:00:00|2146.100098|  0.6444170061260637|     0.644417|  1.0|       1.0|
|2016-09-27 00:00:00|2159.929932|  0.5296553758763286|    0.5296554|  1.0|       1.0|
|2016-09-28 00:00:00|2171.370117| -0.9321411325289909|   -0.9321411|  0.0|       1.0|
|2016-09-29 00:00:00|2151.129883|  0.7967969361336731|    0.7967969|  1.0|       1.0|
|2016-09-30 00:00:00| 2168.27002| -0.3260695824222057|   -0.3260696|  0.0|       1.0|
|2016-10-03 00:00:00|2161.199951|-0.49555623000289534|  -0.49555624|  0.0|       1.0|
|2016-10-04 00:00:00| 2150.48999|  0.4296690541675125|   0.42966905|  1.0|       1.0|
|2016-10-05 00:00:00| 2159.72998| 0.04815601994837626|   0.04815602|  1.0|       1.0|
|2016-10-06 00:00:00| 2160.77002|  -0.325348368171079|  -0.32534838|  0.0|       1.0|
|2016-10-07 00:00:00| 2153.73999|  0.4605905098135844|    0.4605905|  1.0|       1.0|
|2016-10-10 00:00:00|2163.659912| -1.2446471763257407|   -1.2446471|  0.0|       1.0|
|2016-10-11 00:00:00| 2136.72998| 0.11465894253984957|  0.114658944|  1.0|       1.0|
|2016-10-12 00:00:00|2139.179932|-0.30992638350910184|   -0.3099264|  0.0|       1.0|
|2016-10-13 00:00:00|2132.550049|0.020160417815362805|  0.020160418|  0.0|       1.0|
|2016-10-14 00:00:00| 2132.97998|-0.30379938212078617|   -0.3037994|  0.0|       1.0|
|2016-10-17 00:00:00|     2126.5|  0.6160403479896491|   0.61604035|  1.0|       1.0|
|2016-10-18 00:00:00|2139.600098|  0.2191970828746938|   0.21919708|  1.0|       1.0|
|2016-10-19 00:00:00|2144.290039| -0.1375723874264593|   -0.1375724|  0.0|       1.0|
|2016-10-20 00:00:00|2141.340088|-0.00841417022030...|  -0.00841417|  0.0|       1.0|
|2016-10-21 00:00:00|2141.159912|  0.4749839534638121|   0.47498396|  1.0|       1.0|
|2016-10-24 00:00:00|2151.330078|-0.37977277794560177|  -0.37977278|  0.0|       1.0|
|2016-10-25 00:00:00|2143.159912|-0.17404114266579604|  -0.17404114|  0.0|       1.0|
|2016-10-26 00:00:00|2139.429932| -0.2986726933387613|   -0.2986727|  0.0|       1.0|
|2016-10-27 00:00:00|2133.040039| -0.3108299365589125|  -0.31082994|  0.0|       1.0|
|2016-10-28 00:00:00|2126.409912|-0.01222765180563...| -0.012227652|  0.0|       1.0|
|2016-10-31 00:00:00|2126.149902| -0.6786883176217423|  -0.67868835|  0.0|       1.0|
|2016-11-01 00:00:00|2111.719971| -0.6525500629458174|  -0.65255004|  0.0|       1.0|
|2016-11-02 00:00:00|2097.939941|-0.44234006982948293|  -0.44234008|  0.0|       1.0|
|2016-11-03 00:00:00|2088.659912|-0.16661305078948002|  -0.16661306|  0.0|       1.0|
|2016-11-04 00:00:00|2085.179932|   2.222354401596069|    2.2223544|  1.0|       1.0|
|2016-11-07 00:00:00| 2131.52002|  0.3771974424148256|   0.37719744|  1.0|       1.0|
|2016-11-08 00:00:00|2139.560059|  1.1077020670818223|     1.107702|  1.0|       1.0|
|2016-11-09 00:00:00| 2163.26001|  0.1950745624886813|  -0.19507456|  1.0|       0.0|
|2016-11-10 00:00:00| 2167.47998|  -0.139795016699532|  -0.13979502|  0.0|       1.0|
|2016-11-11 00:00:00|2164.449951|-0.01155027862319...| -0.011550278|  0.0|       1.0|
|2016-11-14 00:00:00|2164.199951|  0.7480797692708188|   -0.7480798|  1.0|       0.0|
|2016-11-15 00:00:00|2180.389893|-0.15822638011099693|   0.15822639|  0.0|       0.0|
|2016-11-16 00:00:00|2176.939941|  0.4676369709732771|  -0.46763697|  1.0|       0.0|
|2016-11-17 00:00:00|2187.120117|-0.23867984933357117|   0.23867986|  0.0|       0.0|
|2016-11-18 00:00:00|2181.899902|  0.7461400949272281|    0.7461401|  1.0|       1.0|
|2016-11-21 00:00:00|2198.179932| 0.21654319242507222|   -0.2165432|  1.0|       0.0|
|2016-11-22 00:00:00|2202.939941|  0.0808024752228094|   0.08080248|  1.0|       1.0|
|2016-11-23 00:00:00|2204.719971| 0.39143869124048053|   -0.3914387|  1.0|       0.0|
|2016-11-25 00:00:00|2213.350098| -0.5254535651865005|  -0.52545357|  0.0|       1.0|
|2016-11-28 00:00:00|2201.719971| 0.13352928795321764|   0.13352929|  1.0|       1.0|
|2016-11-29 00:00:00|2204.659912| -0.2653403805348535|   -0.2653404|  0.0|       1.0|
|2016-11-30 00:00:00|2198.810059| -0.3515529214704194|  -0.35155293|  0.0|       1.0|
|2016-12-01 00:00:00|2191.080078|0.039700648494513754|   0.03970065|  1.0|       1.0|
|2016-12-02 00:00:00|2191.949951|  0.5821305360635018|   0.58213055|  1.0|       1.0|
|2016-12-05 00:00:00|2204.709961| 0.34108881136406527|    0.3410888|  1.0|       1.0|
|2016-12-06 00:00:00| 2212.22998|  1.3163241734930209|    1.3163241|  1.0|       1.0|
|2016-12-07 00:00:00|2241.350098|  0.2159342712376298|  -0.21593428|  1.0|       0.0|
|2016-12-08 00:00:00|2246.189941|  0.5938984836723531|    0.5938985|  1.0|       1.0|
|2016-12-09 00:00:00|2259.530029|-0.11374347616603382|   0.11374348|  0.0|       0.0|
|2016-12-12 00:00:00|2256.959961|  0.6539774854251376|   -0.6539775|  1.0|       0.0|
|2016-12-13 00:00:00|2271.719971| -0.8117172114256143|    0.8117172|  0.0|       0.0|
|2016-12-14 00:00:00|2253.280029| 0.38832279554189403|   -0.3883228|  1.0|       0.0|
|2016-12-15 00:00:00|2262.030029|-0.17506226483432866|   0.17506227|  0.0|       0.0|
|2016-12-16 00:00:00|2258.070068| 0.19751207295131734|   0.19751208|  1.0|       1.0|
|2016-12-19 00:00:00|2262.530029| 0.36375123841505275|  -0.36375123|  1.0|       0.0|
|2016-12-20 00:00:00| 2270.76001|-0.24573614012164843|  -0.24573614|  0.0|       1.0|
|2016-12-21 00:00:00|2265.179932|-0.18629738593322426|  -0.18629739|  0.0|       1.0|
|2016-12-22 00:00:00|2260.959961| 0.12517152222139497|   0.12517153|  1.0|       1.0|
|2016-12-23 00:00:00|2263.790039| 0.22483728227059685|   0.22483729|  1.0|       1.0|
|2016-12-27 00:00:00|2268.879883| -0.8356529202828681|  -0.83565295|  0.0|       1.0|
|2016-12-28 00:00:00|2249.919922|-0.02933046610003201| -0.029330466|  0.0|       1.0|
|2016-12-29 00:00:00| 2249.26001|-0.46370503870737506|  -0.46370503|  0.0|       1.0|
|2016-12-30 00:00:00|2238.830078|  0.8486575281753026|   0.84865755|  1.0|       1.0|
+-------------------+-----------+--------------------+-------------+-----+----------+

Logistic Regression Accuracy:
+---+---+----+-----------------+
|tpc|tnc|allc|         accuracy|
+---+---+----+-----------------+
| 96| 26| 252|48.41269841269841|
+---+---+----+-----------------+

Long Only Accuracy:
+-----+----+-----------------+
|truec|allc|         accuracy|
+-----+----+-----------------+
|  132| 252|52.38095238095238|
+-----+----+-----------------+

True Positive Count:
+---+
|tpc|
+---+
| 96|
+---+

False Positive Count:
+---+
|fpc|
+---+
| 94|
+---+

True Negative Count:
+---+
|tnc|
+---+
| 26|
+---+

False Negative Count:
+---+
|fnc|
+---+
| 36|
+---+

Effectiveness of Negative Predictions:
+-------------------+
|             eff_np|
+-------------------+
|-2.4734764526095843|
+-------------------+

Effectiveness of Positive Predictions:
+-----------------+
|           eff_pp|
+-----------------+
|9.870169048774208|
+-----------------+

Effectiveness of Long-Only-Model:
+------------------+
|            eff_lo|
+------------------+
|12.343645501383794|
+------------------+


scala> 
scala> 

For 2016 test data, I translate the above report into English:

Class04 Lab


learn4.us About Blog Contact Class01 Class02 Class03 Class04 Class05 Class06 Class07 Class08 Class09 Class10 dan101 Forum Google Hangout Vboxen