update: scraper, and machine learning model

.github/workflows/gru_pipeline.yaml

@@ -48,9 +48,8 @@ jobs:
         if: env.match != 'true'
         run: |
           mkdir datasets
-          wget https://raw.githubusercontent.com/tebakaja/tebakaja_cryptocurrency/production/postman/symbols.json \
-            -O postman/symbols.json
-          go run scraper.go
+          go run scraper.go \
+            --symbols-file=./postman/symbols_test.json
 
       - name: Install Libraries
         if: env.match != 'true'
@@ -62,7 +61,12 @@ jobs:
           mkdir models
           mkdir pickles
           mkdir posttrained
-          python training.py --algorithm=GRU
+
+          python training.py \
+            --epochs=200     \
+            --batchs=32      \
+            --sequences=5    \
+            --algorithm=GRU
 
       - name: Set Pipeline Schedule
         if: env.match != 'true'

.github/workflows/lstm_gru_pipeline.yaml

@@ -48,9 +48,8 @@ jobs:
         if: env.match != 'true'
         run: |
           mkdir datasets
-          wget https://raw.githubusercontent.com/tebakaja/tebakaja_cryptocurrency/production/postman/symbols.json \
-            -O postman/symbols.json
-          go run scraper.go
+          go run scraper.go \
+            --symbols-file=./postman/symbols_test.json
 
       - name: Install Libraries
         if: env.match != 'true'
@@ -62,7 +61,12 @@ jobs:
           mkdir models
           mkdir pickles
           mkdir posttrained
-          python training.py --algorithm=LSTM_GRU
+
+          python training.py \
+            --epochs=200     \
+            --batchs=32      \
+            --sequences=5    \
+            --algorithm=LSTM_GRU
 
       - name: Set Pipeline Schedule
         if: env.match != 'true'

.github/workflows/lstm_pipeline.yaml

@@ -48,9 +48,8 @@ jobs:
         if: env.match != 'true'
         run: |
           mkdir datasets
-          wget https://raw.githubusercontent.com/tebakaja/tebakaja_cryptocurrency/production/postman/symbols.json \
-            -O postman/symbols.json
-          go run scraper.go
+          go run scraper.go \
+            --symbols-file=./postman/symbols_test.json
 
       - name: Install Libraries
         if: env.match != 'true'
@@ -62,7 +61,12 @@ jobs:
           mkdir models
           mkdir pickles
           mkdir posttrained
-          python training.py --algorithm=LSTM
+
+          python training.py \
+            --epochs=200     \
+            --batchs=32      \
+            --sequences=5    \
+            --algorithm=LSTM
 
       - name: Set Pipeline Schedule
         if: env.match != 'true'

postman/symbols_test.json

@@ -0,0 +1,9 @@
+{
+  "symbols": [
+      "BTC-USD",
+      "ETH-USD",
+      "USDT-USD",
+      "BNB-USD",
+      "SOL-USD"
+  ]
+}

scraper.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"sync"
 	"time"
+	"flag"
 )
 
 
@@ -88,7 +89,11 @@ func getCurrentUnixTimestamp() int64 {
 
 
 func main() {
-	jsonFile, err := os.Open("./postman/symbols.json")
+	symbols_file := flag.String("symbols-file", "default", "symbols file")
+	flag.Parse()
+
+	// jsonFile, err := os.Open("./postman/symbols.json")
+	jsonFile, err := os.Open(*symbols_file)
 	if err != nil {
 		log.Fatalf("[ERROR] failed to open JSON file: %v", err)
 	}

training.py

@@ -167,14 +167,12 @@ class PostProcessor:
         with open(filename, 'w') as f:
             json.dump(data, f)
 
-def main(algorithm: str):
+def main(algorithm: str, sequence_length: int, epochs: int, batch_size: int):
     datasets_path = './datasets'
     models_path   = './models'
     posttrained   = './posttrained'
     pickle_file   = './pickles'
 
-    sequence_length = 60
-    epochs = 200
     batch_size = 32
 
     data_processor = DataProcessor(datasets_path)
@@ -230,10 +228,19 @@ def main(algorithm: str):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description = "Tebakaja Model Trainer")
 
-    parser.add_argument('-a', '--algorithm', 
-        type = str, required = True,
+    parser.add_argument('-a', '--algorithm', type = str, required = True,
         help = 'select the algorithm to be trained (LSTM, GRU, LSTM_GRU)')
 
+    parser.add_argument('-e', '--epochs', type = int, required = True, help = 'epochs')
+    parser.add_argument('-b', '--batchs', type = int, required = True, help = 'batch length')
+    parser.add_argument('-s', '--sequences', type = int, required = True, help = 'sequences length')
+
     args = parser.parse_args()
 
-    main(algorithm = args.algorithm)
+    main(
+        epochs     = args.epochs,
+        batch_size = args.batchs,
+        algorithm  = args.algorithm,
+        sequence_length = args.sequences
+    )
+