Feat: Crafting LSTM, GRU, and LSTM_GRU model

.github/workflows/gru_pipeline.yaml

@@ -0,0 +1,187 @@
+name: GRU Pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'   
+  schedule:
+    - cron: "0 7 * * *"
+    # 14 - 7 = 7
+
+jobs:
+  extraction_train_modeling:
+    name: Data Extraction, Training, and Modeling
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          lfs: true
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Read pipeline schedule date
+        id: read_schedule
+        run: |
+          SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
+          echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
+
+      - name: Get current date
+        id: get_date
+        run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Check if dates match
+        id: date_check
+        run: |
+          if [ "$schedule_date" = "$current_date" ]; then
+            echo "match=true" >> $GITHUB_ENV
+          else
+            echo "match=false" >> $GITHUB_ENV
+          fi
+
+      - name: Scraping Yahoo Finance
+        if: env.match != 'true'
+        run: |
+          mkdir datasets
+          wget https://github.com/belajarqywok/cryptocurrency_prediction/raw/main/postman/symbols.json \
+            -O postman/symbols.json
+          go run scraper.go
+
+      - name: Install Libraries
+        if: env.match != 'true'
+        run: pip install -r requirements.txt
+
+      - name: Modeling and Training
+        if: env.match != 'true'
+        run: |
+          mkdir models
+          mkdir pickles
+          mkdir posttrained
+          python training.py --algorithm=GRU
+
+      - name: Set Pipeline Schedule
+        if: env.match != 'true'
+        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
+          
+      - name: Zip Posttrained, Models, and Pickles
+        if: env.match != 'true'
+        run: |
+          zip -r models.zip models
+          zip -r pickles.zip pickles
+          zip -r datasets.zip datasets
+          zip -r posttrained.zip posttrained
+
+      - name: Store Datasets to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: datasets.zip
+          folderId: ${{ secrets.GDRIVE_CRYPTO_ID }}
+          name: datasets.zip
+          overwrite: "true"
+          
+      - name: Store Models to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: models.zip
+          folderId: ${{ secrets.GDRIVE_GRU_ID }}
+          name: models.zip
+          overwrite: "true"
+
+      - name: Store Pickles to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: pickles.zip
+          folderId: ${{ secrets.GDRIVE_GRU_ID }}
+          name: pickles.zip
+          overwrite: "true"
+
+      - name: Store Posttrained to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: posttrained.zip
+          folderId: ${{ secrets.GDRIVE_GRU_ID }}
+          name: posttrained.zip
+          overwrite: "true"
+
+      - name: Remove Temporarary Files and Directories
+        if: env.match != 'true'
+        run: |
+          rm models.zip
+          rm pickles.zip
+          rm datasets.zip
+          rm posttrained.zip
+
+          rm -rf models
+          rm -rf pickles
+          rm -rf datasets
+          rm -rf posttrained
+
+      - name: Commit changes
+        if: env.match != 'true'
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "belajarqywok"
+          git add -A
+          git commit -m "Data Extraction, Training, and Modeling"
+
+      - name: Push changes
+        if: env.match != 'true'
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GH_TOKEN }}
+          branch: main 
+
+  # model_deployment:
+  #   name: Model Deployment
+  #   runs-on: ubuntu-latest
+  #   needs: extraction_train_modeling
+  #   environment: Production
+
+  #   env:
+  #     HF_TOKEN: ${{ secrets.HF_TOKEN }}
+  #     SPACE_NAME: cryptocurrency_prediction
+  #     HF_USERNAME: qywok
+
+  #   steps:
+  #     - name: Set global directory
+  #       run: git config --global --add safe.directory /github/workspace
+
+  #     - uses: actions/checkout@v3     
+  #       with:
+  #         persist-credentials: false
+  #         fetch-depth: 1000
+
+  #     - name: Check git status
+  #       run: git status
+
+  #     - name: Configure git
+  #       run: |
+  #         git config --local user.email "[email protected]"
+  #         git config --local user.name "qywok"
+
+  #     - name: Pull changes from remote
+  #       run: |
+  #         git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
+  #         (git merge --strategy-option theirs)
+
+  #     - name: Add and commit changes
+  #       run: |
+  #         git add -A 
+  #         git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+  #     - name: Push to Hugging Face
+  #       run: |
+  #         git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force

.github/workflows/lstm_gru_pipeline.yaml

@@ -0,0 +1,187 @@
+name: LSTM GRU Pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'   
+  schedule:
+    - cron: "0 9 * * *"
+    # 16 - 7 = 9
+
+jobs:
+  extraction_train_modeling:
+    name: Data Extraction, Training, and Modeling
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          lfs: true
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Read pipeline schedule date
+        id: read_schedule
+        run: |
+          SCHEDULE_DATE=$(cat schedulers/lstm_gru_schedule.ctl)
+          echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
+
+      - name: Get current date
+        id: get_date
+        run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Check if dates match
+        id: date_check
+        run: |
+          if [ "$schedule_date" = "$current_date" ]; then
+            echo "match=true" >> $GITHUB_ENV
+          else
+            echo "match=false" >> $GITHUB_ENV
+          fi
+
+      - name: Scraping Yahoo Finance
+        if: env.match != 'true'
+        run: |
+          mkdir datasets
+          wget https://github.com/belajarqywok/cryptocurrency_prediction/raw/main/postman/symbols.json \
+            -O postman/symbols.json
+          go run scraper.go
+
+      - name: Install Libraries
+        if: env.match != 'true'
+        run: pip install -r requirements.txt
+
+      - name: Modeling and Training
+        if: env.match != 'true'
+        run: |
+          mkdir models
+          mkdir pickles
+          mkdir posttrained
+          python training.py --algorithm=LSTM_GRU
+
+      - name: Set Pipeline Schedule
+        if: env.match != 'true'
+        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
+          
+      - name: Zip Posttrained, Models, and Pickles
+        if: env.match != 'true'
+        run: |
+          zip -r models.zip models
+          zip -r pickles.zip pickles
+          zip -r datasets.zip datasets
+          zip -r posttrained.zip posttrained
+
+      - name: Store Datasets to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: datasets.zip
+          folderId: ${{ secrets.GDRIVE_CRYPTO_ID }}
+          name: datasets.zip
+          overwrite: "true"
+          
+      - name: Store Models to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: models.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: models.zip
+          overwrite: "true"
+
+      - name: Store Pickles to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: pickles.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: pickles.zip
+          overwrite: "true"
+
+      - name: Store Posttrained to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: posttrained.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_GRU_ID }}
+          name: posttrained.zip
+          overwrite: "true"
+
+      - name: Remove Temporarary Files and Directories
+        if: env.match != 'true'
+        run: |
+          rm models.zip
+          rm pickles.zip
+          rm datasets.zip
+          rm posttrained.zip
+
+          rm -rf models
+          rm -rf pickles
+          rm -rf datasets
+          rm -rf posttrained
+
+      - name: Commit changes
+        if: env.match != 'true'
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "belajarqywok"
+          git add -A
+          git commit -m "Data Extraction, Training, and Modeling"
+
+      - name: Push changes
+        if: env.match != 'true'
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GH_TOKEN }}
+          branch: main 
+
+  # model_deployment:
+  #   name: Model Deployment
+  #   runs-on: ubuntu-latest
+  #   needs: extraction_train_modeling
+  #   environment: Production
+
+  #   env:
+  #     HF_TOKEN: ${{ secrets.HF_TOKEN }}
+  #     SPACE_NAME: cryptocurrency_prediction
+  #     HF_USERNAME: qywok
+
+  #   steps:
+  #     - name: Set global directory
+  #       run: git config --global --add safe.directory /github/workspace
+
+  #     - uses: actions/checkout@v3     
+  #       with:
+  #         persist-credentials: false
+  #         fetch-depth: 1000
+
+  #     - name: Check git status
+  #       run: git status
+
+  #     - name: Configure git
+  #       run: |
+  #         git config --local user.email "[email protected]"
+  #         git config --local user.name "qywok"
+
+  #     - name: Pull changes from remote
+  #       run: |
+  #         git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
+  #         (git merge --strategy-option theirs)
+
+  #     - name: Add and commit changes
+  #       run: |
+  #         git add -A 
+  #         git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+  #     - name: Push to Hugging Face
+  #       run: |
+  #         git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force

.github/workflows/lstm_pipeline.yaml

@@ -0,0 +1,187 @@
+name: LSTM Pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'   
+  schedule:
+    - cron: "0 11 * * *"
+    # 18 - 7 = 11
+
+jobs:
+  extraction_train_modeling:
+    name: Data Extraction, Training, and Modeling
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+          lfs: true
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Read pipeline schedule date
+        id: read_schedule
+        run: |
+          SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
+          echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
+
+      - name: Get current date
+        id: get_date
+        run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Check if dates match
+        id: date_check
+        run: |
+          if [ "$schedule_date" = "$current_date" ]; then
+            echo "match=true" >> $GITHUB_ENV
+          else
+            echo "match=false" >> $GITHUB_ENV
+          fi
+
+      - name: Scraping Yahoo Finance
+        if: env.match != 'true'
+        run: |
+          mkdir datasets
+          wget https://github.com/belajarqywok/cryptocurrency_prediction/raw/main/postman/symbols.json \
+            -O postman/symbols.json
+          go run scraper.go
+
+      - name: Install Libraries
+        if: env.match != 'true'
+        run: pip install -r requirements.txt
+
+      - name: Modeling and Training
+        if: env.match != 'true'
+        run: |
+          mkdir models
+          mkdir pickles
+          mkdir posttrained
+          python training.py --algorithm=LSTM
+
+      - name: Set Pipeline Schedule
+        if: env.match != 'true'
+        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
+          
+      - name: Zip Posttrained, Models, and Pickles
+        if: env.match != 'true'
+        run: |
+          zip -r models.zip models
+          zip -r pickles.zip pickles
+          zip -r datasets.zip datasets
+          zip -r posttrained.zip posttrained
+
+      - name: Store Datasets to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: datasets.zip
+          folderId: ${{ secrets.GDRIVE_CRYPTO_ID }}
+          name: datasets.zip
+          overwrite: "true"
+          
+      - name: Store Models to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: models.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_ID }}
+          name: models.zip
+          overwrite: "true"
+
+      - name: Store Pickles to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: pickles.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_ID }}
+          name: pickles.zip
+          overwrite: "true"
+
+      - name: Store Posttrained to Google Drive 
+        if: env.match != 'true'
+        uses: adityak74/google-drive-upload-git-action@main
+        with:
+          credentials: ${{ secrets.GDRIVE_CRED }}
+          filename: posttrained.zip
+          folderId: ${{ secrets.GDRIVE_LSTM_ID }}
+          name: posttrained.zip
+          overwrite: "true"
+
+      - name: Remove Temporarary Files and Directories
+        if: env.match != 'true'
+        run: |
+          rm models.zip
+          rm pickles.zip
+          rm datasets.zip
+          rm posttrained.zip
+
+          rm -rf models
+          rm -rf pickles
+          rm -rf datasets
+          rm -rf posttrained
+
+      - name: Commit changes
+        if: env.match != 'true'
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "belajarqywok"
+          git add -A
+          git commit -m "Data Extraction, Training, and Modeling"
+
+      - name: Push changes
+        if: env.match != 'true'
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GH_TOKEN }}
+          branch: main 
+
+  # model_deployment:
+  #   name: Model Deployment
+  #   runs-on: ubuntu-latest
+  #   needs: extraction_train_modeling
+  #   environment: Production
+
+  #   env:
+  #     HF_TOKEN: ${{ secrets.HF_TOKEN }}
+  #     SPACE_NAME: cryptocurrency_prediction
+  #     HF_USERNAME: qywok
+
+  #   steps:
+  #     - name: Set global directory
+  #       run: git config --global --add safe.directory /github/workspace
+
+  #     - uses: actions/checkout@v3     
+  #       with:
+  #         persist-credentials: false
+  #         fetch-depth: 1000
+
+  #     - name: Check git status
+  #       run: git status
+
+  #     - name: Configure git
+  #       run: |
+  #         git config --local user.email "[email protected]"
+  #         git config --local user.name "qywok"
+
+  #     - name: Pull changes from remote
+  #       run: |
+  #         git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
+  #         (git merge --strategy-option theirs)
+
+  #     - name: Add and commit changes
+  #       run: |
+  #         git add -A 
+  #         git diff-index --quiet HEAD || git commit -m "Model Deployment"
+
+  #     - name: Push to Hugging Face
+  #       run: |
+  #         git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force

.gitignore

@@ -0,0 +1,19 @@
+# Postman
+/postman/dataset.url
+
+# Environments
+/bin
+/Lib
+/lib64
+/Include
+/Scripts
+
+# Pycache
+/__pycache__
+/restful/__pycache__
+
+# Temp
+/models
+/pickles
+/datasets
+/posttrained

.vercelignore

@@ -0,0 +1,7 @@
+/.github
+
+/bin
+/include
+/lib
+
+/postman

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.9-bullseye
+
+LABEL organization="R6Q - Infraprasta University"
+LABEL team="Group 5"
+
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+RUN apt-get update && \
+    apt-get install -y gcc python3-dev gnupg curl
+
+RUN pip install cython
+
+RUN cd /app/restful/cutils && \
+    python setup.py build_ext --inplace && \
+    chmod 777 * && cd ../..
+
+RUN pip install gdown
+
+RUN --mount=type=secret,id=MODELS_ID,mode=0444,required=true \
+	gdown https://drive.google.com/uc?id=$(cat /run/secrets/MODELS_ID) && \
+    unzip models.zip && rm models.zip
+
+RUN --mount=type=secret,id=PICKLES_ID,mode=0444,required=true \
+	gdown https://drive.google.com/uc?id=$(cat /run/secrets/PICKLES_ID) && \
+    unzip pickles.zip && rm pickles.zip
+
+RUN --mount=type=secret,id=DATASETS_ID,mode=0444,required=true \
+	gdown https://drive.google.com/uc?id=$(cat /run/secrets/DATASETS_ID) && \
+    unzip datasets.zip && rm datasets.zip
+
+RUN --mount=type=secret,id=POSTTRAINED_ID,mode=0444,required=true \
+	gdown https://drive.google.com/uc?id=$(cat /run/secrets/POSTTRAINED_ID) && \
+    unzip posttrained.zip && rm posttrained.zip
+
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--workers", "10", "--port", "7860"]

Makefile

@@ -0,0 +1,2 @@
+cutils:
+	cd restful/cutils && python setup.py build_ext --inplace && cd ../..

README.md

@@ -0,0 +1,16 @@
+---
+title: Cryptocurrency Prediction Service
+emoji: 📉
+colorFrom: purple
+colorTo: pink
+sdk: docker
+python_version: "3.10"
+pinned: false
+---
+
+
+<h1 align="center">Cryptocurrency Prediction Service</h1>
+<hr />
+<p align="center">
+  <img src="./diagram/cryptocurrency_prediction.jpg"/>
+</p>

app.py

@@ -0,0 +1,28 @@
+from fastapi import FastAPI
+from restful.routes import route
+from fastapi.responses import RedirectResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI(
+    title = "Cryptocurency Prediction Service",
+    version = "1.0"
+)
+
+# CORS Middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins = ["*"],
+    allow_methods = ["*"],
+    allow_headers = ["*"],
+    allow_credentials = True,
+)
+
+app.include_router(
+    router = route,
+    prefix = '/crypto',
+    tags = ['Crypto']
+)
+
+@app.get("/", tags = ['Main'])
+def root():
+    return RedirectResponse(url="/docs")

converter.py

@@ -0,0 +1,40 @@
+import json
+
+"""
+
+    Data Mining Assignment - Group 5
+
+"""
+
+class JSONProcessor:
+    def __init__(self, input_file: str, output_file: str) -> None:
+        self.input_file:  str = input_file
+        self.output_file: str = output_file
+        self.data = None
+
+    def load_json(self) -> None:
+        with open(self.input_file, 'r') as file:
+            self.data = json.load(file)
+
+    def extract_symbols(self) -> list:
+        if self.data is None:
+            raise ValueError("data not loaded. call load_json() first.")
+        quotes = self.data['finance']['result'][0]['quotes']
+        return [quote['symbol'] for quote in quotes]
+
+    def save_json(self, data: list) -> None:
+        with open(self.output_file, 'w') as file:
+            json.dump({'symbols': data}, file, indent = 4)
+            print(f'saved: {self.output_file}')
+
+def main():
+    input_file  = './postman/response.json'
+    output_file = './postman/symbols.json'
+    
+    processor = JSONProcessor(input_file, output_file)
+    processor.load_json()
+    symbols = processor.extract_symbols()
+    processor.save_json(symbols)
+
+
+if __name__ == "__main__": main()

dev.requirements.txt

@@ -0,0 +1,74 @@
+absl-py==2.1.0
+annotated-types==0.7.0
+anyio==4.4.0
+astunparse==1.6.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+dnspython==2.6.1
+email_validator==2.1.1
+exceptiongroup==1.2.1
+fastapi==0.111.0
+fastapi-cli==0.0.4
+flatbuffers==24.3.25
+gast==0.5.4
+google-pasta==0.2.0
+grpcio==1.64.0
+h11==0.14.0
+h5py==3.11.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+idna==3.7
+importlib_metadata==7.1.0
+Jinja2==3.1.4
+joblib==1.4.2
+keras==3.3.3
+libclang==18.1.1
+Markdown==3.6
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+ml-dtypes==0.3.2
+namex==0.0.8
+numpy==1.26.4
+opt-einsum==3.3.0
+optree==0.11.0
+orjson==3.10.3
+packaging==24.0
+pandas==2.2.2
+protobuf==4.25.3
+pydantic==2.7.2
+pydantic_core==2.18.3
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+requests==2.32.3
+rich==13.7.1
+scikit-learn==1.5.0
+scipy==1.13.1
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.37.2
+tensorboard==2.16.2
+tensorboard-data-server==0.7.2
+tensorflow==2.16.1
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==2.4.0
+threadpoolctl==3.5.0
+typer==0.12.3
+typing_extensions==4.12.1
+tzdata==2024.1
+ujson==5.10.0
+urllib3==2.2.1
+uvicorn==0.30.1
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==12.0
+Werkzeug==3.0.3
+wrapt==1.16.0
+zipp==3.19.1

go.mod

@@ -0,0 +1,3 @@
+module github.com/cryptocurrency_prediction
+
+go 1.20

postman/Yahoo Finance.postman_collection.json

@@ -0,0 +1,69 @@
+{
+	"info": {
+		"_postman_id": "249fd388-44f6-45c2-9ad5-37da9c2af089",
+		"name": "Yahoo Finance",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "cryptocurrencies",
+			"request": {
+				"method": "POST",
+				"header": [
+					{
+						"key": "Cookie",
+						"value": "GUC=AQEBCAFmWUlmh0IaaAQw&s=AQAAAH-PIsT_&g=ZlgBjg; A1=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; A3=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; axids=gam=y-BdfSS7lE2uLV0LrGZqbRPm.8FUDjf.82~A&dv360=eS1EdjNSYkpGRTJ1R2RYQTAwYnNhcFJmQ0ZZN3BtTmNGan5B&ydsp=y-wmHAUIFE2uKC4PXfccNh1ff.Lz1oO0cj~A&tbla=y-gt8RDdJE2uKuvojQP3_mil11ZyoZelyw~A; tbla_id=f1c3e4ae-853f-47af-ba52-d13fe18de92e-tuctd4c1d85; PRF=t%3DBTC-USD%252BETH-USD%252BLTC-USD%252BLTC-INR%252BCU%253DF%26newChartbetateaser%3D0%252C1718255372183; A1S=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; cmp=t=1717308407&j=0&u=1---; gpp=DBAA; gpp_sid=-1",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\"offset\":0,\"size\":50,\"sortType\":\"DESC\",\"sortField\":\"intradaymarketcap\",\"quoteType\":\"CRYPTOCURRENCY\",\"query\":{\"operator\":\"and\",\"operands\":[{\"operator\":\"eq\",\"operands\":[\"currency\",\"USD\"]},{\"operator\":\"eq\",\"operands\":[\"exchange\",\"CCC\"]}]},\"userId\":\"\",\"userIdType\":\"guid\"}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://query2.finance.yahoo.com/v1/finance/screener?crumb=55ovV9srjcg&lang=en-US&region=US&formatted=true&corsDomain=finance.yahoo.com",
+					"protocol": "https",
+					"host": [
+						"query2",
+						"finance",
+						"yahoo",
+						"com"
+					],
+					"path": [
+						"v1",
+						"finance",
+						"screener"
+					],
+					"query": [
+						{
+							"key": "crumb",
+							"value": "55ovV9srjcg"
+						},
+						{
+							"key": "lang",
+							"value": "en-US"
+						},
+						{
+							"key": "region",
+							"value": "US"
+						},
+						{
+							"key": "formatted",
+							"value": "true"
+						},
+						{
+							"key": "corsDomain",
+							"value": "finance.yahoo.com"
+						}
+					]
+				}
+			},
+			"response": []
+		}
+	]
+}

postman/symbols.json

@@ -0,0 +1,54 @@
+{
+    "symbols": [
+        "BTC-USD",
+        "ETH-USD",
+        "USDT-USD",
+        "BNB-USD",
+        "SOL-USD",
+        "STETH-USD",
+        "USDC-USD",
+        "XRP-USD",
+        "DOGE-USD",
+        "ADA-USD",
+        "TON11419-USD",
+        "SHIB-USD",
+        "AVAX-USD",
+        "WSTETH-USD",
+        "WETH-USD",
+        "LINK-USD",
+        "WBTC-USD",
+        "DOT-USD",
+        "TRX-USD",
+        "WTRX-USD",
+        "BCH-USD",
+        "NEAR-USD",
+        "MATIC-USD",
+        "LTC-USD",
+        "PEPE24478-USD",
+        "EETH-USD",
+        "UNI7083-USD",
+        "ICP-USD",
+        "LEO-USD",
+        "DAI-USD",
+        "WEETH-USD",
+        "ETC-USD",
+        "EZETH-USD",
+        "APT21794-USD",
+        "RNDR-USD",
+        "BTCB-USD",
+        "HBAR-USD",
+        "WHBAR-USD",
+        "WBETH-USD",
+        "IMX10603-USD",
+        "KAS-USD",
+        "ATOM-USD",
+        "ARB11841-USD",
+        "MNT27075-USD",
+        "FIL-USD",
+        "WIF-USD",
+        "XLM-USD",
+        "USDE29470-USD",
+        "CRO-USD",
+        "AR-USD"
+    ]
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "cryptocurrency-prediction"
+version = "0.1.0"
+description = "Data Mining Assignment - Group 5"
+authors = ["belajarqywok <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

pyvenv.cfg

@@ -0,0 +1,3 @@
+home = /usr/bin
+include-system-site-packages = false
+version = 3.10.12

requirements.txt

@@ -0,0 +1,13 @@
+fastapi==0.111.0
+h5py==3.11.0
+joblib==1.4.2
+keras==3.3.3
+numpy==1.26.4
+pandas==2.2.2
+protobuf==4.25.3
+pydantic==2.7.2
+pydantic_core==2.18.3
+scikit-learn==1.5.0
+scipy==1.13.1
+tensorflow==2.16.1
+uvicorn==0.30.1

restful/controllers.py

@@ -0,0 +1,110 @@
+import os
+from http import HTTPStatus
+from fastapi.responses import JSONResponse
+from restful.services import cryptocurrency_svc
+from restful.schemas import CryptocurrencyPredictionSchema
+
+
+# Cryptocurrency Controller
+class cryptocurrency_controller:
+    # Cryptocurrency Service
+    __SERVICE = cryptocurrency_svc()
+
+    # Cryptocurrency List
+    async def crypto_list(self) -> JSONResponse:
+        try:
+            DATASETS_PATH = './datasets'
+            DATASETS = sorted(
+                [
+                    item.replace(".csv", "") for item in os.listdir(DATASETS_PATH)
+                    if os.path.isfile(os.path.join(DATASETS_PATH, item)) and item.endswith('.csv')
+                ]
+            )
+
+            return JSONResponse(
+                content = {
+                    'message': 'Success',
+                    'status_code': HTTPStatus.OK,
+                    'data': DATASETS
+                },
+                status_code = HTTPStatus.OK
+            )
+
+        except Exception as error_message:
+            print(error_message)
+            return JSONResponse(
+                content = {
+                    'message': 'Internal Server Error',
+                    'status_code': HTTPStatus.INTERNAL_SERVER_ERROR,
+                    'data': None
+                },
+                status_code = HTTPStatus.INTERNAL_SERVER_ERROR
+            )
+
+    # Cryptocurrency Controller
+    async def prediction(self, payload: CryptocurrencyPredictionSchema) -> JSONResponse:
+        try:
+            DATASETS_PATH = './datasets'
+            DATASETS = sorted(
+                [
+                    item.replace(".csv", "") for item in os.listdir(DATASETS_PATH)
+                    if os.path.isfile(os.path.join(DATASETS_PATH, item)) and item.endswith('.csv')
+                ]
+            )
+
+            # Validation
+            if (payload.days > 31) or (payload.days < 1):
+                return JSONResponse(
+                    content = {
+                        'message': 'prediction days cannot be more than a month and cannot be less than 1',
+                        'status_code': HTTPStatus.BAD_REQUEST,
+                        'data': None
+                    },
+                    status_code = HTTPStatus.BAD_REQUEST
+                )
+
+            if payload.currency not in DATASETS:
+                return JSONResponse(
+                    content = {
+                        'message': f'cryptocurrency {payload.currency} is not available.',
+                        'status_code': HTTPStatus.BAD_REQUEST,
+                        'data': None
+                    },
+                    status_code = HTTPStatus.BAD_REQUEST
+                )
+
+
+            prediction: dict = await self.__SERVICE.prediction(payload)
+
+            if not prediction :
+                return JSONResponse(
+                    content = {
+                        'message': 'prediction could not be generated, please try again.',
+                        'status_code': HTTPStatus.BAD_REQUEST,
+                        'data': None
+                    },
+                    status_code = HTTPStatus.BAD_REQUEST
+                )
+
+            return JSONResponse(
+                content = {
+                    'message': 'prediction success',
+                    'status_code': HTTPStatus.OK,
+                    'data': {
+						'currency': payload.currency,
+						'predictions': prediction
+					}
+                },
+                status_code = HTTPStatus.OK
+            )
+
+        except Exception as error_message:
+            print(error_message)
+            return JSONResponse(
+                content = {
+                    'message': 'internal server error',
+                    'status_code': HTTPStatus.INTERNAL_SERVER_ERROR,
+                    'data': None
+                },
+                status_code = HTTPStatus.INTERNAL_SERVER_ERROR
+            )

restful/cutils/setup.py

@@ -0,0 +1,9 @@
+from setuptools import setup
+from Cython.Build import cythonize
+import numpy
+
+setup(
+    ext_modules=cythonize("utilities.pyx"),
+    include_dirs=[numpy.get_include()]
+)
+

restful/cutils/utilities.pyx

@@ -0,0 +1,56 @@
+import os
+from joblib import load
+from numpy import append, expand_dims
+from pandas import read_json, to_datetime, Timedelta
+from tensorflow.keras.models import load_model
+import cython
+
+cdef class Utilities:
+    async def cryptocurrency_prediction_utils(self,
+        int days, int sequence_length, str model_name) -> tuple:
+        cdef str model_path = os.path.join('./models', f'{model_name}.keras')
+        model = load_model(model_path)
+
+        cdef str dataframe_path = os.path.join('./posttrained', f'{model_name}-posttrained.json')
+        dataframe = read_json(dataframe_path)
+        dataframe.set_index('Date', inplace=True)
+
+        minmax_scaler = load(os.path.join('./pickles', f'{model_name}_minmax_scaler.pickle'))
+        standard_scaler = load(os.path.join('./pickles', f'{model_name}_standard_scaler.pickle'))
+        
+        # Prediction
+        lst_seq = dataframe[-sequence_length:].values
+        lst_seq = expand_dims(lst_seq, axis=0)
+
+        cdef dict predicted_prices = {}
+        last_date = to_datetime(dataframe.index[-1])
+
+        for _ in range(days):
+            predicted_price = model.predict(lst_seq)
+            last_date = last_date + Timedelta(days=1)
+
+            predicted_prices[last_date] = minmax_scaler.inverse_transform(predicted_price)
+            predicted_prices[last_date] = standard_scaler.inverse_transform(predicted_prices[last_date])
+
+            lst_seq = append(lst_seq[:, 1:, :], [predicted_price], axis=1)
+
+        predictions = [
+            {'date': date.strftime('%Y-%m-%d'), 'price': float(price)} \
+                for date, price in predicted_prices.items()
+        ]
+
+        # Actual
+        df_date = dataframe.index[-sequence_length:].values
+        df_date = [to_datetime(date) for date in df_date]
+
+        dataframe[['Close']] = minmax_scaler.inverse_transform(dataframe)
+        dataframe[['Close']] = standard_scaler.inverse_transform(dataframe)
+        df_close = dataframe.iloc[-sequence_length:]['Close'].values
+
+        actuals = [
+            {'date': date.strftime('%Y-%m-%d'), 'price': close} \
+                for date, close in zip(df_date, df_close)
+        ]
+        
+        return actuals, predictions
+

restful/routes.py

@@ -0,0 +1,25 @@
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse
+from restful.controllers import cryptocurrency_controller
+from restful.schemas import CryptocurrencyPredictionSchema
+
+# Route
+route = APIRouter()
+
+# Controller
+__CONTROLLER = cryptocurrency_controller()
+
+# Cryptocurrency List
+@route.get(path = '/lists')
+async def cryptocurrency_list_route() -> JSONResponse:
+    # Cryptocurrency Controller
+    return await __CONTROLLER.crypto_list()
+
+# Cryptocurrency Prediction
+@route.post(path = '/prediction')
+async def cryptocurrency_pred_route(
+    payload: CryptocurrencyPredictionSchema = Body(...)
+) -> JSONResponse:
+    # Cryptocurrency Controller
+    return await __CONTROLLER.prediction(payload = payload)
+

restful/schemas.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+class CryptocurrencyPredictionSchema(BaseModel) :
+	days: int
+	currency: str
+	

restful/services.py

@@ -0,0 +1,19 @@
+from restful.cutils.utilities import Utilities
+from restful.schemas import CryptocurrencyPredictionSchema
+
+class cryptocurrency_svc:
+	# Prediction Utilities
+	__PRED_UTILS = Utilities()
+
+	# Prediction Service
+	async def prediction(self, payload: CryptocurrencyPredictionSchema) -> dict:
+		days: int     = payload.days
+		currency: str = payload.currency
+
+		actuals, predictions = await self.__PRED_UTILS.cryptocurrency_prediction_utils(
+			days            = days,
+			model_name      = currency,
+			sequence_length = 60
+		)
+
+		return {'actuals': actuals, 'predictions': predictions}

restful/utilities.py

@@ -0,0 +1,61 @@
+import os
+from joblib import load
+from numpy import append, expand_dims
+from pandas import read_json, to_datetime, Timedelta
+
+from tensorflow.keras.models import load_model
+
+
+class Utilities:
+	def __init__(self) -> None:
+		self.model_path = './models'
+		self.posttrained_path = './posttrained'
+		self.scaler_path = './pickles'
+
+	async def cryptocurrency_prediction_utils(self,
+		days: int, sequence_length: int, model_name: str) -> tuple:
+		model_path = os.path.join(self.model_path, f'{model_name}.keras')
+		model = load_model(model_path)
+
+		dataframe_path = os.path.join(self.posttrained_path, f'{model_name}-posttrained.json')
+		dataframe = read_json(dataframe_path)
+		dataframe.set_index('Date', inplace = True)
+
+		minmax_scaler = load(os.path.join(self.scaler_path, f'{model_name}_minmax_scaler.pickle'))
+		standard_scaler = load(os.path.join(self.scaler_path, f'{model_name}_standard_scaler.pickle'))
+
+		lst_seq = dataframe[-sequence_length:].values
+		lst_seq = expand_dims(lst_seq, axis = 0)
+
+		# Predicted
+		predicted_prices = {}
+		last_date = to_datetime(dataframe.index[-1])
+
+		for _ in range(days):
+			predicted_price = model.predict(lst_seq)
+			last_date       = last_date + Timedelta(days = 1)
+
+			predicted_prices[last_date] = minmax_scaler.inverse_transform(predicted_price)
+			predicted_prices[last_date] = standard_scaler.inverse_transform(predicted_prices[last_date])
+
+			lst_seq = append(lst_seq[:, 1:, :], [predicted_price], axis = 1)
+
+		predictions = [
+			{'date': date.strftime('%Y-%m-%d'), 'price': float(price)} \
+				for date, price in predicted_prices.items()
+    ]
+
+		# Actual
+		df_date = dataframe.index[-sequence_length:].values
+		df_date = [to_datetime(date) for date in df_date]
+
+		dataframe[['Close']] = minmax_scaler.inverse_transform(dataframe)
+		dataframe[['Close']] = standard_scaler.inverse_transform(dataframe)
+		df_close = dataframe.iloc[-sequence_length:]['Close'].values
+
+		actuals = [
+			{'date': date.strftime('%Y-%m-%d'), 'price': close} \
+				for date, close in zip(df_date, df_close)
+		]
+        
+		return actuals, predictions

schedulers/gru_schedule.ctl

@@ -0,0 +1 @@
+2024-07-28

schedulers/lstm_gru_schedule.ctl

@@ -0,0 +1 @@
+2024-07-28

schedulers/lstm_schedule.ctl

@@ -0,0 +1 @@
+2024-07-28