.

.dockerignore

@@ -0,0 +1,10 @@
+.venv
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.log
+.git
+.gitignore
+.cache
+/root/.cache

Dockerfile

@@ -0,0 +1,26 @@
+# Dockerfile
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 libgl1 && \
+    rm -rf /var/lib/apt/lists/*
+
+# ✅ 캐시/토큰 경로를 /data로 강제
+ENV HF_HOME=/data \
+    TRANSFORMERS_CACHE=/data/transformers \
+    HF_HUB_CACHE=/data/hub \
+    HF_HUB_DISABLE_TELEMETRY=1 \
+    TOKENIZERS_PARALLELISM=false \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# ✅ 디렉터리 생성 + 퍼미션(쓰기 가능)
+RUN mkdir -p /data/transformers /data/hub && chmod -R 777 /data
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+COPY . /app
+
+EXPOSE 7860
+CMD ["sh", "-c", "uvicorn main:app --host 0.0.0.0 --port ${PORT:-7860} --workers 1"]

main.py

@@ -0,0 +1,160 @@
+import os
+import threading
+from typing import List, Dict, Iterator
+
+import torch
+from fastapi import FastAPI, Body
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel, Field
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    TextIteratorStreamer,
+    Conversation, # granite 모델의 apply_chat_template을 위해 추가
+)
+# from peft import PeftModel # LoRA 미사용으로 제거
+
+# ----------------- 환경 기본값 -----------------
+# Hugging Face 캐시/토큰 경로를 쓰기 가능한 위치로 지정 (Spaces에서는 /data가 안전)
+os.environ["HF_HOME"] = "/data"
+os.environ["TRANSFORMERS_CACHE"] = "/data/transformers"
+os.environ["HF_HUB_CACHE"] = "/data/hub"
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("PYTORCH_FORCE_MPS_FALLBACK", "1")
+
+# CPU-only: 4bit 비활성화, float32 (granite-1b는 4bit 지원 정보가 없으므로 float16/bfloat16 대신 float32 사용)
+USE_4BIT = False
+COMPUTE_DTYPE = torch.float32
+
+# 베이스 모델 경로 (granite 모델로 변경)
+MODEL_ID = "ibm-granite/granite-4.0-h-1b"
+# ADAPTER_ID는 사용하지 않음
+
+# 디바이스 설정: GPU 사용 가능 여부 확인
+if torch.cuda.is_available():
+    device_map = "cuda"
+    COMPUTE_DTYPE = torch.bfloat16 # GPU 사용 시 bfloat16을 사용하여 메모리 절약 및 성능 향상 (float32도 가능)
+elif torch.backends.mps.is_available():
+    device_map = "mps"
+    COMPUTE_DTYPE = torch.float32
+else:
+    device_map = "cpu"
+    COMPUTE_DTYPE = torch.float32
+
+# 스레드 수
+try:
+    torch.set_num_threads(max(1, os.cpu_count() or 1))
+except Exception:
+    pass
+
+# ----------------- 로드 -----------------
+print(f"[BOOT] Base: {MODEL_ID}")
+print(f"[BOOT] Device: {device_map}")
+print(f"[BOOT] DType: {COMPUTE_DTYPE}")
+
+# 토크나이저 로드
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=False, trust_remote_code=True)
+print("[BOOT] Tokenizer loaded.")
+
+# pad 토큰 보정 (granite 모델의 경우 필요할 수 있음)
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+# 베이스 모델 로드
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    device_map=device_map,
+    trust_remote_code=True,
+    torch_dtype=COMPUTE_DTYPE,
+    low_cpu_mem_usage=True if device_map == "cpu" else False,
+)
+model.eval()
+
+# ----------------- API 스키마/앱 -----------------
+class ChatMessage(BaseModel):
+    role: str = Field(..., description="system | user | assistant")
+    content: str
+
+class ChatRequest(BaseModel):
+    messages: List[ChatMessage]
+    max_new_tokens: int = 128
+    temperature: float = 0.7
+    top_p: float = 0.9
+    repetition_penalty: float = 1.1
+
+class ChatResponse(BaseModel):
+    text: str
+
+app = FastAPI(title="IBM Granite 4.0 H 1B API")
+
+@app.get("/")
+def health():
+    return {"status": "ok", "base": MODEL_ID, "device": device_map, "use_4bit": USE_4BIT}
+
+def build_prompt(messages: List[Dict[str, str]]) -> str:
+    # granite 모델은 Hugging Face `Conversation`을 사용한 템플릿 적용이 권장됨
+    hf_messages = [
+        {"role": m["role"], "content": m["content"]}
+        for m in messages
+    ]
+    # Conversation 객체를 생성하여 apply_chat_template에 전달
+    conversation = Conversation(hf_messages)
+    
+    return tokenizer.apply_chat_template(
+        conversation,
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+
+@app.post("/v1/chat", response_model=ChatResponse)
+def chat(req: ChatRequest):
+    prompt = build_prompt([m.dict() for m in req.messages])
+    inputs = tokenizer(prompt, return_tensors="pt")
+    # 모델의 디바이스로 이동
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+
+    with torch.no_grad():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=req.max_new_tokens,
+            do_sample=True,
+            temperature=req.temperature,
+            top_p=req.top_p,
+            repetition_penalty=req.repetition_penalty,
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+
+    # 입력 토큰을 제외한 생성된 텍스트만 디코딩
+    text = tokenizer.decode(output_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
+    return ChatResponse(text=text)
+
+def stream_generate(req: ChatRequest) -> Iterator[str]:
+    prompt = build_prompt([m.dict() for m in req.messages])
+    inputs = tokenizer(prompt, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    # skip_prompt=True로 설정하여 스트림에 프롬프트가 포함되지 않도록 함
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    gen_kwargs = dict(
+        **inputs,
+        max_new_tokens=req.max_new_tokens,
+        do_sample=True,
+        temperature=req.temperature,
+        top_p=req.top_p,
+        repetition_penalty=req.repetition_penalty,
+        eos_token_id=tokenizer.eos_token_id,
+        pad_token_id=tokenizer.eos_token_id,
+        streamer=streamer,
+    )
+
+    thread = threading.Thread(target=model.generate, kwargs=gen_kwargs)
+    thread.start()
+
+    for token_text in streamer:
+        # 스트리밍 응답 형식에 맞게 JSON 형태로 yield
+        yield f'{{"delta": {token_text.__repr__()}}}\n'
+
+@app.post("/v1/chat/stream")
+def chat_stream(req: ChatRequest = Body(...)):
+    return StreamingResponse(stream_generate(req), media_type="application/x-ndjson")

requirements.txt

@@ -0,0 +1,15 @@
+torch==2.2.2
+transformers>=4.45.2
+tokenizers>=0.20.1
+accelerate>=0.34.2
+safetensors>=0.4.5
+huggingface_hub>=0.24.6
+einops>=0.8.0
+sentencepiece>=0.1.99
+protobuf>=4.25.3
+
+fastapi>=0.112
+uvicorn[standard]>=0.30
+peft>=0.11.1
+unsloth
+bitsandbytes==0.43.3

simplerequest.txt

@@ -0,0 +1,13 @@
+$body = @{
+  messages = @(
+    @{ role = "system"; content = "" },
+    @{ role = "user"; content = "간단히 자기소개해줘" }
+  )
+  max_new_tokens = 128
+  temperature = 0.7
+} | ConvertTo-Json -Depth 3
+
+Invoke-RestMethod -Uri https://wildojisan-qwen2-5-1-5b-instruct-basic-test.hf.space/v1/chat `
+  -Method POST `
+  -ContentType "application/json" `
+  -Body $body