tripathiarpan20 commited on
Commit
9a69b64
·
verified ·
1 Parent(s): 3d74b48

vidaio: upload miner files

Browse files
Files changed (2) hide show
  1. chute_config.yml +3 -2
  2. miner.py +19 -3
chute_config.yml CHANGED
@@ -1,5 +1,5 @@
1
  Image:
2
- from_base: parachutes/python:3.12
3
  set_user: root
4
  run_command:
5
  - apt-get update && apt-get install -y build-essential ninja-build nasm clang wget unzip nvidia-cuda-dev nvidia-cuda-toolkit pkg-config libx264-dev libx265-dev libvpx-dev git python3-pip python3-venv
@@ -10,7 +10,8 @@ Image:
10
  run_command:
11
  - wget https://github.com/FFmpeg/nv-codec-headers/archive/876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip && unzip 876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip && cd nv-codec-headers-876af32a202d0de83bd1d36fe74ee0f7fcf86b0d && make && make install && cd / && rm -rf nv-codec-headers-* 876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip
12
  - git clone --depth 1 --branch 1.4.2 https://code.videolan.org/videolan/dav1d.git && cd dav1d && mkdir build && cd build && meson setup .. --default-library=static --buildtype release && ninja && ninja install && ldconfig && cd / && rm -rf dav1d
13
- - export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH" && git clone https://github.com/FFmpeg/FFmpeg ffmpeg && cd ffmpeg && git reset --hard 33b215d1554a14e87416a24f8e6034312e629af7 && ./configure --enable-gpl --enable-libx264 --enable-libx265 --enable-libvpx --enable-libdav1d --enable-nonfree --enable-cuda-nvcc --enable-libnpp --extra-cflags="-I/usr/local/cuda/include" --extra-ldflags="-L/usr/local/cuda/lib64" --disable-static --enable-shared --nvccflags="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_75,code=compute_75 -O2" && make -j && make install && ldconfig && cd / && rm -rf ffmpeg
 
14
  set_user: chutes
15
  run_command:
16
  - uv pip install --upgrade setuptools wheel
 
1
  Image:
2
+ from_base: parachutes/python:3.12-cu13 #needs to align with `libnvidia-encode` install to initialise NVENC sessions for av1_nvenc and hevc_nvenc compression
3
  set_user: root
4
  run_command:
5
  - apt-get update && apt-get install -y build-essential ninja-build nasm clang wget unzip nvidia-cuda-dev nvidia-cuda-toolkit pkg-config libx264-dev libx265-dev libvpx-dev git python3-pip python3-venv
 
10
  run_command:
11
  - wget https://github.com/FFmpeg/nv-codec-headers/archive/876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip && unzip 876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip && cd nv-codec-headers-876af32a202d0de83bd1d36fe74ee0f7fcf86b0d && make && make install && cd / && rm -rf nv-codec-headers-* 876af32a202d0de83bd1d36fe74ee0f7fcf86b0d.zip
12
  - git clone --depth 1 --branch 1.4.2 https://code.videolan.org/videolan/dav1d.git && cd dav1d && mkdir build && cd build && meson setup .. --default-library=static --buildtype release && ninja && ninja install && ldconfig && cd / && rm -rf dav1d
13
+ # Build ffmpeg (pinned commit 33b215d) with CUDA + x264 + dav1d, no VMAF, no libnpp
14
+ - export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH" && git init ffmpeg && cd ffmpeg && git remote add origin https://github.com/FFmpeg/FFmpeg.git && git fetch --depth 1 origin 33b215d1554a14e87416a24f8e6034312e629af7 && git checkout FETCH_HEAD && ./configure --enable-gpl --enable-libx264 --enable-libx265 --enable-libvpx --enable-libdav1d --enable-nonfree --enable-cuda-nvcc --extra-cflags="-I/usr/local/cuda/include" --extra-ldflags="-L/usr/local/cuda/lib64" --disable-static --enable-shared --nvccflags="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_75,code=compute_75 -O2" && make -j$(nproc) && make install && ldconfig && cd / && rm -rf ffmpeg
15
  set_user: chutes
16
  run_command:
17
  - uv pip install --upgrade setuptools wheel
miner.py CHANGED
@@ -30,6 +30,10 @@ class Miner:
30
  def __repr__(self) -> str:
31
  return "CompressionMiner(ffmpeg)"
32
 
 
 
 
 
33
  def _vmaf_to_crf(self, vmaf: float, encoder: str) -> int:
34
  if "x264" in encoder or "x265" in encoder:
35
  if vmaf >= 93:
@@ -38,6 +42,14 @@ class Miner:
38
  crf = 23
39
  else:
40
  crf = 28
 
 
 
 
 
 
 
 
41
  else:
42
  if vmaf >= 93:
43
  crf = 25
@@ -45,7 +57,7 @@ class Miner:
45
  crf = 30
46
  else:
47
  crf = 35
48
- print(f"[miner] VMAF threshold {vmaf} with encoder {encoder} -> CRF {crf}")
49
  return crf
50
 
51
  @staticmethod
@@ -89,8 +101,12 @@ class Miner:
89
 
90
  if codec_mode == "CRF":
91
  crf = self._vmaf_to_crf(vmaf_threshold, encoder)
92
- cmd += ["-c:v", encoder, "-crf", str(crf)]
93
- print(f"[miner] Using CRF mode: crf={crf}")
 
 
 
 
94
  elif codec_mode == "CBR":
95
  cmd += ["-c:v", encoder, "-b:v", f"{target_bitrate}M"]
96
  print(f"[miner] Using CBR mode: bitrate={target_bitrate}M")
 
30
  def __repr__(self) -> str:
31
  return "CompressionMiner(ffmpeg)"
32
 
33
+ @staticmethod
34
+ def _is_nvenc(encoder: str) -> bool:
35
+ return "nvenc" in encoder
36
+
37
  def _vmaf_to_crf(self, vmaf: float, encoder: str) -> int:
38
  if "x264" in encoder or "x265" in encoder:
39
  if vmaf >= 93:
 
42
  crf = 23
43
  else:
44
  crf = 28
45
+ elif self._is_nvenc(encoder):
46
+ # NVENC uses -cq values (0-51 scale, lower = better quality)
47
+ if vmaf >= 93:
48
+ crf = 19
49
+ elif vmaf >= 89:
50
+ crf = 24
51
+ else:
52
+ crf = 30
53
  else:
54
  if vmaf >= 93:
55
  crf = 25
 
57
  crf = 30
58
  else:
59
  crf = 35
60
+ print(f"[miner] VMAF threshold {vmaf} with encoder {encoder} -> CRF/CQ {crf}")
61
  return crf
62
 
63
  @staticmethod
 
101
 
102
  if codec_mode == "CRF":
103
  crf = self._vmaf_to_crf(vmaf_threshold, encoder)
104
+ if self._is_nvenc(encoder):
105
+ cmd += ["-c:v", encoder, "-rc", "vbr", "-cq", str(crf)]
106
+ print(f"[miner] Using NVENC CQ mode: cq={crf}")
107
+ else:
108
+ cmd += ["-c:v", encoder, "-crf", str(crf)]
109
+ print(f"[miner] Using CRF mode: crf={crf}")
110
  elif codec_mode == "CBR":
111
  cmd += ["-c:v", encoder, "-b:v", f"{target_bitrate}M"]
112
  print(f"[miner] Using CBR mode: bitrate={target_bitrate}M")