perf: add OpenMP thread pinning, passive wait policy, and core utilization tuning
Browse files- Dockerfile +5 -0
- app/llm/manager.py +13 -0
Dockerfile
CHANGED
|
@@ -20,6 +20,11 @@ RUN mkdir -p /data/users
|
|
| 20 |
|
| 21 |
ENV PYTHONUNBUFFERED=1
|
| 22 |
ENV HF_API_TOKEN=""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
EXPOSE 7860
|
| 25 |
|
|
|
|
| 20 |
|
| 21 |
ENV PYTHONUNBUFFERED=1
|
| 22 |
ENV HF_API_TOKEN=""
|
| 23 |
+
ENV OMP_NUM_THREADS=2
|
| 24 |
+
ENV OMP_PROC_BIND=CLOSE
|
| 25 |
+
ENV OMP_WAIT_POLICY=PASSIVE
|
| 26 |
+
ENV KMP_BLOCKTIME=0
|
| 27 |
+
ENV MKL_NUM_THREADS=2
|
| 28 |
|
| 29 |
EXPOSE 7860
|
| 30 |
|
app/llm/manager.py
CHANGED
|
@@ -1,5 +1,16 @@
|
|
| 1 |
import os
|
| 2 |
import psutil
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import torch
|
| 4 |
import logging
|
| 5 |
|
|
@@ -78,6 +89,8 @@ def calculate_hardware_tuning():
|
|
| 78 |
"device_map": "auto" if has_cuda else "cpu",
|
| 79 |
"max_model_params_b": 3 if ram_gb <= 16 else 7,
|
| 80 |
"inference_engine": "llama.cpp (GGUF Q2/Q4)" if HAS_LLAMA_CPP else "transformers",
|
|
|
|
|
|
|
| 81 |
}
|
| 82 |
|
| 83 |
if not has_cuda:
|
|
|
|
| 1 |
import os
|
| 2 |
import psutil
|
| 3 |
+
|
| 4 |
+
# Configure OpenMP multi-thread execution & core affinity before importing C++ backends
|
| 5 |
+
physical_cores = psutil.cpu_count(logical=False) or psutil.cpu_count() or 2
|
| 6 |
+
optimal_threads = str(max(1, min(4, physical_cores)))
|
| 7 |
+
|
| 8 |
+
os.environ.setdefault("OMP_NUM_THREADS", optimal_threads)
|
| 9 |
+
os.environ.setdefault("OMP_PROC_BIND", "CLOSE") # Pin threads close to shared L3 cache
|
| 10 |
+
os.environ.setdefault("OMP_WAIT_POLICY", "PASSIVE") # Yield idle threads immediately to save CPU
|
| 11 |
+
os.environ.setdefault("KMP_BLOCKTIME", "0") # Release Intel OpenMP threads without spinning
|
| 12 |
+
os.environ.setdefault("MKL_NUM_THREADS", optimal_threads)
|
| 13 |
+
|
| 14 |
import torch
|
| 15 |
import logging
|
| 16 |
|
|
|
|
| 89 |
"device_map": "auto" if has_cuda else "cpu",
|
| 90 |
"max_model_params_b": 3 if ram_gb <= 16 else 7,
|
| 91 |
"inference_engine": "llama.cpp (GGUF Q2/Q4)" if HAS_LLAMA_CPP else "transformers",
|
| 92 |
+
"openmp_threads": int(os.environ.get("OMP_NUM_THREADS", optimal_threads)),
|
| 93 |
+
"openmp_binding": os.environ.get("OMP_PROC_BIND", "CLOSE"),
|
| 94 |
}
|
| 95 |
|
| 96 |
if not has_cuda:
|