hari7775 commited on
Commit
534bb06
·
1 Parent(s): 65ebd96

perf: add OpenMP thread pinning, passive wait policy, and core utilization tuning

Browse files
Files changed (2) hide show
  1. Dockerfile +5 -0
  2. app/llm/manager.py +13 -0
Dockerfile CHANGED
@@ -20,6 +20,11 @@ RUN mkdir -p /data/users
20
 
21
  ENV PYTHONUNBUFFERED=1
22
  ENV HF_API_TOKEN=""
 
 
 
 
 
23
 
24
  EXPOSE 7860
25
 
 
20
 
21
  ENV PYTHONUNBUFFERED=1
22
  ENV HF_API_TOKEN=""
23
+ ENV OMP_NUM_THREADS=2
24
+ ENV OMP_PROC_BIND=CLOSE
25
+ ENV OMP_WAIT_POLICY=PASSIVE
26
+ ENV KMP_BLOCKTIME=0
27
+ ENV MKL_NUM_THREADS=2
28
 
29
  EXPOSE 7860
30
 
app/llm/manager.py CHANGED
@@ -1,5 +1,16 @@
1
  import os
2
  import psutil
 
 
 
 
 
 
 
 
 
 
 
3
  import torch
4
  import logging
5
 
@@ -78,6 +89,8 @@ def calculate_hardware_tuning():
78
  "device_map": "auto" if has_cuda else "cpu",
79
  "max_model_params_b": 3 if ram_gb <= 16 else 7,
80
  "inference_engine": "llama.cpp (GGUF Q2/Q4)" if HAS_LLAMA_CPP else "transformers",
 
 
81
  }
82
 
83
  if not has_cuda:
 
1
  import os
2
  import psutil
3
+
4
+ # Configure OpenMP multi-thread execution & core affinity before importing C++ backends
5
+ physical_cores = psutil.cpu_count(logical=False) or psutil.cpu_count() or 2
6
+ optimal_threads = str(max(1, min(4, physical_cores)))
7
+
8
+ os.environ.setdefault("OMP_NUM_THREADS", optimal_threads)
9
+ os.environ.setdefault("OMP_PROC_BIND", "CLOSE") # Pin threads close to shared L3 cache
10
+ os.environ.setdefault("OMP_WAIT_POLICY", "PASSIVE") # Yield idle threads immediately to save CPU
11
+ os.environ.setdefault("KMP_BLOCKTIME", "0") # Release Intel OpenMP threads without spinning
12
+ os.environ.setdefault("MKL_NUM_THREADS", optimal_threads)
13
+
14
  import torch
15
  import logging
16
 
 
89
  "device_map": "auto" if has_cuda else "cpu",
90
  "max_model_params_b": 3 if ram_gb <= 16 else 7,
91
  "inference_engine": "llama.cpp (GGUF Q2/Q4)" if HAS_LLAMA_CPP else "transformers",
92
+ "openmp_threads": int(os.environ.get("OMP_NUM_THREADS", optimal_threads)),
93
+ "openmp_binding": os.environ.get("OMP_PROC_BIND", "CLOSE"),
94
  }
95
 
96
  if not has_cuda: