Skip to content

Commit 11b18e5

Browse files
ZhangYulonggZhangYulongg
authored
add cache queue port (#3904) (#3926)
* add cache queue port * add cache queue port * add cache queue port
1 parent e2c764f commit 11b18e5

22 files changed

Lines changed: 74 additions & 15 deletions

‎.github/workflows/_accuracy_test.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,14 @@ jobs:
8080
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
8181
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
8282
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
83+
FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
8384
echo "Test ENV Parameter:"
8485
echo "========================================================="
8586
echo "FLASK_PORT=${FLASK_PORT}"
8687
echo "FD_API_PORT=${FD_API_PORT}"
8788
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
8889
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
90+
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
8991
echo "DEVICES=${DEVICES}"
9092
echo "========================================================="
9193
@@ -99,7 +101,7 @@ jobs:
99101
exit 1
100102
fi
101103
102-
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
104+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
103105
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
104106
echo "==== LOG_FILE is ${LOG_FILE} ===="
105107
@@ -133,6 +135,7 @@ jobs:
133135
-e "FD_API_PORT=${FD_API_PORT}" \
134136
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
135137
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
138+
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
136139
-e "FLASK_PORT=${FLASK_PORT}" \
137140
-v "${MODEL_CACHE_DIR}:/MODELDATA" \
138141
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \

‎.github/workflows/_base_test.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,14 @@ jobs:
8080
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
8181
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
8282
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
83+
FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
8384
echo "Test ENV Parameter:"
8485
echo "========================================================="
8586
echo "FLASK_PORT=${FLASK_PORT}"
8687
echo "FD_API_PORT=${FD_API_PORT}"
8788
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
8889
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
90+
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
8991
echo "DEVICES=${DEVICES}"
9092
echo "========================================================="
9193
@@ -99,7 +101,7 @@ jobs:
99101
exit 1
100102
fi
101103
102-
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
104+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
103105
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
104106
echo "==== LOG_FILE is ${LOG_FILE} ===="
105107
@@ -134,6 +136,7 @@ jobs:
134136
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
135137
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
136138
-e "FLASK_PORT=${FLASK_PORT}" \
139+
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
137140
-v "${MODEL_CACHE_DIR}:/MODELDATA" \
138141
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
139142
-v "${CACHE_DIR}/.cache:/root/.cache" \

‎.github/workflows/_logprob_test_linux.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,14 @@ jobs:
7070
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
7171
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
7272
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
73+
FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
7374
echo "Test ENV Parameter:"
7475
echo "========================================================="
7576
echo "FLASK_PORT=${FLASK_PORT}"
7677
echo "FD_API_PORT=${FD_API_PORT}"
7778
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
7879
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
80+
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
7981
echo "DEVICES=${DEVICES}"
8082
echo "========================================================="
8183
@@ -89,7 +91,7 @@ jobs:
8991
exit 1
9092
fi
9193
92-
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
94+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
9395
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
9496
echo "==== LOG_FILE is ${LOG_FILE} ===="
9597
@@ -123,6 +125,7 @@ jobs:
123125
-e "FD_API_PORT=${FD_API_PORT}" \
124126
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
125127
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
128+
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
126129
-e "FLASK_PORT=${FLASK_PORT}" \
127130
-v "${MODEL_CACHE_DIR}:/MODELDATA" \
128131
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \

‎.github/workflows/_pre_ce_test.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,14 @@ jobs:
8181
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
8282
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
8383
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
84+
FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
8485
echo "Test ENV Parameter:"
8586
echo "========================================================="
8687
echo "FLASK_PORT=${FLASK_PORT}"
8788
echo "FD_API_PORT=${FD_API_PORT}"
8889
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
8990
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
91+
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
9092
echo "DEVICES=${DEVICES}"
9193
echo "========================================================="
9294
@@ -96,7 +98,7 @@ jobs:
9698
touch "${CACHE_DIR}/gitconfig"
9799
fi
98100
99-
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
101+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
100102
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
101103
echo "==== LOG_FILE is ${LOG_FILE} ===="
102104
@@ -134,6 +136,7 @@ jobs:
134136
-e "FD_API_PORT=${FD_API_PORT}" \
135137
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
136138
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
139+
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
137140
-e "FLASK_PORT=${FLASK_PORT}" \
138141
-e "fd_wheel_url=${fd_wheel_url}" \
139142
--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '

‎.github/workflows/_unit_test_coverage.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,14 @@ jobs:
102102
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
103103
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
104104
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
105+
FD_CACHE_QUEUE_PORT=$((42098 + DEVICE_PORT * 100))
105106
echo "Test ENV Parameter:"
106107
echo "========================================================="
107108
echo "FLASK_PORT=${FLASK_PORT}"
108109
echo "FD_API_PORT=${FD_API_PORT}"
109110
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
110111
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
112+
echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
111113
echo "DEVICES=${DEVICES}"
112114
echo "========================================================="
113115
@@ -117,7 +119,7 @@ jobs:
117119
touch "${CACHE_DIR}/gitconfig"
118120
fi
119121
120-
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
122+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
121123
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
122124
echo "==== LOG_FILE is ${LOG_FILE} ===="
123125
@@ -156,6 +158,7 @@ jobs:
156158
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
157159
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
158160
-e "FLASK_PORT=${FLASK_PORT}" \
161+
-e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
159162
-e TZ="Asia/Shanghai" \
160163
-e "fd_wheel_url=${fd_wheel_url}" \
161164
-e "BASE_REF=${BASE_REF}" \

‎tests/ce/deploy/deploy.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,12 @@ def get_available_port(env_key: str, default_start: int):
5959
FD_API_PORT = get_available_port("FD_API_PORT", FLASK_PORT + 1)
6060
FD_ENGINE_QUEUE_PORT = get_available_port("FD_ENGINE_QUEUE_PORT", FD_API_PORT + 1)
6161
FD_METRICS_PORT = get_available_port("FD_METRICS_PORT", FD_ENGINE_QUEUE_PORT + 1)
62+
FD_CACHE_QUEUE_PORT = get_available_port("FD_CACHE_QUEUE_PORT", FD_METRICS_PORT + 1)
6263
DEFAULT_PARAMS = {
6364
"--port": FD_API_PORT,
6465
"--engine-worker-queue-port": FD_ENGINE_QUEUE_PORT,
6566
"--metrics-port": FD_METRICS_PORT,
67+
"--cache-queue-port": FD_CACHE_QUEUE_PORT,
6668
"--enable-logprob": True,
6769
}
6870

@@ -179,7 +181,7 @@ def stop_server(signum=None, frame=None):
179181
except Exception as e:
180182
print(f"Failed to stop server: {e}, {str(traceback.format_exc())}")
181183

182-
for port in [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT]:
184+
for port in [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT, FD_CACHE_QUEUE_PORT]:
183185
try:
184186
output = subprocess.check_output(f"lsof -i:{port} -t", shell=True).decode().strip()
185187
for pid in output.splitlines():

‎tests/ci_use/EB_Lite/test_EB_Lite_serving.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@
2828
FD_API_PORT = int(os.getenv("FD_API_PORT", 8188))
2929
FD_ENGINE_QUEUE_PORT = int(os.getenv("FD_ENGINE_QUEUE_PORT", 8133))
3030
FD_METRICS_PORT = int(os.getenv("FD_METRICS_PORT", 8233))
31+
FD_CACHE_QUEUE_PORT = int(os.getenv("FD_CACHE_QUEUE_PORT", 8234))
3132

3233
# List of ports to clean before and after tests
33-
PORTS_TO_CLEAN = [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT]
34+
PORTS_TO_CLEAN = [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT, FD_CACHE_QUEUE_PORT]
3435

3536

3637
def is_port_open(host: str, port: int, timeout=1.0):
@@ -100,6 +101,8 @@ def setup_and_run_server():
100101
str(FD_ENGINE_QUEUE_PORT),
101102
"--metrics-port",
102103
str(FD_METRICS_PORT),
104+
"--cache-queue-port",
105+
str(FD_CACHE_QUEUE_PORT),
103106
"--max-model-len",
104107
"32768",
105108
"--max-num-seqs",

‎tests/ci_use/EB_VL_Lite/test_EB_VL_Lite_serving.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,10 @@
2929
FD_API_PORT = int(os.getenv("FD_API_PORT", 8188))
3030
FD_ENGINE_QUEUE_PORT = int(os.getenv("FD_ENGINE_QUEUE_PORT", 8133))
3131
FD_METRICS_PORT = int(os.getenv("FD_METRICS_PORT", 8233))
32+
FD_CACHE_QUEUE_PORT = int(os.getenv("FD_CACHE_QUEUE_PORT", 8234))
3233

3334
# List of ports to clean before and after tests
34-
PORTS_TO_CLEAN = [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT]
35+
PORTS_TO_CLEAN = [FD_API_PORT, FD_ENGINE_QUEUE_PORT, FD_METRICS_PORT, FD_CACHE_QUEUE_PORT]
3536

3637

3738
def is_port_open(host: str, port: int, timeout=1.0):
@@ -103,6 +104,8 @@ def setup_and_run_server():
103104
str(FD_ENGINE_QUEUE_PORT),
104105
"--metrics-port",
105106
str(FD_METRICS_PORT),
107+
"--cache-queue-port",
108+
str(FD_CACHE_QUEUE_PORT),
106109
"--enable-mm",
107110
"--max-model-len",
108111
"32768",

‎tests/ci_use/ERNIE_0dot3B/test_ernie_03b_pd.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,18 @@
2727
FD_API_PORT = int(os.getenv("FD_API_PORT", 8188))
2828
FD_ENGINE_QUEUE_PORT = int(os.getenv("FD_ENGINE_QUEUE_PORT", 8133))
2929
FD_METRICS_PORT = int(os.getenv("FD_METRICS_PORT", 8233))
30+
FD_CACHE_QUEUE_PORT = int(os.getenv("FD_CACHE_QUEUE_PORT", 8333))
3031

3132
# List of ports to clean before and after tests
3233
PORTS_TO_CLEAN = [
3334
FD_API_PORT,
3435
FD_ENGINE_QUEUE_PORT,
3536
FD_METRICS_PORT,
37+
FD_CACHE_QUEUE_PORT,
3638
FD_API_PORT + 1,
3739
FD_ENGINE_QUEUE_PORT + 1,
3840
FD_METRICS_PORT + 1,
41+
FD_CACHE_QUEUE_PORT + 1,
3942
]
4043

4144

@@ -116,6 +119,8 @@ def setup_and_run_server():
116119
str(FD_ENGINE_QUEUE_PORT),
117120
"--metrics-port",
118121
str(FD_METRICS_PORT),
122+
"--cache-queue-port",
123+
str(FD_CACHE_QUEUE_PORT),
119124
"--max-model-len",
120125
"8192",
121126
"--max-num-seqs",
@@ -157,7 +162,7 @@ def setup_and_run_server():
157162
"--metrics-port",
158163
str(FD_METRICS_PORT + 1),
159164
"--cache-queue-port",
160-
str(FD_API_PORT + 2),
165+
str(FD_CACHE_QUEUE_PORT + 1),
161166
"--max-model-len",
162167
"8192",
163168
"--max-num-seqs",

‎tests/ci_use/Qwen2-7B-Instruct_offline/test_Qwen2-7B-Instruct_offline.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from fastdeploy import LLM, SamplingParams
2525

2626
FD_ENGINE_QUEUE_PORT = int(os.getenv("FD_ENGINE_QUEUE_PORT", 8313))
27+
FD_CACHE_QUEUE_PORT = int(os.getenv("FD_CACHE_QUEUE_PORT", 8333))
2728
MAX_WAIT_SECONDS = 60
2829

2930

@@ -87,6 +88,7 @@ def llm(model_path):
8788
model=model_path,
8889
tensor_parallel_size=1,
8990
engine_worker_queue_port=FD_ENGINE_QUEUE_PORT,
91+
cache_queue_port=FD_CACHE_QUEUE_PORT,
9092
max_model_len=32768,
9193
quantization="wint8",
9294
)

0 commit comments

Comments
 (0)