nginx 反向代理 fastapi 和 ollama 服務

uvicorn 和 fastapi

使用 Uvicorn 要注意的大概就是 proxy_set_header 那邊的設定。其他 https 的部份，請使用 Certbot 幫忙搞定。

server {
    server_name domain.shinder.cc; # managed by Certbot

    add_header Accept-Ranges bytes;
    location / {
        proxy_pass http://127.0.0.1:8000;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_buffering off;
        proxy_cache off;
    }

    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    ssl_certificate /etc/letsencrypt/live/domain.shinder.cc/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/domain.shinder.cc/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}

server {
    if ($host = domain.shinder.cc) {
        return 301 https://$host$request_uri;
    } # managed by Certbot

    listen 80 ;
    listen [::]:80 ;
    server_name domain.shinder.cc;
    return 404; # managed by Certbot
}

ollama

一般為了安全性 ollama 服務是不對外的。在此的情境是，想讓遠端使用資源，但限定 IP。對外開放的通訊埠為 11435。

server {
    server_name _;

    add_header Accept-Ranges bytes;
    location / {
        proxy_pass http://127.0.0.1:11434;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_buffering off;
        proxy_cache off;

        allow   192.168.1.100;  # 允許 192.168.1.100 訪問
        deny    all;             # 拒絕所有其他 IP 訪問
    }

    listen [::]:11435;
    listen 11435;
}

使用前端 JS 接收 Ollama 回應的 Ndjson 格式

基本上就是使用 fetch API 來處理串流資料，利用 async/await 就可以使用無窮迴圈來處理。

<input type="text" id="chat" value="台灣位於哪裡？" />
<button onclick="startAsk()">開始</button>
<div id="result_div"></div>

const chat = document.querySelector("#chat");
const result_div = document.querySelector("#result_div");
const model = "mistral-nemo";  // ollama 使用的 LLM 模型名稱

const startAsk = async () => {
  const response = await fetch(`http://localhost:11434/api/generate`, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
    },
    body: JSON.stringify({ model, prompt: chat.value }),
  });
  if (!response.ok) {
    console.log("回應發生錯誤!!!");
    return;
  }
  // ['content-type', 'application/x-ndjson']
  console.log(...response.headers);

  const reader = response.body.getReader();  // 讀取器
  const decoder = new TextDecoder("utf8");   // 解碼器

  while (true) {
    const { value, done } = await reader.read();
    const json = decoder.decode(value); // 基本上每一行都是 json
    console.log(json);
    try {
      const obj = JSON.parse(json);
      result_div.innerText += obj.response;
    } catch (ex) {
      console.log(ex);
      console.log("不是 JSON:", `---${json}---`);
    }
    if (done) {
      result_div.innerText += "\n\n";
      break;
    }
  }
};