Python使用FastAPI实现大文件分片上传与断点续传功能

本文主要是介绍Python使用FastAPI实现大文件分片上传与断点续传功能，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

《Python使用FastAPI实现大文件分片上传与断点续传功能》大文件直传常遇到超时、网络抖动失败、失败后只能重传的问题,分片上传+断点续传可以把大文件拆成若干小块逐个上传,并在中断后从已完成分片继...

一、接口设计

接口	方法	描述
/upload/init	POST	初始化上传，创建 upload_id 与分片元数据
/upload/chunk	POST	上传单个分片，参数：upload_id、index
/upload/status	GET	查询某 upload_id 已上传分片列表
/upload/merge	POST	全量合并分片，生成最终文件

说明：index 从 1 开始；建议固定 chunk_size（如 5MB/10MB），便于计算总分片数 total_chunks = ceil(size/chunk_size)。

二、服务端实现（FastAPI）

2.1 运行环境

python -m venv venv
source venv/bin/activate  # Windows 使用 venv\Scripts\activate
pip install fastapi uvicorn pydantic[dotenv] python-multipart

注：本文仅用到标准库 + FastAPI 依赖；如需对象存储支持，再安装 minio 或 boto3。

2.2 目录结构建议

.
├─ server.py              # FastAPI 服务端
├─ upload.db              # SQLite 元数据库（运行后生成）
└─ uploads/
   └─ {upload_id}/
      ├─ chunks/         # 分片临时目录
      ├─ meta.json       # 元信息（冗余存储，辅助排查）
      └─ final/          # 最终文件目录

2.3 server.py（可直接运行）

# server.py
import hashlib
import json
import os
import sqlite3
import threading
from contextlib import contextmanager
from math import ceil
from pathlib import Path
from typing import List, Optional

from fastapi import FastAPI, UploadFile, File, HTTPException, Query, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
import uvicorn

BASE_DIR = Path(__file__).parent.resolve()
DATA_DIR = BASE_DIR / "uploads"
DB_PATH = BASE_DIR / "upload.db"
DATA_DIR.mkdir(exist_ok=True)

app = FastAPI(title="Chunked Upload (FastAPI)")

# ---------- 并发锁（同一 upload_id 合并上锁） ----------
_merge_locks = {}
_merge_locks_global = threading.Lock()

def _get_lock(upload_id: str) -> threading.Lock:
    with _merge_locks_global:
        if upload_id not in _merge_locks:
            _merge_locks[upload_id] = threading.Lock()
        return _merge_locks[upload_id]

# ---------- SQLite 简单封装 ----------
def init_db():
    with sqlite3.connect(DB_PATH) as conn:
        c = conn.cursor()
        c.execute("""CREATE TABLE IF NOT EXISTS uploads(
            upload_id TEXT PRIMARY KEY,
            filename TEXT,
            size INTEGER,
            chunk_size INTEGER,
            total_chunks INTEGER,
            file_hash TEXT
        )""")
        c.execute("""CREATE TABLE IF NOT EXISTS chunks(
            upload_id TEXT,
            idx INTEGER,
            size INTEGER,
            hash TEXT,
            PRIMARY KEY(upload_id, idx)
        )""")
        conn.commit()

@contextmanager
def db() :
    conn = sqlite3.connect(DB_PATH)
    try:
        yield conn
    finally:
        conn.close()

# ---------- 工具 ----------
def sha256_of_bytes(b: bytes) -> str:
    h = hashlib.sha256()
    h.update(b)
    return h.hexdigest()

def md5_of_bytes(b: bytes) -> str:
    h = hashlib.md5()
    h.update(b)
    return h.hexdigest()

def ensure_upload_folders(upload_id: str) -> Path:
    root = DATA_DIR / upload_id
    (root / "chunks").mkdir(parents=True, exist_ok=True)
    (root / "final").mkdir(parents=True, exist_ok=True)
    return root

def write_meta(root: Path, meta: dict):
    (root / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))

# ---------- 请求/响应模型 ----------
class InitReq(BaseModel):
    filename: str
    size: int
    chunk_size: int
    file_hash: Optional[str] = None   # 可选：客户端传文件整体 SHA256/MD5

class InitResp(BaseModel):
    upload_id: str
    total_chunks: int

class StatusResp(BaseModel):
    upload_id: str
    uploaded: List[int]
    total_chunks: int

class MergeReq(BaseModel):
    upload_id: str
    expected_hash: Optional[str] = None  # 可选：合并后对整文件校验

# ---------- 路由 ----------
@app.post("/upload/init", response_model=InitResp)
def init_upload(req: InitReq):
    if req.size <= 0 or req.chunk_size <= 0:
        raise HTTPException(400, "size/chunk_size 必须为正整数")
    total = ceil(req.size / rwww.chinasem.cneq.chunk_size)

    upload_id = hashlib.sha1(f"{req.filename}:{req.size}:{rexmFwiantaq.chunk_size}:{req.file_hash or ''}".encode()).hexdigest()
    root = ensure_upload_folders(upload_id)

    with db() as conn:
        c = conn.cursor()
        # UPSERT
        c.execute("INSERT OR IGNORE INTO uploads(upload_id, filename, size, chunk_size, total_chunks, file_hash) VALUES(?,?,?,?,?,?)",
                  (upload_id, req.filename, req.size, req.chunk_size, total, req.file_hash))
        conn.commit()

    write_meta(root, {
        "upload_id": upload_id,
        "filename": req.filename,
        "size": req.size,
        "chunk_size": req.chunk_size,
        "total_chunks": total,
        "file_hash": req.file_hash
    })
    return InitResp(upload_id=upload_id, total_chunks=total)

@app.post("/upload/chunk")
async def upload_chunk(
    upload_id: str = Query(..., description="init 返回的 upload_id"),
    index: int = Query(..., ge=1, description="分片序号，从1开始"),
    content_hash: Optional[str] = Query(None, description="可选：分片内容校验（sha256或md5，客户端自定义）"),
    file: UploadFile = File(...)
):
    # 检查 upload 是否存在
    with db() as conn:
        c = conn.cursor()
       js row = c.execute("SELECT total_chunks FROM uploads WHERE upload_id=?", (upload_id,)).fetchone()
        if not row:
            raise HTTPException(404, "upload_id 不存在，请先 /upload/init")
        total_chunks = row[0]

    if index > total_chunks:
        raise HTTPException(400, f"index 超出总分片数：{total_chunks}")

    root = ensure_upload_folders(upload_id)
    chunk_path = root / "chunks" / f"{index}.part"

    # 将分片写入临时文件
    data = await file.read()
    if content_hash:
        # 这里兼容 md5/sha256：长度32大概率md5，长度64大概率sha256，也可自定义一个 query param 指定算法
        calc = md5_of_bytes(data) if len(content_hash) == 32 else sha256_of_bytes(data)
        if calc != content_hash:
            raise HTTPException(400, "content_hash 校验失败")

    with open(chunk_path, "wb") as f:
        f.write(data)

    # 记录 DB
    with db() as conn:
        c = conn.cursor()
        c.execute("INSERT OR REPLACE INTO chunks(upload_id, idx, size, hash) VALUES(?,?,?,?)",
                  (upload_id, index, len(data), content_hash))
        conn.commit()

    return JSONResponse({"message": "ok", "index": index, "size": len(data)})

@app.get("/upload/status", response_model=StatusResp)
def upload_status(upload_xmFwiantaid: str):
    with db() as conn:
        c = conn.cursor()
        up = c.execute("SELECT total_chunks FROM uploads WHERE upload_id=?", (upload_id,)).fetchone()
        if not up:
            raise HTTPException(404, "upload_id 不存在")
        total = up[0]
        rows = c.execute("SELECT idx FROM chunks WHERE upload_id=? ORDER BY idx", (upload_id,)).fetchall()
        uploaded = [r[0] for r in rows]
    return StatusResp(upload_id=upload_id, uploaded=uploaded, total_chunks=total)

@app.post("/upload/merge")
def upload_merge(req: MergeReq):
    # 防止并发重复合并
    lock = _get_lock(req.upload_id)
    with lock:
        with db() as conn:
            c = conn.cursor()
            up = c.execute("SELECT filename, total_chunks FROM uploads WHERE upload_androidid=?", (req.upload_id,)).fetchone()
            if not up:
                raise HTTPException(404, "upload_id 不存在")
            filename, total = up

            # 校验分片是否齐全
            rows = c.execute("SELECT idx FROM chunks WHERE upload_id=? ORDER BY idx", (req.upload_id,)).fetchall()
            uploaded = [r[0] for r in rows]
            if len(uploaded) != total or uploaded != list(range(1, total + 1)):
                raise HTTPException(400, f"分片不完整，已上传：{uploaded}，应为 1..{total}")

        root = ensure_upload_folders(req.upload_id)
        final_path = root / "final" / filename
        # 合并
        with open(final_path, "wb") as dst:
            for i in range(1, total + 1):
                chunk_path = root / "chunks" / f"{i}.part"
                with open(chunk_path, "rb") as src:
                    dst.write(src.read())

        # 可选：合并后哈希校验
        if req.expected_hash:
            with open(final_path, "rb") as f:
                data = f.read()
            calc = md5_of_bytes(data) if len(req.expected_hash) == 32 else sha256_of_bytes(data)
            if calc != req.expected_hash:
                raise HTTPException(400, "合并后文件校验失败")

        # 也可以在此处清理分片文件，或异步清理
        return JSONResponse({"message": "merged", "final_path": str(final_path)})
        
# ---------- 启动 ----------
if __name__ == "__main__":
    init_db()
    uvicorn.run(app, host="0.0.0.0", port=8000)

启动：python server.py，访问 http://localhost:8000/docs 可用 Swagger 调试。

三、Python 客户端脚本（支持断点续传与失败重试）

3.1 客户端依赖

pip install requests tqdm

3.2 client.py（可直接运行）

# client.py
import math
import os
import time
import hashlib
import requests
from pathlib import Path
from tqdm import tqdm

SERVER = "http://localhost:8000"

def sha256_of_file(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b""):
            h.update(chunk)
    return h.hexdigest()

def md5_of_bytes(b: bytes) -> str:
    import hashlib
    h = hashlib.md5()
    h.update(b)
    return h.hexdigest()

def upload_file(path: str, chunk_size: int = 5 * 1024 * 1024, use_md5_for_chunk=False, retries=3, sleep=1):
    p = Path(path)
    size = p.stat().st_size
    file_hash = sha256_of_file(p)  # 也可改为 MD5
    total_chunks = math.ceil(size / chunk_size)

    # 1) init
    r = requests.post(f"{SERVER}/upload/init", json={
        "filename": p.name,
        "size": size,
        "chunk_size": chunk_size,
        "file_hash": file_hash
    })
    r.raise_for_status()
    j = r.json()
    upload_id = j["upload_id"]
    total_chunks = j["total_chunks"]
    print("upload_id:", upload_id, "total_chunks:", total_chunks)

    # 2) 查询已上传
    def get_status():
        rr = requests.get(f"{SERVER}/upload/status", params={"upload_id": upload_id})
        rr.raise_for_status()
        return rr.json()["uploaded"]

    uploaded = set(get_status())

    # 3) 逐片上传（可按需并发）
    with open(p, "rb") as f:
        for index in tqdm(range(1, total_chunks + 1), desc="Uploading"):
            if index in uploaded:
                continue
            start = (index - 1) * chunk_size
            end = min(size, index * chunk_size)
            f.seek(start)
            data = f.read(end - start)

            content_hash = md5_of_bytes(data) if use_md5_for_chunk else None
            for attempt in range(retries):
                try:
                    files = {"file": (f"{index}.part", data, "application/octet-stream")}
                    params = {"upload_id": upload_id, "index": index}
                    if content_hash:
                        params["content_hash"] = content_hash
                    rr = requests.post(f"{SERVER}/upload/chunk", files=files, params=params, timeout=30)
                    rr.raise_for_status()
                    break
                except Exception as e:
                    print(f"[{index}] retry {attempt+1}/{retries} because {e}")
                    time.sleep(sleep)
            else:
                raise RuntimeError(f"upload chunk {index} failed after retries")

    # 4) 合并
    r = requests.post(f"{SERVER}/upload/merge", json={
        "upload_id": upload_id,
        "expected_hash": file_hash  # 合并后校验整文件
    })
    r.raise_for_status()
    print("merge result:", r.json())

if __name__ == "__main__":
    # 示例：python client.py
    upload_file("bigfile.bin", chunk_size=5 * 1024 * 1024, use_md5_for_chunk=True)

说明：

默认对整文件计算 SHA256，分片使用 MD5（use_md5_for_chunk=True）进行轻量校验。
可按需改为并发上传（例如 concurrent.futures.ThreadPoolExecutor），但要关注带宽/并发上限与服务端限流策略。

四、断点续传原理与实现要点

唯一标识：upload_id 由 (filename, size, chunk_size, file_hash) 派生，确保同一文件同配置产生稳定 ID。
状态存储：用 SQLite 记录 uploads 与 chunks，服务重启后仍可恢复。
状态查询：/upload/status 返回已上传分片列表，客户端跳过已完成分片，直接续传剩余分片。
合并锁：为避免并发重复合并，对 upload_id 加进程内锁。
数据校验：
分片级：content_hash（MD5/SHA256）保障每块完整。
整体级：expected_hash 合并后校验整文件。
失败重试：客户端对分片上传进行有限次重试，并可按需指数回退。
清理策略：合并成功后可异步清理分片文件，释放磁盘。

五、前端思路（可选）

Web 前端（浏览器）常用 Blob.slice 分片 + fetch/XMLHttpRequest 上传。示例片段：

async function upload(file) {
  const chunkSize = 5 * 1024 * 1024;
  const totalChunks = Math.ceil(file.size / chunkSize);

  // 1) init
  const initResp = await fetch('/upload/init', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json'},
    body: JSON.stringify({ filename: file.name, size: file.size, chunk_size: chunkSize })
  }).then(r => r.json());
  const { upload_id } = initResp;

  // 2) 查询已上传
  const status = await fetch(`/upload/status?upload_id=${upload_id}`).then(r => r.json());
  const uploaded = new Set(status.uploaded);

  // 3) 顺序或并发上传
  for (let i = 1; i <= totalChunks; i++) {
    if (uploaded.has(i)) continue;
    const start = (i - 1) * chunkSize;
    const end = Math.min(file.size, i * chunkSize);
    const blob = file.slice(start, end);
    const form = new FormData();
    form.append('file', blob, `${i}.part`);

    await fetch(`/upload/chunk?upload_id=${upload_id}&index=${i}`, { method: 'POST', body: form });
  }

  // 4) merge
  await fetch('/upload/merge', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json'},
    body: JSON.stringify({ upload_id })
  });
}

生产环境可加入：并发上传、进度条展示、速度/剩余时间估算、取消/暂停/继续、切片校验、错误处理等。

六、接入对象存储（MinIO/S3）思路

直传对象存储：客户端直传对象存储（预签名 URL），服务端仅做 init/status/merge 与校验，减少后端带宽占用。
后端合并：分片先上传至桶的 /{upload_id}/chunks/{index}.part，合并阶段服务端从对象存储流式读出到最终对象。
S3 多段上传（Multipart Upload）：直接使用 S3 的 Multipart API，可绕过本地分片落盘，提升性能与可靠性。

示例（仅演示方向，未接入到上面代码）：

# pip install minio
from minio import Minio

client = Minio("127.0.0.1:9000", Access_key="minioadmin", secret_key="minioadmin", secure=False)

# 1) 为每个分片生成预签名 PUT URL，前端直接 PUT 到 MinIO
url = client.presigned_put_object("my-bucket", f"{upload_id}/chunks/{index}.part", expires=timedelta(hours=1))

# 2) 合并阶段：服务端从 MinIO 流式读取所有分片，写入最终对象