wip: 切片上传、文件秒传、断点续传陆续适配中当前进度(本地存储已适配)

This commit is contained in:
Lan
2025-02-23 20:49:51 +08:00
parent 17331112b4
commit fdada809d9
30 changed files with 622 additions and 178 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ jobs:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ secrets.DOCKER_USERNAME }}/filecodebox:latest
tags: ${{ secrets.DOCKER_USERNAME }}/filecodebox:beta
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
- name: Move cache
+3
View File
@@ -154,3 +154,6 @@ for_test.py
data/.env
.backup/
/cloc-1.64.exe
# Ignore node_modules
node_modules/
+28
View File
@@ -0,0 +1,28 @@
from tortoise import connections
async def create_upload_chunk_and_update_file_codes_table():
conn = connections.get("default")
await conn.execute_script(
"""
ALTER TABLE "filecodes" ADD "file_hash" VARCHAR(128);
ALTER TABLE "filecodes" ADD "is_chunked" BOOL NOT NULL DEFAULT False;
ALTER TABLE "filecodes" ADD "upload_id" VARCHAR(128);
CREATE TABLE "uploadchunk" (
id INTEGER not null primary key autoincrement,
"upload_id" VARCHAR(36) NOT NULL,
"chunk_index" INT NOT NULL,
"chunk_hash" VARCHAR(128) NOT NULL,
"total_chunks" INT NOT NULL,
"file_size" BIGINT NOT NULL,
"chunk_size" INT NOT NULL,
"created_at" TIMESTAMPTZ NOT NULL,
"file_name" VARCHAR(255) NOT NULL,
"completed" BOOL NOT NULL
);
"""
)
async def migrate():
await create_upload_chunk_and_update_file_codes_table()
+31 -93
View File
@@ -2,119 +2,55 @@
# @Author : Lan
# @File : models.py
# @Software: PyCharm
from datetime import datetime
from typing import Optional
from tortoise import fields
from tortoise.models import Model
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise import fields, models
from datetime import datetime
from core.utils import get_now
class FileCodes(Model):
id: Optional[int] = fields.IntField(pk=True)
code: Optional[int] = fields.CharField(
description="分享码", max_length=255, index=True, unique=True
)
prefix: Optional[str] = fields.CharField(
max_length=255, description="前缀", default=""
)
suffix: Optional[str] = fields.CharField(
max_length=255, description="后缀", default=""
)
uuid_file_name: Optional[str] = fields.CharField(
max_length=255, description="uuid文件名", null=True
)
file_path: Optional[str] = fields.CharField(
max_length=255, description="文件路径", null=True
)
size: Optional[int] = fields.IntField(description="文件大小", default=0)
text: Optional[str] = fields.TextField(description="文本内容", null=True)
expired_at: Optional[datetime] = fields.DatetimeField(
null=True, description="过期时间"
)
expired_count: Optional[int] = fields.IntField(description="可用次数", default=0)
used_count: Optional[int] = fields.IntField(description="已用次数", default=0)
created_at: Optional[datetime] = fields.DatetimeField(
auto_now_add=True, description="创建时间"
)
#
# file_hash = fields.CharField(
# max_length=128, # SHA-256需要64字符,这里预留扩展空间
# description="文件哈希值",
# unique=True,
# null=True # 允许旧数据为空
# )
# hash_algorithm = fields.CharField(
# max_length=20,
# description="哈希算法类型",
# null=True,
# default="sha256"
# )
# # 新增分片字段
# chunk_size = fields.IntField(
# description="分片大小(字节)",
# default=0
# )
# total_chunks = fields.IntField(
# description="总分片数",
# default=0
# )
# uploaded_chunks = fields.IntField(
# description="已上传分片数",
# default=0
# )
# upload_status = fields.CharField(
# max_length=20,
# description="上传状态",
# default="pending", # pending/in_progress/completed
# choices=["pending", "in_progress", "completed"]
# )
# is_chunked = fields.BooleanField(
# description="是否分片上传",
# default=False
# )
class FileCodes(models.Model):
id = fields.IntField(pk=True)
code = fields.CharField(max_length=255, unique=True, index=True)
prefix = fields.CharField(max_length=255, default="")
suffix = fields.CharField(max_length=255, default="")
uuid_file_name = fields.CharField(max_length=255, null=True)
file_path = fields.CharField(max_length=255, null=True)
size = fields.IntField(default=0)
text = fields.TextField(null=True)
expired_at = fields.DatetimeField(null=True)
expired_count = fields.IntField(default=0)
used_count = fields.IntField(default=0)
created_at = fields.DatetimeField(auto_now_add=True)
file_hash = fields.CharField(max_length=64, null=True)
is_chunked = fields.BooleanField(default=False)
upload_id = fields.CharField(max_length=36, null=True)
async def is_expired(self):
# 按时间
if self.expired_at is None:
return False
if self.expired_at and self.expired_count < 0:
return self.expired_at < await get_now()
# 按次数
else:
return self.expired_count <= 0
async def get_file_path(self):
return f"{self.file_path}/{self.uuid_file_name}"
#
# class FileChunks(Model):
# id = fields.IntField(pk=True)
# file_code = fields.ForeignKeyField(
# "models.FileCodes",
# related_name="chunks",
# on_delete=fields.CASCADE
# )
# chunk_number = fields.IntField(description="分片序号")
# chunk_hash = fields.CharField(
# max_length=128,
# description="分片哈希校验值"
# )
# chunk_path = fields.CharField(
# max_length=255,
# description="分片存储路径"
# )
# created_at = fields.DatetimeField(
# auto_now_add=True,
# description="上传时间"
# )
#
# class Meta:
# unique_together = [("file_code", "chunk_number")]
class UploadChunk(models.Model):
id = fields.IntField(pk=True)
upload_id = fields.CharField(max_length=36, index=True)
chunk_index = fields.IntField()
chunk_hash = fields.CharField(max_length=64)
total_chunks = fields.IntField()
file_size = fields.BigIntField()
chunk_size = fields.IntField()
file_name = fields.CharField(max_length=255)
created_at = fields.DatetimeField(auto_now_add=True)
completed = fields.BooleanField(default=False)
class KeyValue(Model):
@@ -129,3 +65,5 @@ class KeyValue(Model):
file_codes_pydantic = pydantic_model_creator(FileCodes, name="FileCodes")
upload_chunk_pydantic = pydantic_model_creator(UploadChunk, name="UploadChunk")
key_value_pydantic = pydantic_model_creator(KeyValue, name="KeyValue")
+12
View File
@@ -3,3 +3,15 @@ from pydantic import BaseModel
class SelectFileModel(BaseModel):
code: str
class InitChunkUploadModel(BaseModel):
file_name: str
chunk_size: int = 5 * 1024 * 1024
file_size: int
file_hash: str
class CompleteUploadModel(BaseModel):
expire_value: int
expire_style: str
+24
View File
@@ -1,4 +1,5 @@
import datetime
import hashlib
import os
import uuid
@@ -29,6 +30,17 @@ async def get_file_path_name(file: UploadFile) -> Tuple[str, str, str, str, str]
return path, suffix, prefix, file.filename, save_path
async def get_chunk_file_path_name(file_name: str, upload_id: str) -> Tuple[str, str, str, str, str]:
"""获取切片文件的路径和文件名"""
today = datetime.datetime.now()
storage_path = settings.storage_path.strip("/") # 移除开头和结尾的斜杠
base_path = f"share/data/{today.strftime('%Y/%m/%d')}/{upload_id}"
path = f"{storage_path}/{base_path}" if storage_path else base_path
prefix, suffix = os.path.splitext(file_name)
save_path = f"{path}/{prefix}{suffix}"
return path, suffix, prefix, file_name, save_path
async def get_expire_info(
expire_value: int, expire_style: str
) -> Tuple[Optional[datetime.datetime], int, int, str]:
@@ -86,6 +98,18 @@ async def get_random_code(style="num") -> str:
return code
async def calculate_file_hash(file: UploadFile, chunk_size=1024 * 1024) -> str:
sha = hashlib.sha256()
await file.seek(0)
while True:
chunk = await file.read(chunk_size)
if not chunk:
break
sha.update(chunk)
await file.seek(0)
return sha.hexdigest()
ip_limit = {
"error": IPRateLimit(count=settings.uploadCount, minutes=settings.errorMinute),
"upload": IPRateLimit(count=settings.errorCount, minutes=settings.errorMinute),
+134 -5
View File
@@ -1,8 +1,13 @@
import hashlib
import uuid
from fastapi import APIRouter, Form, UploadFile, File, Depends, HTTPException
from starlette import status
from apps.admin.dependencies import admin_required
from apps.base.models import FileCodes
from apps.base.schemas import SelectFileModel
from apps.base.utils import get_expire_info, get_file_path_name, ip_limit
from apps.base.models import FileCodes, UploadChunk
from apps.base.schemas import SelectFileModel, InitChunkUploadModel, CompleteUploadModel
from apps.base.utils import get_expire_info, get_file_path_name, ip_limit, get_chunk_file_path_name
from core.response import APIResponse
from core.settings import settings
from core.storage import storages, FileStorageInterface
@@ -142,13 +147,137 @@ async def download_file(key: str, code: str, ip: str = Depends(ip_limit["error"]
file_storage: FileStorageInterface = storages[settings.file_storage]()
if await get_select_token(code) != key:
ip_limit["error"].add_ip(ip)
has, file_code = await get_code_file_by_code(code, False)
if not has:
return APIResponse(code=404, detail="文件不存在")
return (
APIResponse(detail=file_code.text)
if file_code.text
else await file_storage.get_file_response(file_code)
)
chunk_api = APIRouter(prefix="/chunk", tags=["切片"])
@chunk_api.post("/upload/init/")
async def init_chunk_upload(data: InitChunkUploadModel):
# 秒传检查
existing = await FileCodes.filter(file_hash=data.file_hash).first()
if existing:
if await existing.is_expired():
file_storage: FileStorageInterface = storages[settings.file_storage](
)
await file_storage.delete_file(existing)
await existing.delete()
else:
return APIResponse(detail={
"code": existing.code,
"existed": True,
"name": f'{existing.prefix}{existing.suffix}'
})
# 创建上传会话
upload_id = uuid.uuid4().hex
total_chunks = (data.file_size + data.chunk_size - 1) // data.chunk_size
await UploadChunk.create(
upload_id=upload_id,
chunk_index=-1,
total_chunks=total_chunks,
file_size=data.file_size,
chunk_size=data.chunk_size,
chunk_hash=data.file_hash,
file_name=data.file_name,
)
# 获取已上传的分片列表
uploaded_chunks = await UploadChunk.filter(
upload_id=upload_id,
completed=True
).values_list('chunk_index', flat=True)
return APIResponse(detail={
"existed": False,
"upload_id": upload_id,
"chunk_size": data.chunk_size,
"total_chunks": total_chunks,
"uploaded_chunks": uploaded_chunks
})
@chunk_api.post("/upload/chunk/{upload_id}/{chunk_index}")
async def upload_chunk(
upload_id: str,
chunk_index: int,
chunk: UploadFile = File(...),
):
# 获取上传会话信息
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
if not chunk_info:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
# 检查分片索引有效性
if chunk_index < 0 or chunk_index >= chunk_info.total_chunks:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="无效的分片索引")
# 读取分片数据并计算哈希
chunk_data = await chunk.read()
chunk_hash = hashlib.sha256(chunk_data).hexdigest()
# 更新或创建分片记录
await UploadChunk.update_or_create(
upload_id=upload_id,
chunk_index=chunk_index,
defaults={
'chunk_hash': chunk_hash,
'completed': True,
'file_size': chunk_info.file_size,
'total_chunks': chunk_info.total_chunks,
'chunk_size': chunk_info.chunk_size,
'file_name': chunk_info.file_name
}
)
# 获取文件路径
_, _, _, _, save_path = await get_chunk_file_path_name(chunk_info.file_name, upload_id)
# 保存分片到存储
storage = storages[settings.file_storage]()
await storage.save_chunk(upload_id, chunk_index, chunk_data, chunk_hash, save_path)
return APIResponse(detail={"chunk_hash": chunk_hash})
@chunk_api.post("/upload/complete/{upload_id}")
async def complete_upload(upload_id: str, data: CompleteUploadModel, ip: str = Depends(ip_limit["upload"])):
# 获取上传基本信息
chunk_info = await UploadChunk.filter(upload_id=upload_id, chunk_index=-1).first()
if not chunk_info:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="上传会话不存在")
storage = storages[settings.file_storage]()
# 验证所有分片
completed_chunks = await UploadChunk.filter(
upload_id=upload_id,
completed=True
).count()
if completed_chunks != chunk_info.total_chunks:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="分片不完整")
# 获取文件路径
path, suffix, prefix, _, save_path = await get_chunk_file_path_name(chunk_info.file_name, upload_id)
# 合并文件并计算哈希
await storage.merge_chunks(upload_id, chunk_info, save_path)
# 创建文件记录
expired_at, expired_count, used_count, code = await get_expire_info(data.expire_value, data.expire_style)
await FileCodes.create(
code=code,
file_hash=chunk_info.chunk_hash,
is_chunked=True,
upload_id=upload_id,
size=chunk_info.file_size,
expired_at=expired_at,
expired_count=expired_count,
used_count=used_count,
file_path=path,
uuid_file_name=f"{prefix}{suffix}",
prefix=prefix,
suffix=suffix
)
# 清理临时文件
await storage.clean_chunks(upload_id, save_path)
return APIResponse(detail={"code": code, "name": chunk_info.file_name})
+6 -5
View File
@@ -4,6 +4,7 @@ import os
from tortoise import Tortoise
from core.logger import logger
from core.settings import data_root
@@ -32,7 +33,7 @@ async def init_db():
await execute_migrations()
except Exception as e:
print(f"数据库初始化失败: {str(e)}")
logger.error(f"数据库初始化失败: {str(e)}")
raise
@@ -58,7 +59,7 @@ async def execute_migrations():
)
if not executed[1]:
print(f"执行迁移: {file_name}")
logger.info(f"执行迁移: {file_name}")
# 导入并执行migration
module_path = migration_file.replace("/", ".").replace("\\", ".").replace(".py", "")
try:
@@ -70,11 +71,11 @@ async def execute_migrations():
"INSERT INTO migrates (migration_file) VALUES (?)",
[file_name]
)
print(f"迁移完成: {file_name}")
logger.info(f"迁移完成: {file_name}")
except Exception as e:
print(f"迁移 {file_name} 执行失败: {str(e)}")
logger.error(f"迁移 {file_name} 执行失败: {str(e)}")
raise
except Exception as e:
print(f"迁移过程发生错误: {str(e)}")
logger.error(f"迁移过程发生错误: {str(e)}")
raise
+28
View File
@@ -0,0 +1,28 @@
import logging
import sys
def setup_logger():
# 创建logger对象
_logger = logging.getLogger('FileCodeBox')
_logger.setLevel(logging.INFO)
# 创建控制台处理器
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.INFO)
# 设置日志格式
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
console_handler.setFormatter(formatter)
# 添加处理器到logger
_logger.addHandler(console_handler)
return _logger
# 创建全局logger实例
logger = setup_logger()
+1
View File
@@ -43,6 +43,7 @@ DEFAULT_CONFIG = {
"uploadSize": 1024 * 1024 * 10,
"expireStyle": ["day", "hour", "minute", "forever", "count"],
"uploadMinute": 1,
"enableChunk": 0,
"webdav_url": "",
"webdav_password": "",
"webdav_username": "",
+286 -15
View File
@@ -2,21 +2,25 @@
# @Author : Lan
# @File : storage.py
# @Software: PyCharm
import hashlib
from core.logger import logger
import shutil
from typing import Optional
from urllib.parse import quote, unquote
import aiofiles
import aiohttp
import asyncio
from pathlib import Path
import datetime
import io
import re
import sys
import aioboto3
from botocore.config import Config
from fastapi import HTTPException, Response, UploadFile
from core.response import APIResponse
from core.settings import data_root, settings
from apps.base.models import FileCodes
from apps.base.models import FileCodes, UploadChunk
from core.utils import get_file_url
from fastapi.responses import FileResponse
@@ -61,6 +65,35 @@ class FileStorageInterface:
"""
raise NotImplementedError
async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str):
"""
保存分片文件
:param upload_id: 上传会话ID
:param chunk_index: 分片索引
:param chunk_data: 分片数据
:param chunk_hash: 分片哈希值
:param save_path: 文件保存路径
"""
raise NotImplementedError
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
"""
合并分片文件并返回文件路径和完整哈希值
:param upload_id: 上传会话ID
:param chunk_info: 分片信息
:param save_path: 文件保存路径
:return: (文件路径, 文件哈希值)
"""
raise NotImplementedError
async def clean_chunks(self, upload_id: str, save_path: str):
"""
清理临时分片文件
:param upload_id: 上传会话ID
:param save_path: 文件保存路径
"""
raise NotImplementedError
class SystemFileStorage(FileStorageInterface):
def __init__(self):
@@ -101,6 +134,57 @@ class SystemFileStorage(FileStorageInterface):
filename=filename # 保留原始文件名以备某些场景使用
)
async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str):
"""
保存分片文件到本地文件系统
:param upload_id: 上传会话ID
:param chunk_index: 分片索引
:param chunk_data: 分片数据
:param chunk_hash: 分片哈希值
:param save_path: 文件保存路径
"""
chunk_dir = self.root_path / save_path
chunk_path = chunk_dir.parent / 'chunks' / f"{chunk_index}.part"
if not chunk_path.parent.exists():
chunk_path.parent.mkdir(parents=True)
async with aiofiles.open(chunk_path, "wb") as f:
await f.write(chunk_data)
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
"""
合并本地文件系统的分片文件并返回文件路径和完整哈希值
:param upload_id: 上传会话ID
:param chunk_info: 分片信息
:param save_path: 文件保存路径
:return: (文件路径, 文件哈希值)
"""
output_dir = self.root_path / save_path
output_dir.parent.mkdir(parents=True, exist_ok=True)
file_sha256 = hashlib.sha256()
async with aiofiles.open(output_dir, "wb") as out_file:
for i in range(chunk_info.total_chunks):
# 获取分片记录
chunk_record = await UploadChunk.get(upload_id=upload_id, chunk_index=i)
chunk_path = output_dir.parent / 'chunks' / f"{i}.part"
async with aiofiles.open(chunk_path, "rb") as in_file:
chunk_data = await in_file.read()
current_hash = hashlib.sha256(chunk_data).hexdigest()
if current_hash != chunk_record.chunk_hash:
raise ValueError(f"分片{i}哈希不匹配")
file_sha256.update(chunk_data)
await out_file.write(chunk_data)
return output_dir, file_sha256.hexdigest()
async def clean_chunks(self, upload_id: str, save_path: str):
"""
清理本地文件系统的临时分片文件
:param upload_id: 上传会话ID
:param save_path: 文件保存路径
"""
chunk_dir = (self.root_path / save_path).parent / 'chunks'
if chunk_dir.exists():
shutil.rmtree(chunk_dir)
class S3FileStorage(FileStorageInterface):
def __init__(self):
@@ -205,6 +289,99 @@ class S3FileStorage(FileStorageInterface):
)
return result
async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str):
chunk_key = str(Path(save_path).parent / "chunks" /
upload_id / f"{chunk_index}.part")
async with self.session.client('s3') as s3:
response = await s3.upload_part(
Bucket=self.bucket_name,
Key=chunk_key,
PartNumber=chunk_index + 1,
UploadId=upload_id,
Body=chunk_data
)
await s3.put_object_tagging(
Bucket=self.bucket_name,
Key=chunk_key,
Tagging={
'TagSet': [
{'Key': 'ChunkHash', 'Value': chunk_hash},
{'Key': 'ETag', 'Value': response['ETag']}
]
}
)
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
file_sha256 = hashlib.sha256()
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
async with self.session.client('s3') as s3:
# 获取所有分片
parts = await s3.list_parts(
Bucket=self.bucket_name,
Key=chunk_dir,
UploadId=upload_id
)
part_list = []
for part in parts['Parts']:
part_number = part['PartNumber']
chunk_index = part_number - 1
chunk_record = await UploadChunk.get(upload_id=upload_id, chunk_index=chunk_index)
response = await s3.get_object(
Bucket=self.bucket_name,
Key=f"{chunk_dir}/{chunk_index}.part",
PartNumber=part_number
)
chunk_data = await response['Body'].read()
current_hash = hashlib.sha256(chunk_data).hexdigest()
if current_hash != chunk_record.chunk_hash:
raise Exception(f"分片{chunk_index}哈希不匹配")
file_sha256.update(chunk_data)
part_list.append(
{'PartNumber': part_number, 'ETag': part['ETag']})
# 完成合并
await s3.complete_multipart_upload(
Bucket=self.bucket_name,
Key=save_path,
UploadId=upload_id,
MultipartUpload={'Parts': part_list}
)
return save_path, file_sha256.hexdigest()
async def clean_chunks(self, upload_id: str, save_path: str):
"""
清理 S3 上的临时分片文件
:param upload_id: 上传会话ID
:param save_path: 文件保存路径
"""
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
async with self.session.client('s3') as s3:
try:
# 终止未完成的分片上传会话
await s3.abort_multipart_upload(
Bucket=self.bucket_name,
Key=chunk_dir,
UploadId=upload_id
)
except Exception as e:
# 如果上传会话不存在或其他错误,忽略
logger.info(f"清理 S3 分片时出错: {e}")
try:
# 清理已上传的分片数据
parts = await s3.list_parts(
Bucket=self.bucket_name,
Key=chunk_dir,
UploadId=upload_id
)
for part in parts.get('Parts', []):
await s3.delete_object(
Bucket=self.bucket_name,
Key=f"{chunk_dir}/{part['PartNumber'] - 1}.part"
)
except Exception as e:
# 如果分片数据不存在或其他错误,忽略
logger.info(f"清理 S3 分片数据时出错: {e}")
class OneDriveFileStorage(FileStorageInterface):
def __init__(self):
@@ -235,7 +412,8 @@ class OneDriveFileStorage(FileStorageInterface):
if e.code == "itemNotFound":
client.me.drive.root.create_folder(settings.onedrive_root_path)
self.root_path = (
client.me.drive.root.get_by_path(settings.onedrive_root_path)
client.me.drive.root.get_by_path(
settings.onedrive_root_path)
.get()
.execute_query()
)
@@ -289,9 +467,9 @@ class OneDriveFileStorage(FileStorageInterface):
await asyncio.to_thread(self._delete, await file_code.get_file_path())
def _convert_link_to_download_link(self, link):
p1 = re.search(r"https:\/\/(.+)\.sharepoint\.com", link).group(1)
p2 = re.search(r"personal\/(.+)\/", link).group(1)
p3 = re.search(rf"{p2}\/(.+)", link).group(1)
p1 = re.search(r"https://(.+)\.sharepoint\.com", link).group(1)
p2 = re.search(r"personal/(.+)/", link).group(1)
p3 = re.search(rf"{p2}/(.+)", link).group(1)
return f"https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}"
def _get_file_url(self, save_path, name):
@@ -300,11 +478,12 @@ class OneDriveFileStorage(FileStorageInterface):
expiration_datetime = datetime.datetime.now(
tz=datetime.timezone.utc
) + datetime.timedelta(hours=1)
expiration_datetime = expiration_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
premission = remote_file.create_link(
expiration_datetime = expiration_datetime.strftime(
"%Y-%m-%dT%H:%M:%SZ")
permission = remote_file.create_link(
"view", "anonymous", expiration_datetime=expiration_datetime
).execute_query()
return self._convert_link_to_download_link(premission.link.webUrl)
return self._convert_link_to_download_link(permission.link.webUrl)
async def get_file_response(self, file_code: FileCodes):
try:
@@ -369,12 +548,13 @@ class OpenDALFileStorage(FileStorageInterface):
try:
filename = file_code.prefix + file_code.suffix
content = await self.operator.read(await file_code.get_file_path())
headers = {"Content-Disposition": f'attachment; filename="{filename}"'}
headers = {
"Content-Disposition": f'attachment; filename="{filename}"'}
return Response(
content, headers=headers, media_type="application/octet-stream"
)
except Exception as e:
print(e, file=sys.stderr)
logger.info(e)
raise HTTPException(status_code=404, detail="文件已过期删除")
@@ -459,7 +639,8 @@ class WebDAVFileStorage(FileStorageInterface):
async with aiohttp.ClientSession(auth=self.auth) as session:
content = await file.read() # 注意:大文件需要分块读取
async with session.put(
url, data=content, headers={"Content-Type": file.content_type}
url, data=content, headers={
"Content-Type": file.content_type}
) as resp:
if resp.status not in (200, 201, 204):
content = await resp.text()
@@ -468,7 +649,8 @@ class WebDAVFileStorage(FileStorageInterface):
detail=f"文件上传失败: {content[:200]}",
)
except aiohttp.ClientError as e:
raise HTTPException(status_code=503, detail=f"WebDAV连接异常: {str(e)}")
raise HTTPException(
status_code=503, detail=f"WebDAV连接异常: {str(e)}")
async def delete_file(self, file_code: FileCodes):
"""删除WebDAV文件及空目录"""
@@ -490,7 +672,8 @@ class WebDAVFileStorage(FileStorageInterface):
await self._delete_empty_dirs(file_path, session)
except aiohttp.ClientError as e:
raise HTTPException(status_code=503, detail=f"WebDAV连接异常: {str(e)}")
raise HTTPException(
status_code=503, detail=f"WebDAV连接异常: {str(e)}")
async def get_file_url(self, file_code: FileCodes):
return await get_file_url(file_code.code)
@@ -519,7 +702,95 @@ class WebDAVFileStorage(FileStorageInterface):
},
)
except aiohttp.ClientError as e:
raise HTTPException(status_code=503, detail=f"WebDAV连接异常: {str(e)}")
raise HTTPException(
status_code=503, detail=f"WebDAV连接异常: {str(e)}")
async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str):
chunk_path = str(Path(save_path).parent / "chunks" /
upload_id / f"{chunk_index}.part")
chunk_url = self._build_url(chunk_path)
async with aiohttp.ClientSession(auth=self.auth) as session:
await session.put(chunk_url, data=chunk_data)
propfind_url = self._build_url(chunk_path)
headers = {
'Content-Type': 'application/xml; charset=utf-8', 'Depth': '0'}
body = f"""
<D:propertyupdate xmlns:D="DAV:">
<D:set>
<D:prop>
<ChunkHash xmlns="urn:filecodebox">{chunk_hash}</ChunkHash>
</D:prop>
</D:set>
</D:propertyupdate>
"""
await session.request('PROPPATCH', propfind_url, headers=headers, data=body)
async def merge_chunks(self, upload_id: str, chunk_info: UploadChunk, save_path: str) -> tuple[str, str]:
file_sha256 = hashlib.sha256()
output_url = self._build_url(save_path)
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
async with aiohttp.ClientSession(auth=self.auth) as session:
await session.put(output_url, headers={'Content-Length': '0'})
for i in range(chunk_info.total_chunks):
chunk_path = f"{chunk_dir}/{i}.part"
chunk_url = self._build_url(chunk_path)
propfind_url = self._build_url(chunk_path)
headers = {
'Content-Type': 'application/xml; charset=utf-8', 'Depth': '0'}
body = """
<D:propfind xmlns:D="DAV:">
<D:prop>
<ChunkHash xmlns="urn:filecodebox"/>
</D:prop>
</D:propfind>
"""
async with session.request('PROPFIND', propfind_url, headers=headers, data=body) as resp:
xml_data = await resp.text()
chunk_hash = re.search(
r'<ChunkHash[^>]*>([^<]+)</ChunkHash>', xml_data).group(1)
file_sha256.update(bytes.fromhex(chunk_hash))
async with session.get(chunk_url) as resp:
chunk_data = await resp.read()
await session.request('PATCH', output_url, headers={
'Content-Type': 'application/octet-stream',
'Content-Length': str(len(chunk_data)),
'Content-Range': f'bytes */{chunk_info.file_size}'
}, data=chunk_data)
return save_path, file_sha256.hexdigest()
async def clean_chunks(self, upload_id: str, save_path: str):
"""
清理 WebDAV 上的临时分片文件
:param upload_id: 上传会话ID
:param save_path: 文件保存路径
"""
chunk_dir = str(Path(save_path).parent / "chunks" / upload_id)
chunk_dir_url = self._build_url(chunk_dir)
async with aiohttp.ClientSession(auth=self.auth) as session:
try:
# 检查分片目录是否存在
async with session.request("PROPFIND", chunk_dir_url, headers={"Depth": "1"}) as resp:
if resp.status == 207: # 207 表示 Multi-Status
# 获取目录下的所有分片文件
xml_data = await resp.text()
file_paths = re.findall(
r'<D:href>(.*?)</D:href>', xml_data)
for file_path in file_paths:
if file_path.endswith(".part"):
# 删除分片文件
file_url = self._build_url(file_path)
async with session.delete(file_url) as delete_resp:
if delete_resp.status not in (200, 204, 404):
logger.info(f"删除分片文件失败: {file_path}")
# 删除分片目录
async with session.delete(chunk_dir_url) as delete_resp:
if delete_resp.status not in (200, 204, 404):
logger.info(f"删除分片目录失败: {chunk_dir_url}")
else:
logger.info(f"分片目录不存在: {chunk_dir_url}")
except Exception as e:
logger.info(f"清理 WebDAV 分片时出错: {e}")
storages = {
+2 -1
View File
@@ -3,6 +3,7 @@
# @File : tasks.py
# @Software: PyCharm
import asyncio
import logging
import os
from tortoise.expressions import Q
@@ -32,6 +33,6 @@ async def delete_expire_files():
await file_storage.delete_file(exp)
await exp.delete()
except Exception as e:
print(e)
logging.error(e)
finally:
await asyncio.sleep(600)
+8 -1
View File
@@ -14,12 +14,13 @@ from tortoise.contrib.fastapi import register_tortoise
from apps.base.models import KeyValue
from apps.base.utils import ip_limit
from apps.base.views import share_api
from apps.base.views import share_api, chunk_api
from apps.admin.views import admin_api
from core.database import init_db
from core.response import APIResponse
from core.settings import data_root, settings, BASE_DIR, DEFAULT_CONFIG
from core.tasks import delete_expire_files
from core.logger import logger
from contextlib import asynccontextmanager
from tortoise import Tortoise
@@ -27,6 +28,7 @@ from tortoise import Tortoise
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("正在初始化应用...")
# 初始化数据库
await init_db()
@@ -40,14 +42,17 @@ async def lifespan(app: FastAPI):
# 启动后台任务
task = asyncio.create_task(delete_expire_files())
logger.info("应用初始化完成")
try:
yield
finally:
# 清理操作
logger.info("正在关闭应用...")
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await Tortoise.close_connections()
logger.info("应用已关闭")
async def load_config():
@@ -92,6 +97,7 @@ register_tortoise(
)
app.include_router(share_api)
app.include_router(chunk_api)
app.include_router(admin_api)
@@ -128,6 +134,7 @@ async def get_config():
"explain": settings.page_explain,
"uploadSize": settings.uploadSize,
"expireStyle": settings.expireStyle,
"enableChunk": settings.enableChunk if settings.file_storage == "local" and settings.enableChunk else 0,
"openUpload": settings.openUpload,
"notify_title": settings.notify_title,
"notify_content": settings.notify_content,
@@ -1,4 +1,4 @@
import{c as i,d as k,r as h,o as u,I as v,a as d,b as x,e as t,n as o,g as e,i as n,X as w,F as _,p as M,A as C,x as z,z as L,J as B,K as D,q as F,t as I}from"./index-EADe6V-l.js";import{B as j}from"./box-brjUM9cP.js";/**
import{c as i,d as k,r as h,o as u,I as v,a as d,b as x,e as t,n as o,g as e,i as n,X as w,F as _,p as M,A as C,x as z,z as L,J as B,K as D,q as F,t as I}from"./index-CBPNirpK.js";import{B as j}from"./box-BGTnNORf.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
@@ -1,4 +1,4 @@
import{c as g,d as v,H as w,o as _,a as p,b as u,e as t,n as a,g as e,t as i,i as y,F as k,p as U,x as F,y as M,J as C,K as z}from"./index-EADe6V-l.js";import{F as m}from"./file-CTH8PUKS.js";import{H as D,T as S}from"./trash-COYxyedR.js";/**
import{c as g,d as v,H as w,o as _,a as p,b as u,e as t,n as a,g as e,t as i,i as y,F as k,p as U,x as F,y as M,J as C,K as z}from"./index-CBPNirpK.js";import{F as m}from"./file-CNpWcMOu.js";import{H as D,T as S}from"./trash-DAq3m0gp.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
@@ -1,4 +1,4 @@
import{c as B,d as T,u as q,r as _,f as k,a as c,b as g,e as t,n as o,g as a,l as P,v as A,i as m,q as H,F as v,p as w,t as d,x as I,y as S}from"./index-EADe6V-l.js";import{F as L}from"./file-CTH8PUKS.js";/**
import{c as B,d as T,u as q,r as _,f as k,a as c,b as g,e as t,n as o,g as a,l as P,v as A,i as m,q as H,F as v,p as w,t as d,x as I,y as S}from"./index-CBPNirpK.js";import{F as L}from"./file-CNpWcMOu.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
@@ -1 +1 @@
import{G as y,r as u,d as b,u as v,a as w,b as x,e,n as l,g as o,i as h,h as k,l as S,v as A,q as V,t as D,x as B,A as P,y as _,_ as j}from"./index-EADe6V-l.js";import{B as M}from"./box-brjUM9cP.js";const z=y("adminData",()=>{const d=u(localStorage.getItem("adminPassword")||"");function n(a){d.value=a,localStorage.setItem("token",a)}return{adminPassword:d,updateAdminPwd:n}}),I={class:"mx-auto h-16 w-16 relative"},L={class:"rounded-md shadow-sm -space-y-px"},N=["disabled"],T=b({__name:"LoginView",setup(d){const n=v(),a=u(""),i=u(!1),s=B("isDarkMode"),c=z(),p=()=>{let r=!0;return a.value?a.value.length<6&&(n.showAlert("密码长度至少为6位","error"),r=!1):(n.showAlert("无效的密码","error"),r=!1),r},m=P(),f=async()=>{if(p()){_.post("/admin/login",{password:a.value}).then(r=>{c.updateAdminPwd(r.detail.token),m.push("/admin")}).catch(r=>{n.showAlert(r.response.data.detail,"error")}),i.value=!0;try{await new Promise(r=>setTimeout(r,2e3))}catch{}finally{i.value=!1}}};return(r,t)=>(w(),x("div",{class:l(["min-h-screen flex items-center justify-center py-12 px-4 sm:px-6 lg:px-8 transition-colors duration-200 relative overflow-hidden",o(s)?"bg-gray-900":"bg-gray-50"])},[t[6]||(t[6]=e("div",{class:"absolute inset-0 z-0"},[e("div",{class:"cyber-grid"}),e("div",{class:"floating-particles"})],-1)),e("div",{class:l(["max-w-md w-full space-y-8 backdrop-blur-lg bg-opacity-20 p-8 rounded-xl border border-opacity-20",[o(s)?"bg-gray-800 border-gray-600":"bg-white/70 border-gray-200"]])},[e("div",null,[e("div",I,[t[1]||(t[1]=e("div",{class:"absolute inset-0 bg-gradient-to-r from-cyan-500 via-purple-500 to-pink-500 rounded-full animate-spin-slow"},null,-1)),t[2]||(t[2]=e("div",{class:"absolute -inset-2 bg-gradient-to-r from-cyan-500 via-purple-500 to-pink-500 rounded-full opacity-50 blur-md animate-pulse"},null,-1)),e("div",{class:l(["absolute inset-1 rounded-full flex items-center justify-center",o(s)?"bg-gray-800":"bg-white"])},[h(o(M),{class:l(["h-8 w-8",o(s)?"text-cyan-400":"text-cyan-600"])},null,8,["class"])],2)]),e("h2",{class:l(["mt-6 text-center text-3xl font-extrabold",o(s)?"text-white":"text-gray-900"])}," 登录 ",2)]),e("form",{class:"mt-8 space-y-6",onSubmit:k(f,["prevent"])},[t[5]||(t[5]=e("input",{type:"hidden",name:"remember",value:"true"},null,-1)),e("div",L,[e("div",null,[t[3]||(t[3]=e("label",{for:"password",class:"sr-only"},"密码",-1)),S(e("input",{id:"password",name:"password",type:"password",autocomplete:"current-password",required:"","onUpdate:modelValue":t[0]||(t[0]=g=>a.value=g),class:l(["appearance-none rounded-t-md relative block w-full px-4 py-3 border transition-all duration-200 placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-cyan-500 focus:border-cyan-500 focus:z-10 sm:text-sm backdrop-blur-sm",o(s)?"bg-gray-800/50 border-gray-600 text-white placeholder-gray-400 hover:border-gray-500":"bg-white/50 border-gray-300 text-gray-900 hover:border-gray-400"]),placeholder:"密码"},null,2),[[A,a.value]])])]),e("div",null,[e("button",{type:"submit",class:l(["group relative w-full flex justify-center py-3 px-4 border border-transparent text-sm font-medium rounded-md text-white transition-all duration-300 transform hover:scale-[1.02] focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-cyan-500 shadow-lg hover:shadow-cyan-500/50",o(s)?"bg-gradient-to-r from-cyan-500 to-purple-500 hover:from-cyan-600 hover:to-purple-600":"bg-gradient-to-r from-cyan-600 to-purple-600 hover:from-cyan-700 hover:to-purple-700",i.value?"opacity-75 cursor-not-allowed":""]),disabled:i.value},[t[4]||(t[4]=e("span",{class:"absolute left-0 inset-y-0 flex items-center pl-3"},null,-1)),V(" "+D(i.value?"登录中...":"登录"),1)],10,N)])],32)],2)],2))}}),E=j(T,[["__scopeId","data-v-66eb7914"]]);export{E as default};
import{G as y,r as u,d as b,u as v,a as w,b as x,e,n as l,g as o,i as h,h as k,l as S,v as A,q as V,t as D,x as B,A as P,y as _,_ as j}from"./index-CBPNirpK.js";import{B as M}from"./box-BGTnNORf.js";const z=y("adminData",()=>{const d=u(localStorage.getItem("adminPassword")||"");function n(a){d.value=a,localStorage.setItem("token",a)}return{adminPassword:d,updateAdminPwd:n}}),I={class:"mx-auto h-16 w-16 relative"},L={class:"rounded-md shadow-sm -space-y-px"},N=["disabled"],T=b({__name:"LoginView",setup(d){const n=v(),a=u(""),i=u(!1),s=B("isDarkMode"),c=z(),p=()=>{let r=!0;return a.value?a.value.length<6&&(n.showAlert("密码长度至少为6位","error"),r=!1):(n.showAlert("无效的密码","error"),r=!1),r},m=P(),f=async()=>{if(p()){_.post("/admin/login",{password:a.value}).then(r=>{c.updateAdminPwd(r.detail.token),m.push("/admin")}).catch(r=>{n.showAlert(r.response.data.detail,"error")}),i.value=!0;try{await new Promise(r=>setTimeout(r,2e3))}catch{}finally{i.value=!1}}};return(r,t)=>(w(),x("div",{class:l(["min-h-screen flex items-center justify-center py-12 px-4 sm:px-6 lg:px-8 transition-colors duration-200 relative overflow-hidden",o(s)?"bg-gray-900":"bg-gray-50"])},[t[6]||(t[6]=e("div",{class:"absolute inset-0 z-0"},[e("div",{class:"cyber-grid"}),e("div",{class:"floating-particles"})],-1)),e("div",{class:l(["max-w-md w-full space-y-8 backdrop-blur-lg bg-opacity-20 p-8 rounded-xl border border-opacity-20",[o(s)?"bg-gray-800 border-gray-600":"bg-white/70 border-gray-200"]])},[e("div",null,[e("div",I,[t[1]||(t[1]=e("div",{class:"absolute inset-0 bg-gradient-to-r from-cyan-500 via-purple-500 to-pink-500 rounded-full animate-spin-slow"},null,-1)),t[2]||(t[2]=e("div",{class:"absolute -inset-2 bg-gradient-to-r from-cyan-500 via-purple-500 to-pink-500 rounded-full opacity-50 blur-md animate-pulse"},null,-1)),e("div",{class:l(["absolute inset-1 rounded-full flex items-center justify-center",o(s)?"bg-gray-800":"bg-white"])},[h(o(M),{class:l(["h-8 w-8",o(s)?"text-cyan-400":"text-cyan-600"])},null,8,["class"])],2)]),e("h2",{class:l(["mt-6 text-center text-3xl font-extrabold",o(s)?"text-white":"text-gray-900"])}," 登录 ",2)]),e("form",{class:"mt-8 space-y-6",onSubmit:k(f,["prevent"])},[t[5]||(t[5]=e("input",{type:"hidden",name:"remember",value:"true"},null,-1)),e("div",L,[e("div",null,[t[3]||(t[3]=e("label",{for:"password",class:"sr-only"},"密码",-1)),S(e("input",{id:"password",name:"password",type:"password",autocomplete:"current-password",required:"","onUpdate:modelValue":t[0]||(t[0]=g=>a.value=g),class:l(["appearance-none rounded-t-md relative block w-full px-4 py-3 border transition-all duration-200 placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-cyan-500 focus:border-cyan-500 focus:z-10 sm:text-sm backdrop-blur-sm",o(s)?"bg-gray-800/50 border-gray-600 text-white placeholder-gray-400 hover:border-gray-500":"bg-white/50 border-gray-300 text-gray-900 hover:border-gray-400"]),placeholder:"密码"},null,2),[[A,a.value]])])]),e("div",null,[e("button",{type:"submit",class:l(["group relative w-full flex justify-center py-3 px-4 border border-transparent text-sm font-medium rounded-md text-white transition-all duration-300 transform hover:scale-[1.02] focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-cyan-500 shadow-lg hover:shadow-cyan-500/50",o(s)?"bg-gradient-to-r from-cyan-500 to-purple-500 hover:from-cyan-600 hover:to-purple-600":"bg-gradient-to-r from-cyan-600 to-purple-600 hover:from-cyan-700 hover:to-purple-700",i.value?"opacity-75 cursor-not-allowed":""]),disabled:i.value},[t[4]||(t[4]=e("span",{class:"absolute left-0 inset-y-0 flex items-center pl-3"},null,-1)),V(" "+D(i.value?"登录中...":"登录"),1)],10,N)])],32)],2)],2))}}),E=j(T,[["__scopeId","data-v-66eb7914"]]);export{E as default};
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
.border-progress-container[data-v-2fbf5085]{position:relative;width:100%;height:100%}.border-progress-canvas[data-v-2fbf5085]{position:absolute;top:0;left:0;width:100%;height:100%;transition:all .3s ease}.fade-enter-active[data-v-1a2712ba],.fade-leave-active[data-v-1a2712ba]{transition:opacity .3s ease,transform .3s ease}.fade-enter-from[data-v-1a2712ba],.fade-leave-to[data-v-1a2712ba]{opacity:0;transform:translateY(10px)}@media (min-width: 640px){.sm\:w-120[data-v-1a2712ba]{width:30rem}}.fade-enter-to[data-v-1a2712ba],.fade-leave-from[data-v-1a2712ba]{opacity:1;transform:translateY(0)}.drawer-enter-active[data-v-1a2712ba],.drawer-leave-active[data-v-1a2712ba]{transition:transform .3s ease}.drawer-enter-from[data-v-1a2712ba],.drawer-leave-to[data-v-1a2712ba]{transform:translate(100%)}.list-enter-active[data-v-1a2712ba],.list-leave-active[data-v-1a2712ba]{transition:all .5s ease}.list-enter-from[data-v-1a2712ba],.list-leave-to[data-v-1a2712ba]{opacity:0;transform:translate(30px)}
@@ -1 +0,0 @@
.border-progress-container[data-v-2fbf5085]{position:relative;width:100%;height:100%}.border-progress-canvas[data-v-2fbf5085]{position:absolute;top:0;left:0;width:100%;height:100%;transition:all .3s ease}.fade-enter-active[data-v-3a0327b0],.fade-leave-active[data-v-3a0327b0]{transition:opacity .3s ease,transform .3s ease}.fade-enter-from[data-v-3a0327b0],.fade-leave-to[data-v-3a0327b0]{opacity:0;transform:translateY(10px)}@media (min-width: 640px){.sm\:w-120[data-v-3a0327b0]{width:30rem}}.fade-enter-to[data-v-3a0327b0],.fade-leave-from[data-v-3a0327b0]{opacity:1;transform:translateY(0)}.drawer-enter-active[data-v-3a0327b0],.drawer-leave-active[data-v-3a0327b0]{transition:transform .3s ease}.drawer-enter-from[data-v-3a0327b0],.drawer-leave-to[data-v-3a0327b0]{transform:translate(100%)}.list-enter-active[data-v-3a0327b0],.list-leave-active[data-v-3a0327b0]{transition:all .5s ease}.list-enter-from[data-v-3a0327b0],.list-leave-to[data-v-3a0327b0]{opacity:0;transform:translate(30px)}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,4 +1,4 @@
import{c as a}from"./index-EADe6V-l.js";/**
import{c as a}from"./index-CBPNirpK.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
@@ -1,4 +1,4 @@
import{c as a}from"./index-EADe6V-l.js";/**
import{c as a}from"./index-CBPNirpK.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,4 +1,4 @@
import{c as a}from"./index-EADe6V-l.js";/**
import{c as a}from"./index-CBPNirpK.js";/**
* @license lucide-vue-next v0.445.0 - ISC
*
* This source code is licensed under the ISC license.
+2 -1
View File
@@ -10,8 +10,9 @@
<meta name="description" content="{{description}}" />
<meta name="keywords" content="{{keywords}}" />
<meta name="generator" content="FileCodeBox2.2" />
<meta name="github" content="https://github.com/vastsa/FileCodeBox" />
<title>{{title}}</title>
<script type="module" crossorigin src="/assets/index-EADe6V-l.js"></script>
<script type="module" crossorigin src="/assets/index-CBPNirpK.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-CDOydXMY.css">
</head>
<body>