From a198886d11b6ec07a93634deceed0528e101c282 Mon Sep 17 00:00:00 2001 From: silver-ymz Date: Thu, 24 Aug 2023 14:04:00 +0800 Subject: [PATCH 1/2] feat: support server transfer for opendal Signed-off-by: silver-ymz --- apps/base/views.py | 10 ++--- core/settings.py | 9 ++-- core/storage.py | 100 ++++++++++++++++++++++++++++++--------------- core/utils.py | 19 +++++++++ readme_opendal.md | 30 ++++++++++++++ 5 files changed, 124 insertions(+), 44 deletions(-) create mode 100644 readme_opendal.md diff --git a/apps/base/views.py b/apps/base/views.py index ac6766a..f75c28e 100644 --- a/apps/base/views.py +++ b/apps/base/views.py @@ -3,8 +3,6 @@ # @File : views.py # @Software: PyCharm from fastapi import APIRouter, Form, UploadFile, File, Depends, HTTPException -from starlette.responses import FileResponse - from apps.admin.depends import admin_required from apps.base.models import FileCodes from apps.base.pydantics import SelectFileModel @@ -12,6 +10,7 @@ from apps.base.utils import get_expire_info, get_file_path_name, error_ip_limit, from core.response import APIResponse from core.settings import settings from core.storage import file_storage +from core.utils import get_select_token share_api = APIRouter( prefix='/share', @@ -83,7 +82,7 @@ async def select_file(data: SelectFileModel, ip: str = Depends(error_ip_limit)): @share_api.get('/download') async def download_file(key: str, code: str, ip: str = Depends(error_ip_limit)): - is_valid = await file_storage.get_select_token(code) == key + is_valid = await get_select_token(code) == key if not is_valid: error_ip_limit.add_ip(ip) file_code = await FileCodes.filter(code=code).first() @@ -92,7 +91,4 @@ async def download_file(key: str, code: str, ip: str = Depends(error_ip_limit)): if file_code.text: return APIResponse(detail=file_code.text) else: - file_path = file_storage.root_path / await file_code.get_file_path() - if not file_path.exists(): - return APIResponse(code=404, detail='文件已过期删除') - return FileResponse(file_path, filename=file_code.prefix + file_code.suffix) + return await file_storage.get_file_response(file_code) diff --git a/core/settings.py b/core/settings.py index 1becaa8..ab91d2a 100644 --- a/core/settings.py +++ b/core/settings.py @@ -7,7 +7,7 @@ from pathlib import Path data_root = Path('./data') if not data_root.exists(): data_root.mkdir(parents=True, exist_ok=True) -env_path = data_root / '.env2' +env_path = data_root / '.env' default_value = { 'file_storage': 'local', 'name': '文件快递柜-FileCodeBox', @@ -45,9 +45,9 @@ class Settings: # 更新default_value with open(env_path, 'r', encoding='utf-8') as f: for line in f.readlines(): - key, value = line.strip().split('=') + key, value = line.strip().split('=', maxsplit=1) # 将字符串转换为原本的类型 - if isinstance(default_value[key], int): + if not key.startswith('opendal_') and isinstance(default_value[key], int): value = int(value) default_value[key] = value # 更新self @@ -65,5 +65,8 @@ class Settings: for key, value in self.__dict__.items(): f.write(f'{key}={value}\n') + def items(self): + return self.__dict__.items() + settings = Settings() diff --git a/core/storage.py b/core/storage.py index 9c75c52..718081e 100644 --- a/core/storage.py +++ b/core/storage.py @@ -3,23 +3,57 @@ # @File : storage.py # @Software: PyCharm import asyncio -import hashlib -import time from pathlib import Path import datetime import re - +import sys import aioboto3 -from fastapi import UploadFile +from fastapi import HTTPException, Response, UploadFile +from core.response import APIResponse from core.settings import data_root, settings from apps.base.models import FileCodes +from core.utils import get_file_url +from fastapi.responses import FileResponse -class SystemFileStorage: +class FileStorageInterface: + def __init__(self): + raise NotImplementedError + + async def save_file(self, file: UploadFile, save_path: str): + """ + 保存文件 + """ + raise NotImplementedError + + async def delete_file(self, file_code: FileCodes): + """ + 删除文件 + """ + raise NotImplementedError + + async def get_file_url(self, file_code: FileCodes): + """ + 获取文件分享的url + + 如果服务不支持直接访问文件，可以通过服务器中转下载。 + 此时，此方法可以调用 utils.py 中的 `get_file_url` 方法，获取服务器中转下载的url + """ + raise NotImplementedError + + async def get_file_response(self, file_code: FileCodes): + """ + 获取文件响应 + + 如果服务不支持直接访问文件，则需要实现该方法，返回文件响应 + 其余情况，可以不实现该方法 + """ + raise NotImplementedError + +class SystemFileStorage(FileStorageInterface): def __init__(self): self.chunk_size = 256 * 1024 self.root_path = data_root - self.token = '123456' def _save(self, file, save_path): with open(save_path, 'wb') as f: @@ -39,14 +73,17 @@ class SystemFileStorage: if save_path.exists(): save_path.unlink() - async def get_select_token(self, code): - return hashlib.sha256(f"{code}{int(time.time() / 1000)}000{self.token}".encode()).hexdigest() - async def get_file_url(self, file_code: FileCodes): - return f'/share/download?key={await self.get_select_token(file_code.code)}&code={file_code.code}' + return await get_file_url(file_code.code) + + async def get_file_response(self, file_code: FileCodes): + file_path = file_storage.root_path / await file_code.get_file_path() + if not file_path.exists(): + return APIResponse(code=404, detail='文件已过期删除') + return FileResponse(file_path, filename=file_code.prefix + file_code.suffix) -class S3FileStorage: +class S3FileStorage(FileStorageInterface): def __init__(self): self.access_key_id = settings.s3_access_key_id self.secret_access_key = settings.s3_secret_access_key @@ -72,7 +109,7 @@ class S3FileStorage: return result -class OneDriveFileStorage: +class OneDriveFileStorage(FileStorageInterface): def __init__(self): try: import msal @@ -163,13 +200,13 @@ class OneDriveFileStorage: result = await asyncio.to_thread(self._get_file_url, await file_code.get_file_path(), f'{file_code.prefix}{file_code.suffix}') return result -class OpenDALFileStorage: +class OpenDALFileStorage(FileStorageInterface): def __init__(self): try: import opendal except ImportError: raise ImportError('请先安装 `opendal`, 例如: "pip install opendal"') - self.service = settings.opendal_scheme + self.service = settings.opendal_scheme service_settings = {} for key, value in settings.items(): if key.startswith('opendal_' + self.service): @@ -178,30 +215,25 @@ class OpenDALFileStorage: self.operator = opendal.AsyncOperator(settings.opendal_scheme, **service_settings) async def save_file(self, file: UploadFile, save_path: str): - await self.operator.write(save_path, file.file) + await self.operator.write(save_path, file.file.read()) async def delete_file(self, file_code: FileCodes): await self.operator.delete(await file_code.get_file_path()) async def get_file_url(self, file_code: FileCodes): - # todo: It needs upstream to expose presign api from rust to python - if file_code.prefix == '文本分享': - return file_code.text - result = await self.operator.presign_read(await file_code.get_file_path()) - return result - -class FileStorageTemplate: - def __init__(self): - ... - - async def save_file(self, file: UploadFile, save_path: str): - ... - - async def delete_file(self, file_code: FileCodes): - ... - - async def get_file_url(self, file_code: FileCodes): - ... + return await get_file_url(file_code.code) + + async def get_file_response(self, file_code: FileCodes): + try: + filename=file_code.prefix + file_code.suffix + content = await self.operator.read(await file_code.get_file_path()) + headers = { + "Content-Disposition": f'attachment; filename="{filename}"' + } + return Response(content, headers=headers, media_type="application/octet-stream") + except Exception as e: + print(e, file=sys.stderr) + raise HTTPException(status_code=404, detail="文件已过期删除") storages = { @@ -210,4 +242,4 @@ storages = { 'onedrive': OneDriveFileStorage, 'opendal': OpenDALFileStorage, } -file_storage = storages[settings.file_storage]() +file_storage: FileStorageInterface = storages[settings.file_storage]() diff --git a/core/utils.py b/core/utils.py index ab856b9..e18b8e0 100644 --- a/core/utils.py +++ b/core/utils.py @@ -3,8 +3,10 @@ # @File : utils.py # @Software: PyCharm import datetime +import hashlib import random import string +import time from apps.base.depends import IPRateLimit @@ -36,3 +38,20 @@ async def get_now(): return datetime.datetime.now( datetime.timezone(datetime.timedelta(hours=8)) ) + +async def get_select_token(code: int): + """ + 获取下载token + :param code: + :return: + """ + token = "123456" + return hashlib.sha256(f"{code}{int(time.time() / 1000)}000{token}".encode()).hexdigest() + +async def get_file_url(code: int): + """ + 对于需要通过服务器中转下载的服务，获取文件下载地址 + :param code: + :return: + """ + return f'/share/download?key={await get_select_token(code)}&code={code}' \ No newline at end of file diff --git a/readme_opendal.md b/readme_opendal.md new file mode 100644 index 0000000..25cbd32 --- /dev/null +++ b/readme_opendal.md @@ -0,0 +1,30 @@ +# 通过 OpenDAL 集成存储的配置方法 + +## 需要配置的参数 + +```dotenv +file_storage=opendal +opendal_scheme= +opendal__=... +``` + +以 Gcs 为例，需要配置的参数如下： +```dotenv +file_storage=opendal +opendal_scheme=gcs +opendal_gcs_root= +opendal_gcs_bucket= +opendal_gcs_credential= +``` + +所有支持的服务可以在[此处](https://opendal.apache.org/docs/rust/opendal/services/index.html)查看。 +具体服务的配置参数与 OpenDAL 文档一致。 + +## 补充说明 + +通过 OpenDAL 集成的服务均通过服务器中转下载。因此，每次下载既消耗存储服务的流量，也消耗服务器的流量。 + +OpenDAL 和该项目本身都支持本地存储、`s3`、`onedrive`。不同之处有以下几点: +1. 项目的支持通过预签名实现，不消耗服务器流量。而 OpenDAL 通过服务器中转下载，消耗服务器流量。 +2. 项目的支持对于异常情况可能会有更多的调试信息，方便排查问题。 +3. OpenDAL 项目本身采用 Rust 编写，性能更好。 \ No newline at end of file From 40c2ad70a11c314b6f358a7a8ada3325bfb586db Mon Sep 17 00:00:00 2001 From: silver-ymz Date: Thu, 24 Aug 2023 14:06:00 +0800 Subject: [PATCH 2/2] typo Signed-off-by: silver-ymz --- readme_opendal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme_opendal.md b/readme_opendal.md index 25cbd32..c4d383f 100644 --- a/readme_opendal.md +++ b/readme_opendal.md @@ -25,6 +25,6 @@ opendal_gcs_credential= 通过 OpenDAL 集成的服务均通过服务器中转下载。因此，每次下载既消耗存储服务的流量，也消耗服务器的流量。 OpenDAL 和该项目本身都支持本地存储、`s3`、`onedrive`。不同之处有以下几点: -1. 项目的支持通过预签名实现，不消耗服务器流量。而 OpenDAL 通过服务器中转下载，消耗服务器流量。 +1. 项目的支持通过预签名实现，不消耗服务器流量。而 OpenDAL 通过服务器中转下载，消耗服务器流量。（本地存储除外） 2. 项目的支持对于异常情况可能会有更多的调试信息，方便排查问题。 3. OpenDAL 项目本身采用 Rust 编写，性能更好。 \ No newline at end of file