feat: support server transfer for opendal

Signed-off-by: silver-ymz <yinmingzhuo@gmail.com>

apps/base/views.py

@@ -3,8 +3,6 @@
 # @File    : views.py
 # @Software: PyCharm
 from fastapi import APIRouter, Form, UploadFile, File, Depends, HTTPException
-from starlette.responses import FileResponse
-
 from apps.admin.depends import admin_required
 from apps.base.models import FileCodes
 from apps.base.pydantics import SelectFileModel
@@ -12,6 +10,7 @@ from apps.base.utils import get_expire_info, get_file_path_name, error_ip_limit,
 from core.response import APIResponse
 from core.settings import settings
 from core.storage import file_storage
+from core.utils import get_select_token
 
 share_api = APIRouter(
     prefix='/share',
@@ -83,7 +82,7 @@ async def select_file(data: SelectFileModel, ip: str = Depends(error_ip_limit)):
 
 @share_api.get('/download')
 async def download_file(key: str, code: str, ip: str = Depends(error_ip_limit)):
-    is_valid = await file_storage.get_select_token(code) == key
+    is_valid = await get_select_token(code) == key
     if not is_valid:
         error_ip_limit.add_ip(ip)
     file_code = await FileCodes.filter(code=code).first()
@@ -92,7 +91,4 @@ async def download_file(key: str, code: str, ip: str = Depends(error_ip_limit)):
     if file_code.text:
         return APIResponse(detail=file_code.text)
     else:
-        file_path = file_storage.root_path / await file_code.get_file_path()
+        return await file_storage.get_file_response(file_code)
-        if not file_path.exists():
-            return APIResponse(code=404, detail='文件已过期删除')
-        return FileResponse(file_path, filename=file_code.prefix + file_code.suffix)

core/settings.py

@@ -7,7 +7,7 @@ from pathlib import Path
 data_root = Path('./data')
 if not data_root.exists():
     data_root.mkdir(parents=True, exist_ok=True)
-env_path = data_root / '.env2'
+env_path = data_root / '.env'
 default_value = {
     'file_storage': 'local',
     'name': '文件快递柜-FileCodeBox',
@@ -45,9 +45,9 @@ class Settings:
         # 更新default_value
         with open(env_path, 'r', encoding='utf-8') as f:
             for line in f.readlines():
-                key, value = line.strip().split('=')
+                key, value = line.strip().split('=', maxsplit=1)
                 # 将字符串转换为原本的类型
-                if isinstance(default_value[key], int):
+                if not key.startswith('opendal_') and isinstance(default_value[key], int):
                     value = int(value)
                 default_value[key] = value
         # 更新self
@@ -65,5 +65,8 @@ class Settings:
             for key, value in self.__dict__.items():
                 f.write(f'{key}={value}\n')
 
+    def items(self):
+        return self.__dict__.items()
+
 
 settings = Settings()

core/storage.py

@@ -3,23 +3,57 @@
 # @File    : storage.py
 # @Software: PyCharm
 import asyncio
-import hashlib
-import time
 from pathlib import Path
 import datetime
 import re
-
+import sys
 import aioboto3
-from fastapi import UploadFile
+from fastapi import HTTPException, Response, UploadFile
+from core.response import APIResponse
 from core.settings import data_root, settings
 from apps.base.models import FileCodes
+from core.utils import get_file_url
+from fastapi.responses import FileResponse
 
 
-class SystemFileStorage:
+class FileStorageInterface:
+    def __init__(self):
+        raise NotImplementedError
+
+    async def save_file(self, file: UploadFile, save_path: str):
+        """
+        保存文件
+        """
+        raise NotImplementedError
+
+    async def delete_file(self, file_code: FileCodes):
+        """
+        删除文件
+        """
+        raise NotImplementedError
+
+    async def get_file_url(self, file_code: FileCodes):
+        """
+        获取文件分享的url
+
+        如果服务不支持直接访问文件，可以通过服务器中转下载。
+        此时，此方法可以调用 utils.py 中的 `get_file_url` 方法，获取服务器中转下载的url
+        """
+        raise NotImplementedError
+
+    async def get_file_response(self, file_code: FileCodes):
+        """
+        获取文件响应
+
+        如果服务不支持直接访问文件，则需要实现该方法，返回文件响应
+        其余情况，可以不实现该方法
+        """
+        raise NotImplementedError
+
+class SystemFileStorage(FileStorageInterface):
     def __init__(self):
         self.chunk_size = 256 * 1024
         self.root_path = data_root
-        self.token = '123456'
 
     def _save(self, file, save_path):
         with open(save_path, 'wb') as f:
@@ -39,14 +73,17 @@ class SystemFileStorage:
         if save_path.exists():
             save_path.unlink()
 
-    async def get_select_token(self, code):
-        return hashlib.sha256(f"{code}{int(time.time() / 1000)}000{self.token}".encode()).hexdigest()
-
     async def get_file_url(self, file_code: FileCodes):
-        return f'/share/download?key={await self.get_select_token(file_code.code)}&code={file_code.code}'
+        return await get_file_url(file_code.code)
+    
+    async def get_file_response(self, file_code: FileCodes):
+        file_path = file_storage.root_path / await file_code.get_file_path()
+        if not file_path.exists():
+            return APIResponse(code=404, detail='文件已过期删除')
+        return FileResponse(file_path, filename=file_code.prefix + file_code.suffix)
 
 
-class S3FileStorage:
+class S3FileStorage(FileStorageInterface):
     def __init__(self):
         self.access_key_id = settings.s3_access_key_id
         self.secret_access_key = settings.s3_secret_access_key
@@ -72,7 +109,7 @@ class S3FileStorage:
             return result
 
 
-class OneDriveFileStorage:
+class OneDriveFileStorage(FileStorageInterface):
     def __init__(self):
         try:
             import msal
@@ -163,13 +200,13 @@ class OneDriveFileStorage:
         result = await asyncio.to_thread(self._get_file_url, await file_code.get_file_path(), f'{file_code.prefix}{file_code.suffix}')
         return result
 
-class OpenDALFileStorage:
+class OpenDALFileStorage(FileStorageInterface):
     def __init__(self):
         try:
             import opendal
         except ImportError:
             raise ImportError('请先安装 `opendal`, 例如: "pip install opendal"')
-        self.service = settings.opendal_scheme   
+        self.service = settings.opendal_scheme
         service_settings = {}
         for key, value in settings.items():
             if key.startswith('opendal_' + self.service):
@@ -178,30 +215,25 @@ class OpenDALFileStorage:
         self.operator = opendal.AsyncOperator(settings.opendal_scheme, **service_settings)
 
     async def save_file(self, file: UploadFile, save_path: str):
-        await self.operator.write(save_path, file.file)
+        await self.operator.write(save_path, file.file.read())
 
     async def delete_file(self, file_code: FileCodes):
         await self.operator.delete(await file_code.get_file_path())
 
     async def get_file_url(self, file_code: FileCodes):
-        # todo: It needs upstream to expose presign api from rust to python
+        return await get_file_url(file_code.code)
-        if file_code.prefix == '文本分享':
+    
-            return file_code.text
+    async def get_file_response(self, file_code: FileCodes):
-        result = await self.operator.presign_read(await file_code.get_file_path())
+        try:
-        return result
+            filename=file_code.prefix + file_code.suffix
-
+            content = await self.operator.read(await file_code.get_file_path())
-class FileStorageTemplate:
+            headers = {
-    def __init__(self):
+                "Content-Disposition": f'attachment; filename="{filename}"'
-        ...
+            }
-
+            return Response(content, headers=headers, media_type="application/octet-stream")
-    async def save_file(self, file: UploadFile, save_path: str):
+        except Exception as e:
-        ...
+            print(e, file=sys.stderr)
-
+            raise HTTPException(status_code=404, detail="文件已过期删除")
-    async def delete_file(self, file_code: FileCodes):
-        ...
-
-    async def get_file_url(self, file_code: FileCodes):
-        ...
 
 
 storages = {
@@ -210,4 +242,4 @@ storages = {
     'onedrive': OneDriveFileStorage,
     'opendal': OpenDALFileStorage,
 }
-file_storage = storages[settings.file_storage]()
+file_storage: FileStorageInterface = storages[settings.file_storage]()

core/utils.py

@@ -3,8 +3,10 @@
 # @File    : utils.py
 # @Software: PyCharm
 import datetime
+import hashlib
 import random
 import string
+import time
 
 from apps.base.depends import IPRateLimit
 
@@ -36,3 +38,20 @@ async def get_now():
     return datetime.datetime.now(
         datetime.timezone(datetime.timedelta(hours=8))
     )
+
+async def get_select_token(code: int):
+    """
+    获取下载token
+    :param code:
+    :return:
+    """
+    token = "123456"
+    return hashlib.sha256(f"{code}{int(time.time() / 1000)}000{token}".encode()).hexdigest()
+
+async def get_file_url(code: int):
+    """
+    对于需要通过服务器中转下载的服务，获取文件下载地址
+    :param code:
+    :return:
+    """
+    return f'/share/download?key={await get_select_token(code)}&code={code}'

readme_opendal.md

@@ -0,0 +1,30 @@
+# 通过 OpenDAL 集成存储的配置方法
+
+## 需要配置的参数
+
+```dotenv
+file_storage=opendal
+opendal_scheme=<service_name>
+opendal_<service_name>_<service_setting>=...
+```
+
+以 Gcs 为例，需要配置的参数如下：
+```dotenv
+file_storage=opendal
+opendal_scheme=gcs
+opendal_gcs_root=<root>
+opendal_gcs_bucket=<bucket_name>
+opendal_gcs_credential=<base64_credential>
+```
+
+所有支持的服务可以在[此处](https://opendal.apache.org/docs/rust/opendal/services/index.html)查看。
+具体服务的配置参数与 OpenDAL 文档一致。
+
+## 补充说明
+
+通过 OpenDAL 集成的服务均通过服务器中转下载。因此，每次下载既消耗存储服务的流量，也消耗服务器的流量。
+
+OpenDAL 和该项目本身都支持本地存储、`s3`、`onedrive`。不同之处有以下几点:
+1. 项目的支持通过预签名实现，不消耗服务器流量。而 OpenDAL 通过服务器中转下载，消耗服务器流量。
+2. 项目的支持对于异常情况可能会有更多的调试信息，方便排查问题。
+3. OpenDAL 项目本身采用 Rust 编写，性能更好。