后端移动文件夹，前端在docker中编译，进一步减小镜像大小
This commit is contained in:
buyfakett
@@ -1,4 +0,0 @@
|
||||
# @Time : 2023/8/11 20:06
|
||||
# @Author : Lan
|
||||
# @File : __init__.py.py
|
||||
# @Software: PyCharm
|
||||
@@ -1,15 +0,0 @@
|
||||
# @Time : 2023/8/14 11:48
|
||||
# @Author : Lan
|
||||
# @File : response.py
|
||||
# @Software: PyCharm
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
from pydantic.v1.generics import GenericModel
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class APIResponse(GenericModel, Generic[T]):
|
||||
code: int = 200
|
||||
message: str = 'ok'
|
||||
detail: T
|
||||
@@ -1,85 +0,0 @@
|
||||
# @Time : 2023/8/15 09:51
|
||||
# @Author : Lan
|
||||
# @File : settings.py
|
||||
# @Software: PyCharm
|
||||
from pathlib import Path
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
data_root = BASE_DIR / 'data'
|
||||
if not data_root.exists():
|
||||
data_root.mkdir(parents=True, exist_ok=True)
|
||||
env_path = data_root / '.env2'
|
||||
default_value = {
|
||||
'file_storage': 'local',
|
||||
'name': '文件快递柜-FileCodeBox',
|
||||
'description': '开箱即用的文件快传系统',
|
||||
'keywords': 'FileCodeBox, 文件快递柜, 口令传送箱, 匿名口令分享文本, 文件',
|
||||
'max_save_seconds': 0,
|
||||
's3_access_key_id': '',
|
||||
's3_secret_access_key': '',
|
||||
's3_bucket_name': '',
|
||||
's3_endpoint_url': '',
|
||||
's3_region_name': 'auto',
|
||||
's3_signature_version': 's3v2',
|
||||
's3_hostname': '',
|
||||
's3_proxy': 0,
|
||||
'aws_session_token': '',
|
||||
'onedrive_domain': '',
|
||||
'onedrive_client_id': '',
|
||||
'onedrive_username': '',
|
||||
'onedrive_password': '',
|
||||
'onedrive_root_path': 'filebox_storage',
|
||||
'onedrive_proxy': 0,
|
||||
'admin_token': 'FileCodeBox2023',
|
||||
'openUpload': 1,
|
||||
'uploadSize': 1024 * 1024 * 10,
|
||||
'uploadMinute': 1,
|
||||
'opacity': 0.9,
|
||||
'background': '',
|
||||
'uploadCount': 10,
|
||||
'errorMinute': 1,
|
||||
'errorCount': 1,
|
||||
'port': 12345,
|
||||
}
|
||||
|
||||
|
||||
class Settings:
|
||||
__instance = None
|
||||
|
||||
def __init__(self):
|
||||
# 读取.env
|
||||
if not (env_path).exists():
|
||||
with open(env_path, 'w', encoding='utf-8') as f:
|
||||
for key, value in default_value.items():
|
||||
f.write(f'{key}={value}\n')
|
||||
# 更新default_value
|
||||
with open(env_path, 'r', encoding='utf-8') as f:
|
||||
for line in f.readlines():
|
||||
key, value = line.strip().split('=', maxsplit=1)
|
||||
# 将字符串转换为原本的类型
|
||||
if not key.startswith('opendal_') and isinstance(default_value[key], int):
|
||||
value = int(value)
|
||||
default_value[key] = value
|
||||
|
||||
# 更新self
|
||||
for key, value in default_value.items():
|
||||
self.__setattr__(key, value)
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not cls.__instance:
|
||||
cls.__instance = super(Settings, cls).__new__(cls, *args, **kwargs)
|
||||
return cls.__instance
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
if not key.startswith('opendal_') and type(value) == str and value.isnumeric():
|
||||
value = int(value)
|
||||
self.__dict__[key] = value
|
||||
with open(env_path, 'w', encoding='utf-8') as f:
|
||||
for key, value in self.__dict__.items():
|
||||
f.write(f'{key}={value}\n')
|
||||
|
||||
def items(self):
|
||||
return self.__dict__.items()
|
||||
|
||||
|
||||
settings = Settings()
|
||||
-291
@@ -1,291 +0,0 @@
|
||||
# @Time : 2023/8/11 20:06
|
||||
# @Author : Lan
|
||||
# @File : storage.py
|
||||
# @Software: PyCharm
|
||||
import aiohttp
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import datetime
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import aioboto3
|
||||
from botocore.config import Config
|
||||
from fastapi import HTTPException, Response, UploadFile
|
||||
from core.response import APIResponse
|
||||
from core.settings import data_root, settings
|
||||
from apps.base.models import FileCodes
|
||||
from core.utils import get_file_url
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
|
||||
class FileStorageInterface:
|
||||
|
||||
async def save_file(self, file: UploadFile, save_path: str):
|
||||
"""
|
||||
保存文件
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
"""
|
||||
删除文件
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
"""
|
||||
获取文件分享的url
|
||||
|
||||
如果服务不支持直接访问文件，可以通过服务器中转下载。
|
||||
此时，此方法可以调用 utils.py 中的 `get_file_url` 方法，获取服务器中转下载的url
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
"""
|
||||
获取文件响应
|
||||
|
||||
如果服务不支持直接访问文件，则需要实现该方法，返回文件响应
|
||||
其余情况，可以不实现该方法
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SystemFileStorage(FileStorageInterface):
|
||||
def __init__(self):
|
||||
self.chunk_size = 256 * 1024
|
||||
self.root_path = data_root
|
||||
|
||||
def _save(self, file, save_path):
|
||||
with open(save_path, 'wb') as f:
|
||||
chunk = file.read(self.chunk_size)
|
||||
while chunk:
|
||||
f.write(chunk)
|
||||
chunk = file.read(self.chunk_size)
|
||||
|
||||
async def save_file(self, file: UploadFile, save_path: str):
|
||||
save_path = self.root_path / save_path
|
||||
if not save_path.parent.exists():
|
||||
save_path.parent.mkdir(parents=True)
|
||||
await asyncio.to_thread(self._save, file.file, save_path)
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
save_path = self.root_path / await file_code.get_file_path()
|
||||
if save_path.exists():
|
||||
save_path.unlink()
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
return await get_file_url(file_code.code)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
file_path = file_storage.root_path / await file_code.get_file_path()
|
||||
if not file_path.exists():
|
||||
return APIResponse(code=404, detail='文件已过期删除')
|
||||
return FileResponse(file_path, filename=file_code.prefix + file_code.suffix)
|
||||
|
||||
|
||||
class S3FileStorage(FileStorageInterface):
|
||||
def __init__(self):
|
||||
self.access_key_id = settings.s3_access_key_id
|
||||
self.secret_access_key = settings.s3_secret_access_key
|
||||
self.bucket_name = settings.s3_bucket_name
|
||||
self.s3_hostname = settings.s3_hostname
|
||||
self.region_name = settings.s3_region_name
|
||||
self.signature_version = settings.s3_signature_version
|
||||
self.endpoint_url = settings.s3_endpoint_url or f'https://{self.s3_hostname}'
|
||||
self.aws_session_token = settings.aws_session_token
|
||||
self.proxy = settings.s3_proxy
|
||||
self.session = aioboto3.Session(aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key)
|
||||
if not settings.s3_endpoint_url:
|
||||
self.endpoint_url = f'https://{self.s3_hostname}'
|
||||
else:
|
||||
# 如果提供了 s3_endpoint_url，则优先使用它
|
||||
self.endpoint_url = settings.s3_endpoint_url
|
||||
|
||||
async def save_file(self, file: UploadFile, save_path: str):
|
||||
async with self.session.client("s3", endpoint_url=self.endpoint_url, aws_session_token=self.aws_session_token, region_name=self.region_name,
|
||||
config=Config(signature_version=self.signature_version)) as s3:
|
||||
await s3.put_object(Bucket=self.bucket_name, Key=save_path, Body=await file.read(), ContentType=file.content_type)
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
async with self.session.client("s3", endpoint_url=self.endpoint_url, region_name=self.region_name, config=Config(signature_version=self.signature_version)) as s3:
|
||||
await s3.delete_object(Bucket=self.bucket_name, Key=await file_code.get_file_path())
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
async with self.session.client("s3", endpoint_url=self.endpoint_url, region_name=self.region_name, config=Config(signature_version=self.signature_version)) as s3:
|
||||
link = await s3.generate_presigned_url('get_object', Params={'Bucket': self.bucket_name, 'Key': await file_code.get_file_path()}, ExpiresIn=3600)
|
||||
tmp = io.BytesIO()
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(link) as resp:
|
||||
tmp.write(await resp.read())
|
||||
tmp.seek(0)
|
||||
content = tmp.read()
|
||||
tmp.close()
|
||||
return Response(content, media_type="application/octet-stream", headers={"Content-Disposition": f'attachment; filename="{filename.encode("utf-8").decode("latin-1")}"'})
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail='服务代理下载异常，请稍后再试')
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
if file_code.prefix == '文本分享':
|
||||
return file_code.text
|
||||
if self.proxy:
|
||||
return await get_file_url(file_code.code)
|
||||
else:
|
||||
async with self.session.client("s3", endpoint_url=self.endpoint_url, region_name=self.region_name, config=Config(signature_version=self.signature_version)) as s3:
|
||||
result = await s3.generate_presigned_url('get_object', Params={'Bucket': self.bucket_name, 'Key': await file_code.get_file_path()}, ExpiresIn=3600)
|
||||
return result
|
||||
|
||||
|
||||
class OneDriveFileStorage(FileStorageInterface):
|
||||
def __init__(self):
|
||||
try:
|
||||
import msal
|
||||
from office365.graph_client import GraphClient
|
||||
from office365.runtime.client_request_exception import ClientRequestException
|
||||
except ImportError:
|
||||
raise ImportError('请先安装`msal`和`Office365-REST-Python-Client`')
|
||||
self.msal = msal
|
||||
self.domain = settings.onedrive_domain
|
||||
self.client_id = settings.onedrive_client_id
|
||||
self.username = settings.onedrive_username
|
||||
self.password = settings.onedrive_password
|
||||
self.proxy = settings.onedrive_proxy
|
||||
self._ClientRequestException = ClientRequestException
|
||||
|
||||
try:
|
||||
client = GraphClient(self.acquire_token_pwd)
|
||||
self.root_path = client.me.drive.root.get_by_path(settings.onedrive_root_path).get().execute_query()
|
||||
except ClientRequestException as e:
|
||||
if e.code == 'itemNotFound':
|
||||
client.me.drive.root.create_folder(settings.onedrive_root_path)
|
||||
self.root_path = client.me.drive.root.get_by_path(settings.onedrive_root_path).get().execute_query()
|
||||
else:
|
||||
raise e
|
||||
except Exception as e:
|
||||
raise Exception('OneDrive验证失败，请检查配置是否正确\n' + str(e))
|
||||
|
||||
def acquire_token_pwd(self):
|
||||
authority_url = f'https://login.microsoftonline.com/{self.domain}'
|
||||
app = self.msal.PublicClientApplication(
|
||||
authority=authority_url,
|
||||
client_id=self.client_id
|
||||
)
|
||||
result = app.acquire_token_by_username_password(username=self.username,
|
||||
password=self.password,
|
||||
scopes=['https://graph.microsoft.com/.default'])
|
||||
return result
|
||||
|
||||
def _get_path_str(self, path):
|
||||
if isinstance(path, str):
|
||||
path = path.replace('\\', '/').replace('//', '/').split('/')
|
||||
elif isinstance(path, Path):
|
||||
path = str(path).replace('\\', '/').replace('//', '/').split('/')
|
||||
else:
|
||||
raise TypeError('path must be str or Path')
|
||||
path[-1] = path[-1].split('.')[0]
|
||||
return '/'.join(path)
|
||||
|
||||
def _save(self, file, save_path):
|
||||
content = file.file.read()
|
||||
name = file.filename
|
||||
path = self._get_path_str(save_path)
|
||||
self.root_path.get_by_path(path).upload(name, content).execute_query()
|
||||
|
||||
async def save_file(self, file: UploadFile, save_path: str):
|
||||
await asyncio.to_thread(self._save, file, save_path)
|
||||
|
||||
def _delete(self, save_path):
|
||||
path = self._get_path_str(save_path)
|
||||
try:
|
||||
self.root_path.get_by_path(path).delete_object().execute_query()
|
||||
except self._ClientRequestException as e:
|
||||
if e.code == 'itemNotFound':
|
||||
pass
|
||||
else:
|
||||
raise e
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
await asyncio.to_thread(self._delete, await file_code.get_file_path())
|
||||
|
||||
def _convert_link_to_download_link(self, link):
|
||||
p1 = re.search(r'https:\/\/(.+)\.sharepoint\.com', link).group(1)
|
||||
p2 = re.search(r'personal\/(.+)\/', link).group(1)
|
||||
p3 = re.search(rf'{p2}\/(.+)', link).group(1)
|
||||
return f'https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}'
|
||||
|
||||
def _get_file_url(self, save_path, name):
|
||||
path = self._get_path_str(save_path)
|
||||
remote_file = self.root_path.get_by_path(path + '/' + name)
|
||||
expiration_datetime = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(hours=1)
|
||||
expiration_datetime = expiration_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
premission = remote_file.create_link("view", "anonymous", expiration_datetime=expiration_datetime).execute_query()
|
||||
return self._convert_link_to_download_link(premission.link.webUrl)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
link = await asyncio.to_thread(self._get_file_url, await file_code.get_file_path(), filename)
|
||||
tmp = io.BytesIO()
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(link) as resp:
|
||||
tmp.write(await resp.read())
|
||||
tmp.seek(0)
|
||||
content = tmp.read()
|
||||
tmp.close()
|
||||
return Response(content, media_type="application/octet-stream", headers={"Content-Disposition": f'attachment; filename="{filename.encode("utf-8").decode("latin-1")}"'})
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail='服务代理下载异常，请稍后再试')
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
if self.proxy:
|
||||
return await get_file_url(file_code.code)
|
||||
else:
|
||||
return await asyncio.to_thread(self._get_file_url, await file_code.get_file_path(), f'{file_code.prefix}{file_code.suffix}')
|
||||
|
||||
|
||||
class OpenDALFileStorage(FileStorageInterface):
|
||||
def __init__(self):
|
||||
try:
|
||||
import opendal
|
||||
except ImportError:
|
||||
raise ImportError('请先安装 `opendal`, 例如: "pip install opendal"')
|
||||
self.service = settings.opendal_scheme
|
||||
service_settings = {}
|
||||
for key, value in settings.items():
|
||||
if key.startswith('opendal_' + self.service):
|
||||
setting_name = key.split('_', 2)[2]
|
||||
service_settings[setting_name] = value
|
||||
self.operator = opendal.AsyncOperator(settings.opendal_scheme, **service_settings)
|
||||
|
||||
async def save_file(self, file: UploadFile, save_path: str):
|
||||
await self.operator.write(save_path, file.file.read())
|
||||
|
||||
async def delete_file(self, file_code: FileCodes):
|
||||
await self.operator.delete(await file_code.get_file_path())
|
||||
|
||||
async def get_file_url(self, file_code: FileCodes):
|
||||
return await get_file_url(file_code.code)
|
||||
|
||||
async def get_file_response(self, file_code: FileCodes):
|
||||
try:
|
||||
filename = file_code.prefix + file_code.suffix
|
||||
content = await self.operator.read(await file_code.get_file_path())
|
||||
headers = {
|
||||
"Content-Disposition": f'attachment; filename="{filename}"'
|
||||
}
|
||||
return Response(content, headers=headers, media_type="application/octet-stream")
|
||||
except Exception as e:
|
||||
print(e, file=sys.stderr)
|
||||
raise HTTPException(status_code=404, detail="文件已过期删除")
|
||||
|
||||
|
||||
storages = {
|
||||
'local': SystemFileStorage,
|
||||
's3': S3FileStorage,
|
||||
'onedrive': OneDriveFileStorage,
|
||||
'opendal': OpenDALFileStorage,
|
||||
}
|
||||
file_storage: FileStorageInterface = storages[settings.file_storage]()
|
||||
@@ -1,27 +0,0 @@
|
||||
# @Time : 2023/8/15 22:00
|
||||
# @Author : Lan
|
||||
# @File : tasks.py
|
||||
# @Software: PyCharm
|
||||
import asyncio
|
||||
|
||||
from tortoise.expressions import Q
|
||||
|
||||
from apps.base.models import FileCodes
|
||||
from apps.base.utils import error_ip_limit, upload_ip_limit
|
||||
from core.storage import file_storage
|
||||
from core.utils import get_now
|
||||
|
||||
|
||||
async def delete_expire_files():
|
||||
while True:
|
||||
try:
|
||||
await error_ip_limit.remove_expired_ip()
|
||||
await upload_ip_limit.remove_expired_ip()
|
||||
expire_data = await FileCodes.filter(Q(expired_at__lt=await get_now()) | Q(expired_count=0)).all()
|
||||
for exp in expire_data:
|
||||
await file_storage.delete_file(exp)
|
||||
await exp.delete()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
await asyncio.sleep(600)
|
||||
@@ -1,95 +0,0 @@
|
||||
# @Time : 2023/8/13 19:54
|
||||
# @Author : Lan
|
||||
# @File : utils.py
|
||||
# @Software: PyCharm
|
||||
import datetime
|
||||
import hashlib
|
||||
import random
|
||||
import string
|
||||
import time
|
||||
|
||||
from apps.base.depends import IPRateLimit
|
||||
|
||||
|
||||
async def get_random_num():
|
||||
"""
|
||||
获取随机数
|
||||
:return:
|
||||
"""
|
||||
return random.randint(10000, 99999)
|
||||
|
||||
|
||||
r_s = string.ascii_uppercase + string.digits
|
||||
|
||||
|
||||
async def get_random_string():
|
||||
"""
|
||||
获取随机字符串
|
||||
:return:
|
||||
"""
|
||||
return ''.join(random.choice(r_s) for _ in range(5))
|
||||
|
||||
|
||||
async def get_now():
|
||||
"""
|
||||
获取当前时间
|
||||
:return:
|
||||
"""
|
||||
return datetime.datetime.now(
|
||||
datetime.timezone(datetime.timedelta(hours=8))
|
||||
)
|
||||
|
||||
|
||||
async def get_select_token(code: str):
|
||||
"""
|
||||
获取下载token
|
||||
:param code:
|
||||
:return:
|
||||
"""
|
||||
token = "123456"
|
||||
return hashlib.sha256(f"{code}{int(time.time() / 1000)}000{token}".encode()).hexdigest()
|
||||
|
||||
|
||||
async def get_file_url(code: str):
|
||||
"""
|
||||
对于需要通过服务器中转下载的服务，获取文件下载地址
|
||||
:param code:
|
||||
:return:
|
||||
"""
|
||||
return f'/share/download?key={await get_select_token(code)}&code={code}'
|
||||
|
||||
|
||||
async def max_save_times_desc(max_save_seconds: int):
|
||||
"""
|
||||
获取最大保存时间的描述
|
||||
:param max_save_seconds:
|
||||
:return:
|
||||
"""
|
||||
|
||||
def gen_desc_zh(value: int, desc: str):
|
||||
if value > 0:
|
||||
return f'{value}{desc}'
|
||||
else:
|
||||
return ''
|
||||
|
||||
def gen_desc_en(value: int, desc: str):
|
||||
if value > 0:
|
||||
ret = f'{value} {desc}'
|
||||
if value > 1:
|
||||
ret += 's'
|
||||
ret += ' '
|
||||
return ret
|
||||
else:
|
||||
return ''
|
||||
|
||||
max_timedelta = datetime.timedelta(seconds=max_save_seconds)
|
||||
desc_zh, desc_en = '最长保存时间：', 'Max save time: '
|
||||
desc_zh += gen_desc_zh(max_timedelta.days, '天')
|
||||
desc_en += gen_desc_en(max_timedelta.days, 'day')
|
||||
desc_zh += gen_desc_zh(max_timedelta.seconds // 3600, '小时')
|
||||
desc_en += gen_desc_en(max_timedelta.seconds // 3600, 'hour')
|
||||
desc_zh += gen_desc_zh(max_timedelta.seconds % 3600 // 60, '分钟')
|
||||
desc_en += gen_desc_en(max_timedelta.seconds % 3600 // 60, 'minute')
|
||||
desc_zh += gen_desc_zh(max_timedelta.seconds % 60, '秒')
|
||||
desc_en += gen_desc_en(max_timedelta.seconds % 60, 'second')
|
||||
return desc_zh, desc_en
|
||||
Reference in New Issue
Block a user