diff --git a/.gitignore b/.gitignore index 9bbd9c69c..5111785c9 100644 --- a/.gitignore +++ b/.gitignore @@ -162,7 +162,6 @@ plugins/my_* plugins/l2tp plugins/openlitespeed plugins/tamper_proof -plugins/msonedrive plugins/cryptocurrency_trade plugins/gdrive plugins/mtproxy diff --git a/plugins/msonedrive/class/msodclient.py b/plugins/msonedrive/class/msodclient.py new file mode 100644 index 000000000..d17efab9c --- /dev/null +++ b/plugins/msonedrive/class/msodclient.py @@ -0,0 +1,820 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import re +import json +import io + +sys.path.append(os.getcwd() + "/class/core") +import mw + +import oauthlib +import requests +import datetime +from requests_oauthlib import OAuth2Session + +DEBUG = False + + +def setDebug(d=False): + DEBUG = d + + +class UnauthorizedError(Exception): + pass + + +class ObjectNotFoundError(Exception): + pass + + +class msodclient: + + plugin_dir = '' + server_dir = '' + credential_file = 'credentials.json' + user_conf = "user.conf" + token_file = 'token.pickle' + + def __init__(self, plugin_dir, server_dir): + self.plugin_dir = plugin_dir + self.server_dir = server_dir + self.load() + + def setDebug(self, d=False): + DEBUG = d + + def load(self): + credential_path = os.path.join(self.plugin_dir, self.credential_file) + credential = json.loads(mw.readFile(credential_path)) + # print(credential) + self.credential = credential["onedrive-international"] + + self.authorize_url = '{0}{1}'.format( + self.credential['authority'], + self.credential['authorize_endpoint']) + self.token_url = '{0}{1}'.format( + self.credential['authority'], + self.credential['token_endpoint']) + + self.token_path = os.path.join(self.server_dir, self.token_file) + self.root_uri = self.credential["api_uri"] + "/me/drive/root" + + self.backup_path = 'backup' + + def store_token(self, token): + """存储token""" + enstr = mw.enDoubleCrypt('msodc', json.dumps(token)) + mw.writeFile(self.token_path, enstr) + return True + + def get_store_token(self): + rdata = mw.readFile(self.token_path) + destr = mw.deDoubleCrypt('msodc', rdata) + return json.loads(destr) + + def clear_token(self): + """清除token记录""" + try: + if os.path.isfile(self.token_path): + os.remove(self.token_path) + except: + if DEBUG: + print("清除token失败。") + + def refresh_token(self, origin_token): + """刷新token""" + + os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = '1' + os.environ['OAUTHLIB_IGNORE_SCOPE_CHANGE'] = '1' + refresh_token = origin_token["refresh_token"] + aad_auth = OAuth2Session( + self.credential["client_id"], + scope=self.credential["scopes"], + redirect_uri=self.credential["redirect_uri"]) + + new_token = aad_auth.refresh_token( + self.token_url, + refresh_token=refresh_token, + client_id=self.credential["client_id"], + client_secret=self.credential["client_secret"]) + return new_token + + def get_token_from_authorized_url(self, authorized_url, expected_state=None): + """通过授权编码获取访问token""" + + # 忽略token scope与已请求的scope不一致 + os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = '1' + os.environ['OAUTHLIB_IGNORE_SCOPE_CHANGE'] = '1' + aad_auth = OAuth2Session(self.credential["client_id"], + state=expected_state, + scope=self.credential['scopes'], + redirect_uri=self.credential['redirect_uri']) + + token = aad_auth.fetch_token( + self.token_url, + client_secret=self.credential["client_secret"], + authorization_response=authorized_url) + + return token + + def get_token(self): + token = self.get_store_token() + now = time.time() + + expire_time = token["expires_at"] - 300 + if now >= expire_time: + new_token = self.refresh_token(token) + self.store_token(new_token) + return new_token + + return token + + def get_sign_in_url(self): + """生成签名地址""" + + # Initialize the OAuth client + aad_auth = OAuth2Session(self.credential["client_id"], + scope=self.credential['scopes'], + redirect_uri=self.credential['redirect_uri']) + + sign_in_url, state = aad_auth.authorization_url(self.authorize_url, + prompt='login') + + return sign_in_url, state + + def get_authorized_header(self): + token_obj = self.get_token() + token = token_obj["access_token"] + header = { + "Authorization": "Bearer " + token, + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/67.0.3396.99 Safari/537.36' + } + return header + + def get_user_from_ms(self): + """查询用户信息""" + try: + headers = self.get_authorized_header() + user_api_base = self.credential["api_uri"] + "/me" + # select_user_info_uri = self.build_uri(base=user_api_base) + response = requests.get(user_api_base, headers=headers) + if DEBUG: + print("Debug get user:") + print(response.status_code) + print(response.text) + if response.status_code == 200: + response_data = response.json() + user_principal_name = response_data["userPrincipalName"] + return user_principal_name + except oauthlib.oauth2.rfc6749.errors.InvalidGrantError: + self.clear_auth() + if DEBUG: + print("用户授权已过期。") + return None + + def clear_auth(self): + self.clear_token() + self.clear_user() + + def clear_user(self): + try: + # 清空user + path = os.path.join(self.server_dir, self.user_conf) + if os.path.isfile(path): + os.remove(path) + except: + if DEBUG: + print("清除user失败。") + + def store_user(self): + """更新并存储用户信息""" + user = self.get_user_from_ms() + if user: + path = os.path.join(self.server_dir, self.user_conf) + mw.writeFile(path, user) + else: + raise RuntimeError("无法获取用户信息。") + + # --------------------- 文件操作功能 ---------------------- + + # 取目录路径 + def get_path(self, path): + sep = ":" + if path == '/': + path = '' + if path[-1:] == '/': + path = path[:-1] + if path[:1] != "/" and path[:1] != sep: + path = "/" + path + if path == '/': + path = '' + # if path[:1] != sep: + # path = sep + path + try: + from urllib.parse import quote + except: + from urllib import quote + # path = quote(path) + + return path.replace('//', '/') + + def build_uri(self, path="", operate=None, base=None): + """构建请求URL + + API请求URI格式参考: + https://graph.microsoft.com/v1.0/me/drive/root:/bt_backup/:content + --------------------------------------------- ---------- -------- + base path operate + 各部分之间用“：”连接。 + :param path 子资源路径 + :param operate 对文件进行的操作，比如content,children + :return 请求url + """ + + if base is None: + base = self.root_uri + path = self.get_path(path) + sep = ":" + if operate: + if operate[:1] != "/": + operate = "/" + operate + + if path: + uri = base + sep + path + if operate: + uri += sep + operate + else: + uri = base + if operate: + uri += operate + + return uri + + def get_list(self, path="/"): + """获取存储空间中的所有文件对象""" + + list_uri = self.build_uri(path, operate="/children") + if DEBUG: + print("List uri:") + print(list_uri) + + data = [] + response = requests.get(list_uri, headers=self.get_authorized_header()) + status_code = response.status_code + if status_code == 200: + if DEBUG: + print("DEBUG:") + print(response.json()) + response_data = response.json() + drive_items = response_data["value"] + + for item in drive_items: + tmp = {} + tmp['name'] = item["name"] + tmp['size'] = item["size"] + if "folder" in item: + # print("{} is folder:".format(item["name"])) + # print(item["folder"]) + tmp["type"] = None + tmp['download'] = "" + if "file" in item: + tmp["type"] = "File" + tmp['download'] = item["@microsoft.graph.downloadUrl"] + # print("{} is file:".format(item["name"])) + # print(item["file"]) + + formats = ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"] + t = None + for time_format in formats: + try: + t = datetime.datetime.strptime( + item["lastModifiedDateTime"], time_format) + break + except: + continue + t += datetime.timedelta(hours=8) + ts = int( + (time.mktime(t.timetuple()) + t.microsecond / 1000000.0)) + tmp['time'] = ts + data.append(tmp) + + mlist = {'path': path, 'list': data} + return mlist + + def get_object(self, object_name): + """查询对象信息""" + try: + get_uri = self.build_uri(path=object_name) + if DEBUG: + print("Get uri:") + print(get_uri) + response = requests.get(get_uri, + headers=self.get_authorized_header()) + if response.status_code in [200]: + response_data = response.json() + if DEBUG: + print("Object info:") + print(response_data) + return response_data + if response.status_code == 404: + if DEBUG: + print("对象不存在。") + if DEBUG: + print("Get Object debug:") + print(response.status_code) + print(response.text) + except Exception as e: + if DEBUG: + print("Get object has excepiton:") + print(e) + return None + + def is_folder(self, obj): + if "folder" in obj: + return True + return False + + def delete_object_by_os(self, object_name): + """删除对象 + + :param object_name: + :return: True 删除成功 + 其他 删除失败 + """ + obj = self.get_object(object_name) + if obj is None: + if DEBUG: + print("对象不存在，删除操作未执行。") + return True + if self.is_folder(obj): + child_count = obj["folder"]["childCount"] + if child_count > 0: + if DEBUG: + print("文件夹不是空文件夹无法删除。") + return False + + headers = self.get_authorized_header() + delete_uri = self.build_uri(object_name) + response = requests.delete(delete_uri, headers=headers) + if response.status_code == 204: + if DEBUG: + print("对象: {} 已被删除。".format(object_name)) + return True + return False + + def delete_object(self, object_name, retries=2): + """删除对象 + + :param object_name: + :param retries: 重试次数，默认2次 + :return: True 删除成功 + 其他 删除失败 + """ + + try: + return self.delete_object_by_os(object_name) + except Exception as e: + print("删除文件异常：") + print(e) + + # 重试 + if retries > 0: + print("重新尝试删除文件{}...".format(object_name)) + return self.delete_object( + object_name, + retries=retries - 1) + return False + + def build_object_name(self, data_type, file_name): + """根据数据类型构建对象存储名称 + + :param data_type: + :param file_name: + :return: + """ + + import re + + prefix_dict = { + "site": "web", + "database": "db", + "path": "path", + } + file_regx = prefix_dict.get(data_type) + "_(.+)_20\d+_\d+(?:\.|_)" + sub_search = re.search(file_regx, file_name) + sub_path_name = "" + if sub_search: + sub_path_name = sub_search.groups()[0] + sub_path_name += '/' + + # 构建OS存储路径 + object_name = self.backup_path + '/' + \ + data_type + '/' + \ + sub_path_name + \ + file_name + + if object_name[:1] == "/": + object_name = object_name[1:] + + return object_name + + def delete_file(self, file_name, data_type=None): + """删除文件 + + 根据传入的文件名称和文件数据类型构建对象名称，再删除 + :param file_name: + :param data_type: 数据类型 site/database/path + :return: True 删除成功 + 其他 删除失败 + """ + + object_name = self.build_object_name(data_type, file_name) + return self.delete_object(object_name) + + def create_dir_by_step(self, parent_folder, sub_folder): + create_uri = self.build_uri(path=parent_folder, operate="/children") + + if DEBUG: + print("Create dir uri:") + print(create_uri) + post_data = { + "name": sub_folder, + "folder": {"@odata.type": "microsoft.graph.folder"}, + "@microsoft.graph.conflictBehavior": "fail" + } + + headers = self.get_authorized_header() + headers.update({"Content-type": "application/json"}) + response = requests.post(create_uri, headers=headers, json=post_data) + if response.status_code in [201, 409]: + if DEBUG: + if response.status_code == 409: + print("目录：{} 已经存在。".format(sub_folder)) + return True + else: + if DEBUG: + print("目录：{} 创建失败：".format(sub_folder)) + print(response.status_code) + print(response.text) + return False + + def create_dir(self, dir_name): + """创建远程目录 + + # API 请求结构 + # POST /me/drive/root/children + # or + # POST /me/drive/root:/bt_backup/:/children + # Content - Type: application / json + + # { + # "name": "New Folder", + # "folder": {}, + # "@microsoft.graph.conflictBehavior": "rename" + # } + + # Response: status code == 201 新创建/ 409 已存在 + # @microsoft.graph.conflictBehavior: fail/rename/replace + + :param dir_name: 目录名称 + :param parent_id: 父目录ID + :return: True/False + """ + + dir_name = self.get_path(dir_name.strip()) + onedrive_business_reserved = r"[\*<>?:|#%]" + if re.search(onedrive_business_reserved, dir_name) \ + or dir_name[-1] == "." or dir_name[:1] == "~": + if DEBUG: + print("文件夹名称包含非法字符。") + return False + + parent_folder = self.get_path(os.path.split(dir_name)[0]) + sub_folder = os.path.split(dir_name)[1] + + # print("create_dir:", dir_name) + obj = self.get_object(dir_name) + # 判断对象是否存在 + if obj is None: + if not self.create_dir_by_step(parent_folder, sub_folder): + + # 兼容OneDrive 商业版文件夹创建 + folder_array = dir_name.split("/") + parent_folder = self.get_path(folder_array[0]) + for i in range(1, len(folder_array)): + sub_folder = folder_array[i] + if DEBUG: + print("Parent folder: {}".format(parent_folder)) + print("Sub folder: {}".format(sub_folder)) + if self.create_dir_by_step(parent_folder, sub_folder): + parent_folder += "/" + folder_array[i] + else: + return False + return True + else: + if self.is_folder(obj): + if DEBUG: + print("文件夹已存在。") + return True + + def resumable_upload(self, + local_file_name, + object_name=None, + progress_callback=None, + progress_file_name=None, + multipart_threshold=1024 * 1024 * 2, + part_size=1024 * 1024 * 5, + store_dir="/tmp", + auto_cancel=True, + retries=5, + ): + """断点续传 + + :param local_file_name: 本地文件名称 + :param object_name: 指定OS中存储的对象名称 + :param part_size: 指定分片上传的每个分片的大小。必须是320*1024的整数倍。 + :param multipart_threshold: 文件长度大于该值时，则用分片上传。 + :param progress_callback: 进度回调函数，默认是把进度信息输出到标准输出。 + :param progress_file_name: 进度信息保存文件，进度格式参见[report_progress] + :param store_dir: 上传分片存储目录, 默认/tmp。 + :param auto_cancel: 当备份失败是否自动取消上传记录 + :param retries: 上传重试次数 + :return: True上传成功/False or None上传失败 + """ + + try: + file_size_separation_value = 4 * 1024 * 1024 + if part_size % 320 != 0: + if DEBUG: + print("Part size 必须是320的整数倍。") + return False + + if object_name is None: + temp_file_name = os.path.split(local_file_name)[1] + object_name = os.path.join(self.backup_path, temp_file_name) + + # if progress_file_name: + # os.environ[PROGRESS_FILE_NAME] = progress_file_name + # progress_callback = report_progress + + print("|-正在上传到 {}...".format(object_name)) + dir_name = os.path.split(object_name)[0] + if not self.create_dir(dir_name): + if DEBUG: + print("目录创建失败！") + return False + + local_file_size = os.path.getsize(local_file_name) + # if local_file_size < file_size_separation_value: + if False: + # 小文件上传 + upload_uri = self.build_uri(path=object_name, + operate="/content") + if DEBUG: + print("Upload uri:") + print(upload_uri) + headers = self.get_authorized_header() + # headers.update({ + # "Content-Type": "application/octet-stream" + # }) + # files = {"file": (object_name, open(local_file_name, "rb"))} + file_data = open(local_file_name, "rb") + response = requests.put(upload_uri, + headers=headers, + data=file_data) + if DEBUG: + print("status code:") + print(response.status_code) + # print(response.text) + if response.status_code in [201, 200]: + if DEBUG: + print("文件上传成功！") + return True + else: + # 大文件上传 + + # 1. 创建上传session + create_session_uri = self.build_uri( + path=object_name, + operate="createUploadSession") + headers = self.get_authorized_header() + response = requests.post(create_session_uri, headers=headers) + if response.status_code == 200: + response_data = response.json() + upload_url = response_data["uploadUrl"] + expiration_date_time = response_data["expirationDateTime"] + + if DEBUG: + print("上传session已建立。") + print("Upload url: {}".format(upload_url)) + print("Expiration datetime: {}".format( + expiration_date_time)) + + # 2. 分片上传文件 + requests.adapters.DEFAULT_RETRIES = 1 + session = requests.session() + session.keep_alive = False + + # 开始分片上传 + import math + parts = int(math.ceil(local_file_size / part_size)) + for i in range(parts): + if DEBUG: + if i == parts - 1: + num = "最后" + else: + num = "第{}".format(i + 1) + print("正在上传{}部分...".format(num)) + + upload_range_start = i * part_size + upload_range_end = min(upload_range_start + part_size, + local_file_size) + content_length = upload_range_end - upload_range_start + + headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/67.0.3396.99 Safari/537.36' + } + # 开发记录 + # Content-Range和标准的http请求头中的Range作用有所不同 + # Content-Range是OneDrive自定义的分片上传标识，格式也不一样 + headers.update({ + "Content-Length": repr(content_length), + "Content-Range": "bytes {}-{}/{}".format( + upload_range_start, + upload_range_end - 1, + local_file_size), + "Content-Type": "application/octet-stream" + }) + + if DEBUG: + print("Headers:") + print(headers) + + '''# TODO 优化read的读取占用内存''' + f = io.open(local_file_name, "rb") + f.seek(upload_range_start) + upload_data = f.read(content_length) + sub_response = session.put(upload_url, + headers=headers, + data=upload_data) + + expected_status_code = [200, 201, 202] + if sub_response.status_code in expected_status_code: + if DEBUG: + print("Response status code: {}, " + "bytes {}-{} 已上传成功。".format( + sub_response.status_code, + upload_range_start, + upload_range_end - 1) + ) + print(sub_response.text) + if sub_response.status_code in [200, 201]: + if DEBUG: + print("文件 {} 上传成功。".format(object_name)) + return True + else: + print(sub_response.status_code) + print(sub_response.text) + _error_msg = "Bytes {}-{} 分片上传失败。".format( + upload_range_start, + upload_range_end + ) + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += _error_msg + raise RuntimeError(_error_msg) + + time.sleep(0.5) + else: + raise RuntimeError("session创建失败。") + + except UnauthorizedError as e: + _error_msg = str(e) + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += _error_msg + print(_error_msg) + return False + except Exception as e: + print("文件上传出现错误：") + print(e) + + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += "文件{}上传出现错误：{}".format(object_name, str(e)) + + try: + if upload_url: + if DEBUG: + print("正在清理上传session.") + session.delete(upload_url) + except: + pass + finally: + try: + f.close() + except: + pass + try: + session.close() + except: + pass + + # 重试断点续传 + if retries > 0: + print("重试上传文件....") + return self.resumable_upload( + local_file_name, + object_name=object_name, + store_dir=store_dir, + part_size=part_size, + multipart_threshold=multipart_threshold, + progress_callback=progress_callback, + progress_file_name=progress_file_name, + retries=retries - 1, + ) + else: + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += "文件{}上传失败。".format(object_name) + return False + + def upload_abs_file(self, file_name, remote_dir, *args, **kwargs): + """按照数据类型上传文件 + + :param file_name: 上传文件名称 + :param data_type: 数据类型 site/database/path + :return: True/False + """ + try: + import re + # 根据数据类型提取子分类名称 + # 比如data_type=database，子分类名称是数据库的名称。 + # 提取方式是从file_name中利用正则规则去提取。 + self.error_msg = "" + + file_name = os.path.abspath(file_name) + temp_name = os.path.split(file_name)[1] + object_name = 'backup/' + temp_name + + print(file_name) + print(object_name) + + return self.resumable_upload(file_name, + object_name=object_name, + *args, + **kwargs) + except Exception as e: + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += "文件上传出现错误：{}".format(str(e)) + return False + + def upload_file(self, file_name, data_type, *args, **kwargs): + """按照数据类型上传文件 + + :param file_name: 上传文件名称 + :param data_type: 数据类型 site/database/path + :return: True/False + """ + try: + import re + # 根据数据类型提取子分类名称 + # 比如data_type=database，子分类名称是数据库的名称。 + # 提取方式是从file_name中利用正则规则去提取。 + self.error_msg = "" + + if not file_name or not data_type: + _error_msg = "文件参数错误。" + print(_error_msg) + self.error_msg = _error_msg + return False + + file_name = os.path.abspath(file_name) + temp_name = os.path.split(file_name)[1] + object_name = self.build_object_name(data_type, temp_name) + + # dir_name = os.path.dirname(object_name) + # self.create_dir(dir_name) + if DEBUG: + print(file_name) + print(object_name) + print(dir_name) + + return self.resumable_upload(file_name, + object_name=object_name, + *args, + **kwargs) + except Exception as e: + if self.error_msg: + self.error_msg += r"\n" + self.error_msg += "文件上传出现错误：{}".format(str(e)) + return False diff --git a/plugins/msonedrive/config.conf b/plugins/msonedrive/config.conf new file mode 100644 index 000000000..bbff48ef5 --- /dev/null +++ b/plugins/msonedrive/config.conf @@ -0,0 +1 @@ +{"backup_path": "/bt_backup/", "sign_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?response_type=code&client_id=18c452c4-1946-4181-8ed0-2d81e9de5823&redirect_uri=http%3A%2F%2Flocalhost%2Flogin%2Fauthorized&scope=offline_access+Files.ReadWrite.All+User.Read&state=nRgP4uOEAljlJprv7m6VEIpM851MMN&prompt=login", "user": null, "user_type": "internal"} \ No newline at end of file diff --git a/plugins/msonedrive/credentials.json b/plugins/msonedrive/credentials.json new file mode 100644 index 000000000..123443172 --- /dev/null +++ b/plugins/msonedrive/credentials.json @@ -0,0 +1,12 @@ +{ + "onedrive-international": { + "client_id": "08125e6b-6502-4ac9-9548-ad682f00848d", + "client_secret": "0WA8Q~sZkZFZKv50ryP4ux~.fpVtbHw7BuTZmbQB", + "authority": "https://login.microsoftonline.com/common", + "token_endpoint": "/oauth2/v2.0/token", + "authorize_endpoint": "/oauth2/v2.0/authorize", + "scopes": "offline_access Files.ReadWrite.All User.Read", + "redirect_uri": "http://localhost", + "api_uri": "https://graph.microsoft.com/v1.0" + } +} \ No newline at end of file diff --git a/plugins/msonedrive/ico.png b/plugins/msonedrive/ico.png new file mode 100644 index 000000000..fb837e43f Binary files /dev/null and b/plugins/msonedrive/ico.png differ diff --git a/plugins/msonedrive/index.html b/plugins/msonedrive/index.html new file mode 100644 index 000000000..410ea7984 --- /dev/null +++ b/plugins/msonedrive/index.html @@ -0,0 +1,311 @@ + +
+
+ + +
+
    +
    + + + +
    + +
    +
    + + + +
    名称大小更新时间操作
    +
    +
    +
    + + + + + + + \ No newline at end of file diff --git a/plugins/msonedrive/index.py b/plugins/msonedrive/index.py new file mode 100644 index 000000000..2a451df2c --- /dev/null +++ b/plugins/msonedrive/index.py @@ -0,0 +1,351 @@ +# coding:utf-8 + +import sys +import io +import os +import time +import re +import json + + +# print(sys.platform) +if sys.platform != "darwin": + os.chdir("/www/server/mdserver-web") + + +sys.path.append(os.getcwd() + "/class/core") +import mw +import db + +_ver = sys.version_info +is_py2 = (_ver[0] == 2) +is_py3 = (_ver[0] == 3) + +DEBUG = False + +if is_py2: + reload(sys) + sys.setdefaultencoding('utf-8') + +app_debug = False +if mw.isAppleSystem(): + app_debug = True + + +def getPluginName(): + return 'msonedrive' + + +def getPluginDir(): + return mw.getPluginDir() + '/' + getPluginName() + + +def getServerDir(): + return mw.getServerDir() + '/' + getPluginName() + + +def in_array(name, arr=[]): + for x in arr: + if name == x: + return True + return False + + +sys.path.append(getPluginDir() + "/class") +from msodclient import msodclient + + +msodc = msodclient(getPluginDir(), getServerDir()) +msodc.setDebug(False) + + +def getArgs(): + args = sys.argv[2:] + tmp = {} + args_len = len(args) + if args_len == 1: + t = args[0].strip('{').strip('}') + if t.strip() == '': + tmp = [] + else: + t = t.split(':', 1) + tmp[t[0]] = t[1] + tmp[t[0]] = t[1] + elif args_len > 1: + for i in range(len(args)): + t = args[i].split(':', 1) + tmp[t[0]] = t[1] + return tmp + + +def checkArgs(data, ck=[]): + for i in range(len(ck)): + if not ck[i] in data: + return (False, mw.returnJson(False, '参数:(' + ck[i] + ')没有!')) + return (True, mw.returnJson(True, 'ok')) + + +def status(): + return 'start' + + +def isAuthApi(): + cfg = getServerDir() + "/user.conf" + if os.path.exists(cfg): + return True + return False + + +def getConf(): + if not isAuthApi(): + sign_in_url, state = msodc.get_sign_in_url() + return mw.returnJson(False, "未授权!", {'auth_url': sign_in_url}) + return mw.returnJson(True, "OK") + + +def setAuthUrl(): + args = getArgs() + data = checkArgs(args, ['url']) + if not data[0]: + return data[1] + + url = args['url'] + + try: + if url.startswith("http://"): + url = url.replace("http://", "https://") + token = msodc.get_token_from_authorized_url(authorized_url=url) + msodc.store_token(token) + msodc.store_user() + return mw.returnJson(True, "授权成功!") + except Exception as e: + return mw.returnJson(False, "授权失败2!:" + str(e)) + return mw.returnJson(False, "授权失败!:" + str(e)) + + +def clearAuth(): + cfg = getServerDir() + "/user.conf" + if os.path.exists(cfg): + os.remove(cfg) + + token = getServerDir() + "/token.pickle" + if os.path.exists(token): + os.remove(token) + + return mw.returnJson(True, "清空授权成功!") + + +def getList(): + cfg = getServerDir() + "/user.conf" + if not os.path.exists(cfg): + return mw.returnJson(False, "未配置,请点击`授权`", []) + + args = getArgs() + data = checkArgs(args, ['path']) + if not data[0]: + return data[1] + + try: + flist = msodc.get_list(args['path']) + return mw.returnJson(True, "ok", flist) + except Exception as e: + return mw.returnJson(False, str(e), []) + + +def createDir(): + cfg = getServerDir() + "/user.conf" + if not os.path.exists(cfg): + return mw.returnJson(False, "未配置OneDrive,请点击`授权`", []) + + args = getArgs() + data = checkArgs(args, ['path', 'name']) + if not data[0]: + return data[1] + + file = args['path'] + "/" + args['name'] + isok = msodc.create_dir(file) + if isok: + return mw.returnJson(True, "创建成功") + return mw.returnJson(False, "创建失败") + + +def deleteDir(): + args = getArgs() + data = checkArgs(args, ['dir_name', 'path']) + if not data[0]: + return data[1] + + file = args['path'] + "/" + args['dir_name'] + file = file.strip('/') + isok = msodc.delete_object(file) + if isok: + return mw.returnJson(True, "删除成功") + return mw.returnJson(False, "文件不为空,删除失败!") + + +def deleteFile(): + args = getArgs() + data = checkArgs(args, ['path', 'filename']) + if not data[0]: + return data[1] + + file = args['path'] + "/" + args['filename'] + file = file.strip('/') + isok = msodc.delete_object(file) + if isok: + return mw.returnJson(True, "删除成功") + return mw.returnJson(False, "删除失败") + + +def findPathName(path, filename): + f = os.scandir(path) + l = [] + for ff in f: + t = {} + if ff.name.find(filename) > -1: + t['filename'] = path + '/' + ff.name + l.append(t) + return l + + +def backupAllFunc(stype): + if not isAuthApi(): + mw.echoInfo("未授权API,无法使用!!!") + return '' + + os.chdir(mw.getRunDir()) + backup_dir = mw.getBackupDir() + run_dir = mw.getRunDir() + + stype = sys.argv[1] + name = sys.argv[2] + num = sys.argv[3] + + prefix_dict = { + "site": "web", + "database": "db", + "path": "path", + } + + backups = [] + sql = db.Sql() + + # print("stype:", stype) + # 提前获取-清理多余备份 + if stype == 'site': + pid = sql.table('sites').where('name=?', (name,)).getField('id') + backups = sql.table('backup').where( + 'type=? and pid=?', ('0', pid)).field('id,filename').select() + if stype == 'database': + db_path = mw.getServerDir() + '/mysql' + pid = mw.M('databases').dbPos(db_path, 'mysql').where( + 'name=?', (name,)).getField('id') + backups = sql.table('backup').where( + 'type=? and pid=?', ('1', pid)).field('id,filename').select() + if stype == 'path': + backup_path = backup_dir + '/path' + _name = 'path_{}'.format(os.path.basename(name)) + backups = findPathName(backup_path, _name) + + # 其他类型关系性数据库(mysql类的) + if stype.find('database_') > -1: + plugin_name = stype.replace('database_', '') + db_path = mw.getServerDir() + '/' + plugin_name + pid = mw.M('databases').dbPos(db_path, 'mysql').where( + 'name=?', (name,)).getField('id') + backups = sql.table('backup').where( + 'type=? and pid=?', ('1', pid)).field('id,filename').select() + + args = stype + " " + name + " " + num + cmd = 'python3 ' + run_dir + '/scripts/backup.py ' + args + if stype.find('database_') > -1: + plugin_name = stype.replace('database_', '') + args = "database " + name + " " + num + cmd = 'python3 ' + run_dir + '/plugins/' + \ + plugin_name + '/scripts/backup.py ' + args + + if stype == 'path': + name = os.path.basename(name) + + # print("cmd:", cmd) + os.system(cmd) + + # 开始执行上传信息. + if stype.find('database_') > -1: + bk_name = 'database' + plugin_name = stype.replace('database_', '') + bk_prefix = plugin_name + '/db' + stype = 'database' + else: + bk_prefix = prefix_dict[stype] + bk_name = stype + + find_path = backup_dir + '/' + bk_name + '/' + bk_prefix + '_' + name + find_new_file = "ls " + find_path + \ + "_* | grep '.gz' | cut -d \ -f 1 | awk 'END {print}'" + + # print(find_new_file) + + filename = mw.execShell(find_new_file)[0].strip() + if filename == "": + mw.echoInfo("not find upload file!") + return False + + mw.echoInfo("准备上传文件 {}".format(filename)) + mw.echoStart('开始上传') + msodc.upload_file(filename, stype) + mw.echoEnd('上传成功') + + # print(backups) + backups = sorted(backups, key=lambda x: x['filename'], reverse=False) + mw.echoStart('开始删除远程备份') + num = int(num) + sep = len(backups) - num + if sep > -1: + for backup in backups: + fn = os.path.basename(backup['filename']) + msodc.delete_file(fn, stype) + mw.echoInfo("---已清理远程过期备份文件：" + fn) + sep -= 1 + if sep < 0: + break + mw.echoEnd('结束删除远程备份') + + return '' + + +def installPreInspection(): + return 'ok' + +if __name__ == "__main__": + func = sys.argv[1] + if func == 'status': + print(status()) + elif func == 'start': + print(start()) + elif func == 'stop': + print(stop()) + elif func == 'restart': + print(restart()) + elif func == 'reload': + print(reload()) + elif func == 'install_pre_inspection': + print(installPreInspection()) + elif func == 'conf': + print(getConf()) + elif func == 'set_auth_url': + print(setAuthUrl()) + elif func == 'clear_auth': + print(clearAuth()) + elif func == "get_list": + print(getList()) + elif func == "create_dir": + print(createDir()) + elif func == "delete_dir": + print(deleteDir()) + elif func == 'delete_file': + print(deleteFile()) + elif in_array(func, ['site', 'database', 'path']) or func.find('database_') > -1: + print(backupAllFunc(func)) + else: + print('error') diff --git a/plugins/msonedrive/info.json b/plugins/msonedrive/info.json new file mode 100644 index 000000000..0fae6c8fd --- /dev/null +++ b/plugins/msonedrive/info.json @@ -0,0 +1,14 @@ +{ + "hook":["backup"], + "title": "OneDrive", + "tip": "lib", + "name": "msonedrive", + "type": "sort", + "ps": "微软家的云网盘服务。", + "versions": "1.0", + "shell": "install.sh", + "checks": "server/msonedrive", + "author": "midoks", + "date": "2023-8-18", + "pid": "4" +} diff --git a/plugins/msonedrive/install.sh b/plugins/msonedrive/install.sh new file mode 100644 index 000000000..549c9012d --- /dev/null +++ b/plugins/msonedrive/install.sh @@ -0,0 +1,41 @@ +#!/bin/bash +PATH=/www/server/panel/pyenv/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:~/bin +export PATH + +curPath=`pwd` +rootPath=$(dirname "$curPath") +rootPath=$(dirname "$rootPath") +serverPath=$(dirname "$rootPath") + +install_tmp=${rootPath}/tmp/mw_install.pl +VERSION=$2 + +if [ -f ${rootPath}/bin/activate ];then + source ${rootPath}/bin/activate +fi + +Install_App() +{ + pip install requests-oauthlib==1.3.0 + mkdir -p $serverPath/msonedrive + echo '正在安装脚本文件...' > $install_tmp + + echo "${VERSION}" > $serverPath/msonedrive/version.pl + echo '安装完成' > $install_tmp + + echo "Successify" +} + + +Uninstall_App() +{ + rm -rf $serverPath/msonedrive +} + +if [ "${1}" == 'install' ];then + Install_App +elif [ "${1}" == 'uninstall' ];then + Uninstall_App +else + echo 'Error!'; +fi diff --git a/plugins/msonedrive/js/msonedrive.js b/plugins/msonedrive/js/msonedrive.js new file mode 100644 index 000000000..205de67b5 --- /dev/null +++ b/plugins/msonedrive/js/msonedrive.js @@ -0,0 +1,258 @@ + +function msodPost(method,args,callback){ + var _args = null; + if (typeof(args) == 'string'){ + _args = JSON.stringify(toArrayObject(args)); + } else { + _args = JSON.stringify(args); + } + + var loadT = layer.msg('正在获取...', { icon: 16, time: 0, shade: 0.3 }); + $.post('/plugins/run', {name:'msonedrive', func:method, args:_args}, function(data) { + layer.close(loadT); + if (!data.status){ + layer.msg(data.msg,{icon:0,time:2000,shade: [0.3, '#000']}); + return; + } + + if(typeof(callback) == 'function'){ + callback(data); + } + },'json'); +} + + +function createDir(){ + layer.open({ + type: 1, + area: "400px", + title: "创建目录", + closeBtn: 1, + shift: 5, + shadeClose: false, + btn: ['确定','取消'], + content:'
    \ +

    \ + 目录名称：\ + \ +

    \ +
    ', + success:function(){ + $("input[name='newPath']").focus().keyup(function(e){ + if(e.keyCode == 13) $(".layui-layer-btn0").click(); + }); + }, + yes:function(index,layero){ + var name = $("input[name='newPath']").val(); + if(name == ''){ + layer.msg('目录名称不能为空!',{icon:2}); + return; + } + var path = $("#myPath").val(); + var dirname = name; + var loadT = layer.msg('正在创建目录['+dirname+']...',{icon:16,time:0,shade: [0.3, '#000']}); + msodPost('create_dir', {path:path,name:dirname}, function(data){ + layer.close(loadT); + var rdata = $.parseJSON(data.data); + if(rdata.status) { + showMsg(rdata.msg, function(){ + layer.close(index); + odList(path); + } ,{icon:1}, 2000); + } else{ + layer.msg(rdata.msg,{icon:2}); + } + }); + } + }); +} + + +//设置API +function authApi(){ + + msodPost('conf', {}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + + // console.log(rdata); + // console.log(rdata.data.auth_url); + var apicon = ''; + if (rdata.status){ + + var html = ''; + html += ''; + + var loadOpen = layer.open({ + type: 1, + title: '已授权', + area: '240px', + content:'
    '+html+'
    ', + success: function(){ + $('#clear_auth').click(function(){ + msodPost('clear_auth', {}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + showMsg(rdata.msg,function(){ + layer.close(loadOpen); + odList('/'); + },{icon:rdata.status?1:2},2000); + }); + }); + } + }); + return true; + + } else{ + apicon = '
    '+$("#check_api").html()+'
    '; + } + + var layer_auth = layer.open({ + type: 1, + area: "620px", + title: "OneDrive授权", + closeBtn: 1, + shift: 5, + shadeClose: false, + content:apicon, + success:function(layero,index){ + console.log(layero,index); + if (!rdata.status){ + $('.check_api .step_two_url').val(rdata.data['auth_url']); + $('.check_api .open_btlink').attr('href',rdata.data['auth_url']); + + $('.check_api .ico-copy').click(function(){ + copyPass(rdata.data['auth_url']); + }); + + $('.check_api .set_auth_btn').click(function(){ + + var url = $('.check_api .OneDrive').val(); + if ( url == ''){ + layer.msg("验证URL不能为空",{icon:2}); + return; + } + // console.log(url); + msodPost('set_auth_url', {url:url}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + var show_time = 2000; + if (!rdata.status){ + show_time = 10000; + } + + showMsg(rdata.msg,function(){ + if (rdata.status){ + layer.close(layer_auth); + odList('/'); + } + },{icon:rdata.status?1:2},show_time); + }); + }); + + + } + + } + }); + }); +} + +//计算当前目录偏移 +function upPathLeft(){ + var UlWidth = $(".place-input ul").width(); + var SpanPathWidth = $(".place-input").width() - 20; + var Ml = UlWidth - SpanPathWidth; + if(UlWidth > SpanPathWidth ){ + $(".place-input ul").css("left",-Ml) + } + else{ + $(".place-input ul").css("left",0) + } +} + +function odList(path){ + msodPost('get_list', {path:path}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + if(rdata.status === false){ + showMsg(rdata.msg,function(){ + authApi(); + },{icon:2},2000); + return; + } + + var mlist = rdata.data; + var listBody = '' + var listFiles = '' + for(var i=0;i\'+mlist.list[i].name+'\ + -\ + -\ + 删除' + }else{ + listFiles += '\'+mlist.list[i].name+'\ + '+toSize(mlist.list[i].size)+'\ + '+getLocalTime(mlist.list[i].time)+'\ + 下载 | 删除' + } + } + listBody += listFiles; + + var pathLi=''; + var tmp = path.split('/') + var pathname = ''; + var n = 0; + for(var i=0;i 0 && tmp[i] == '') continue; + var dirname = tmp[i]; + if(dirname == '') { + dirname = '根目录'; + n++; + } + pathname += '/' + tmp[i]; + pathname = pathname.replace('//','/'); + pathLi += '
  • '+dirname+'
    • '; + } + var um = 1; + if(tmp[tmp.length-1] == '') um = 2; + var backPath = tmp.slice(0,tmp.length-um).join('/') || '/'; + $('#myPath').val(path); + $(".upyunCon .place-input ul").html(pathLi); + $(".upyunlist .list-list").html(listBody); + + upPathLeft(); + + $('#backBtn').unbind().click(function() { + odList(backPath); + }); + + $('.upyunCon .refreshBtn').unbind().click(function(){ + odList(path); + }); + }); +} + + +//删除文件 +function deleteFile(name, is_dir){ + if (is_dir === false){ + safeMessage('删除文件','删除后将无法恢复，真的要删除['+name+']吗?',function(){ + var path = $("#myPath").val(); + var filename = name; + msodPost('delete_file', {filename:filename,path:path}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + showMsg(rdata.msg,function(){ + odList(path); + },{icon:rdata.status?1:2},2000); + }); + }); + } else { + safeMessage('删除文件夹','删除后将无法恢复，真的要删除['+name+']吗?',function(){ + var path = $("#myPath").val(); + msodPost('delete_dir', {dir_name:name,path:path}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + showMsg(rdata.msg,function(){ + odList(path); + },{icon:rdata.status?1:2},2000); + }); + }); + } +} \ No newline at end of file diff --git a/plugins/msonedrive/t/test.py b/plugins/msonedrive/t/test.py new file mode 100644 index 000000000..f0632a34d --- /dev/null +++ b/plugins/msonedrive/t/test.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# coding: utf-8 + +# python3 plugins/msonedrive/t/test.py + + +# https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/08125e6b-6502-4ac9-9548-ad682f00848d/objectId/62b1d655-9828-47ed-be99-65eb18c3a929/isMSAApp~/false/defaultBlade/Overview/appSignInAudience/AzureADandPersonalMicrosoftAccount/servicePrincipalCreated~/true + +# 0WA8Q~sZkZFZKv50ryP4ux~.fpVtbHw7BuTZmbQB +# client_id:d9878fac-8526-4ff6-8036-e1c92dd9dd80 + +# 08125e6b-6502-4ac9-9548-ad682f00848d + + +# https://login.microsoftonline.com/common/oauth2/v2.0/authorize?response_type=code&client_id=d9878fac-8526-4ff6-8036-e1c92dd9dd80&redirect_uri=http://localhost&scope=offline_access+Files.ReadWrite.All+User.Read&state=cwopMdHiPIkze4MvgFL6WfSl8LdYAl&prompt=login + + +# http://localhost/?code=M.C106_BAY.2.3e12c859-6107-0c5b-9ef4-14b3fb8269ba&state=JzHdzHXmA7x6zl7Be6cJ6uOlf9Bg69 + +import sys +import io +import os +import time +import re +import json + + +from requests_oauthlib import OAuth2Session + +sys.path.append(os.getcwd() + "/class/core") +import mw + + +def getPluginName(): + return 'msonedrive' + + +def getPluginDir(): + return mw.getPluginDir() + '/' + getPluginName() + + +def getServerDir(): + return mw.getServerDir() + '/' + getPluginName() + + +sys.path.append(getPluginDir() + "/class") +from msodclient import msodclient + +msodclient.setDebug(True) +msodc = msodclient(getPluginDir(), getServerDir()) + + +# sign_in_url, state = msodc.get_sign_in_url() +# print(sign_in_url) + + +def set_auth_url(url): + try: + if url.startswith("http://"): + url = url.replace("http://", "https://") + token = msodc.get_token_from_authorized_url( + authorized_url=url) + msodc.store_token(token) + msodc.store_user() + return mw.returnJson(True, "授权成功！") + except Exception as e: + print(e) + return mw.returnJson(False, "授权失败2！:" + str(e)) + return mw.returnJson(False, "授权失败！:" + str(e)) + +# url = 'http://localhost/?code=M.C106_BAY.2.310112f3-a158-c400-9667-d158cbd1de6c&state=jEJz0ucR9bpZYD9PGxp2GgRDotrzO6' +# token = set_auth_url(url) +# print(token) + +# token = msodc.get_token() +# print("token:", token) + +# t = msodc.get_list('/backup') +# print(t) + +# t = msodc.create_dir('backup') +# print(t) + +# t = msodc.delete_object('backup') +# print(t) + + +t = msodc.upload_file('web_t1.cn_20230830_134549.tar.gz', 'site') +print(t) +# print(msodc.error_msg) + +# /Users/midoks/Desktop/mwdev/server/mdserver-web/paramiko.log +# backup/site/paramiko.log +# |-正在上传到 backup/site/paramiko.log... +# True + + +# t = msodc.upload_abs_file( +# 'web_t1.cn_20230830_134549.tar.gz', 'site') +# print(t) +# print(msodc.error_msg)