diff --git a/app/db/clear_users.py b/app/db/clear_users.py index 63f7a1d2..c3c71476 100644 --- a/app/db/clear_users.py +++ b/app/db/clear_users.py @@ -1,8 +1,12 @@ import logging -from .db_methods import client +from .db_main import client logger = logging.getLogger('root_logger') client.drop_database('pres-parser-db') + +def drop_database(): + client.drop_database('pres-parser-db') + logger.info("Вся информация очищена!") diff --git a/app/db/db_main.py b/app/db/db_main.py new file mode 100644 index 00000000..1503356e --- /dev/null +++ b/app/db/db_main.py @@ -0,0 +1,38 @@ +from app.db.methods.client import get_client, get_db, get_fs + +client = get_client() +db = get_db() +fs = get_fs() + +users_collection = db['users'] +files_info_collection = db['presentations'] # actually, collection for all files (pres and reports) +checks_collection = db['checks'] +consumers_collection = db['consumers'] +criteria_pack_collection = db['criteria_pack'] +logs_collection = db.create_collection( + 'logs', capped=True, size=5242880) if not db['logs'] else db['logs'] +celery_check_collection = db['celery_check'] # collection for mapping celery_task to check + + +def get_checks_collection(): + return checks_collection + + +def get_users_collection(): + return users_collection + + +def get_criteria_pack_collection(): + return criteria_pack_collection + + +def get_files_info_collection(): + return files_info_collection + + +def get_logs_collection(): + return logs_collection + + +def get_celery_check_collection(): + return celery_check_collection diff --git a/app/db/db_methods.py b/app/db/db_methods.py deleted file mode 100644 index d80d92db..00000000 --- a/app/db/db_methods.py +++ /dev/null @@ -1,445 +0,0 @@ -from datetime import datetime -from os.path import basename - -import pymongo -from bson import ObjectId -from gridfs import GridFSBucket, NoFile -from pymongo import MongoClient -from utils import convert_to - -from .db_types import User, Presentation, Check, Consumers, Logs - -client = MongoClient("mongodb://mongodb:27017") -db = client['pres-parser-db'] -fs = GridFSBucket(db) - -users_collection = db['users'] -files_info_collection = db['presentations'] # actually, collection for all files (pres and reports) -checks_collection = db['checks'] -consumers_collection = db['consumers'] -criteria_pack_collection = db['criteria_pack'] -logs_collection = db.create_collection( - 'logs', capped=True, size=5242880) if not db['logs'] else db['logs'] -celery_check_collection = db['celery_check'] # collection for mapping celery_task to check - - -def get_client(): - return client - - -# Returns user if user was created and None if already exists -def add_user(username, password_hash='', is_LTI=False): - user = User() - user.username = username - user.is_LTI = is_LTI - if not is_LTI: - user.password_hash = password_hash - if users_collection.find_one({'username': username}) is not None: - return None - else: - users_collection.insert_one(user.pack()) - return user - - -# Returns user if there was user with given credentials and None if not -def validate_user(username, password_hash): - user = users_collection.find_one( - {'username': username, 'password_hash': password_hash}) - if user is not None: - return User(user) - else: - return None - - -# Returns user with given username or None -def get_user(username): - user = users_collection.find_one({'username': username}) - if user is not None: - return User(user) - else: - return None - -def get_all_users(): - return users_collection.find() - -# Returns True if user was found and updated and false if not (username can not be changed!) -def edit_user(user): - if users_collection.find_one_and_replace({'username': user.username}, user.pack()) is not None: - return True - else: - return False - - -# Deletes user with given username, deleting also all his presentations and their checks, returns user -def delete_user(username): - user = get_user(username) - for presentation_id in user.presentations: - user, presentation = delete_presentation(user, presentation_id) - user = User(users_collection.find_one_and_delete({'username': username})) - return user - - -# Adds presentations with given name to given user presentations, updates user, returns user and presentations id -def add_file_info_and_content(username, filepath, file_type, file_id=None): - if not file_id: file_id = ObjectId() - # parsed_file's info - filename = basename(filepath) - file_info = Presentation({ - '_id': file_id, - 'name': filename, - 'file_type': file_type - }) - file_info_id = files_info_collection.insert_one(file_info.pack()).inserted_id - assert file_id == file_info_id, f"{file_id} -- {file_info_id}" - # parsed_file's content in GridFS (file_id = file_info_id) - add_file_to_db(filename, filepath, file_info_id) - # add parsed_file to user info - users_collection.update_one({'username': username}, {"$push": {'presentations': file_info_id}}) - return file_info_id - - -# Returns presentations with given id or None -def get_presentation(presentation_id): - file = files_info_collection.find_one({'_id': presentation_id}) - if file is not None: - return Presentation(file) - else: - return None - - -# Returns presentations of given user with given id or None -def find_presentation(user, presentation_name): - presentations = [] - for presentation_id in user.presentations: - presentations.append(get_presentation(presentation_id)) - presentation = next( - (x for x in presentations if x.name == presentation_name), None) - if presentation is not None: - return presentation - else: - return None - - -# Deletes presentations with given id, deleting also its checks, returns presentations -def delete_presentation(user, presentation_id): - if presentation_id in user.presentations: - user.presentations.remove(presentation_id) - edit_user(user) - presentation = get_presentation(presentation_id) - for check_id in presentation.checks: - presentation, check = delete_check(presentation, check_id) - presentation = Presentation( - files_info_collection.find_one_and_delete({'_id': presentation_id})) - return user, presentation - else: - return user, get_presentation(presentation_id) - - -# Adds checks to given presentations, updates presentations, returns presentations and checks id -def add_check(file_id, check): - checks_id = checks_collection.insert_one(check.pack()).inserted_id - files_info_collection.update_one({'_id': file_id}, {"$push": {'checks': checks_id}}) - return checks_id - - -def update_check(check): - return bool(checks_collection.find_one_and_replace({'_id': check._id}, check.pack())) - - -def write_pdf(filename, filepath): - converted_filepath = convert_to(filepath, target_format='pdf') - return add_file_to_db(filename, converted_filepath) - - -def add_file_to_db(filename, filepath, file_id=None): - if not file_id: file_id = ObjectId() - fs.upload_from_stream_with_id(file_id, filename, open(filepath, 'rb')) - return file_id - - -def write_file_from_db_file(file_id, abs_filepath): - with open(abs_filepath, 'wb+') as file: - fs.download_to_stream(file_id, file) - return True - - -# Returns checks with given id or None -def get_check(checks_id): - checks = checks_collection.find_one({'_id': checks_id}) - if checks is not None: - return Check(checks) - else: - return None - - -# Returns presentations parsed_file with given id or None -def get_file_by_check(checks_id): - try: - return fs.open_download_stream(checks_id) - except NoFile: - return None - - -def get_checks_pdf(checks_id): - checks = checks_collection.find_one({'_id': checks_id}) - pdf_id = checks.get('conv_pdf_fs_id') - try: - return fs.open_download_stream(pdf_id) - except NoFile: - return None - - -def find_pdf_by_file_id(file_id): - try: - return fs.open_download_stream(file_id) - except NoFile: - return None - - -# Deletes checks with given id, returns presentations -def delete_check(presentation, checks_id): - if checks_id in presentation.checks: - upd_presentation = files_info_collection.update_one( - {'_id': presentation._id}, {"$pull": {'checks': checks_id}}) - checks = Check( - checks_collection.find_one_and_delete({'_id': checks_id})) - fs.delete(checks_id) - return presentation, checks - else: - return presentation, get_check(checks_id) - - -def get_unpassed_checks(): - return checks_collection.find({'is_passbacked': False}) - - -def set_passbacked_flag(checks_id, flag): - upd_check = {"$set": {}} - upd_check['$set']['is_passbacked'] = flag - - if flag is None: - # flag = None - if user without passback - upd_check['$set']['lms_passback_time'] = None - elif flag: - upd_check['$set']['lms_passback_time'] = datetime.now() - - check = checks_collection.update_one({'_id': checks_id}, upd_check) - return check if check else None - - -def get_latest_users_check(filter=None): - local_filter = filter - user = local_filter.get('user') - username_filter = {'username': user} if user else {} - all_users = [user['username'] for user in users_collection.find(username_filter, {'username': 1})] - latest_checks = [] - for user in all_users: - local_filter['user'] = user - check, count = get_checks_cursor(local_filter, limit=1, sort='_id', order='desc') - if count: - latest_checks.append(check[0]) - return latest_checks, len(latest_checks) - - -def get_latest_user_check_by_moodle(moodle_id): - return list(db.checks.find( - {'lms_user_id': moodle_id} - ).sort('_id', -1).limit(1)) - - -def get_latest_check_cursor(filter, *args, **kwargs): - return get_latest_users_check(filter) - - -# Return no of bytes stored in gridfs -def get_storage(): - files = db.fs.files.find() - ct = 0 - for file in files: - ct += file['length'] - - return ct - - -def get_all_checks(): - return checks_collection.find() - - -def get_checks(filter={}, latest=None, limit=10, offset=0, sort=None, order=None): - if latest: - return get_latest_check_cursor(filter, limit, offset, sort, order) - else: - return get_checks_cursor(filter, limit, offset, sort, order) - - -# get checks cursor with specified parameters - -def get_checks_cursor(filter={}, limit=10, offset=0, sort=None, order=None): - sort = 'lms_passback_time' if sort == 'moodle-date' else sort - - count = checks_collection.count_documents(filter) - rows = checks_collection.find(filter) - - if sort and order in ("asc, desc"): - rows = rows.sort(sort, pymongo.ASCENDING if order == - "asc" else pymongo.DESCENDING) - - rows = rows.skip(offset) if offset else rows - rows = rows.limit(limit) if limit else rows - - return rows, count - - -# get logs cursor with specified parameters - - -def get_logs_cursor(filter={}, limit=10, offset=0, sort=None, order=None): - sort = 'serviceName' if sort == 'service-name' else sort - - count = logs_collection.count_documents(filter) - rows = logs_collection.find(filter) - - if sort and order in ("asc, desc"): - rows = rows.sort(sort, pymongo.ASCENDING if order == - "asc" else pymongo.DESCENDING) - - rows = rows.skip(offset) if offset else rows - rows = rows.limit(limit) if limit else rows - - return rows, count - -def get_user_cursor(filter={}, limit=10, offset=0, sort=None, order=None): - sort = 'username' if sort == 'username' else sort - - count = users_collection.count_documents(filter) - rows = users_collection.find(filter) - - if sort and order in ("asc, desc"): - rows = rows.sort(sort, pymongo.ASCENDING if order == - "asc" else pymongo.DESCENDING) - - rows = rows.skip(offset) if offset else rows - rows = rows.limit(limit) if limit else rows - - return rows, count - - -# Get stats for one user, return a list in the form -# [check_id, login, time of check_id's creation, result(0/1)] - - -def get_user_checks(login): - return checks_collection.find({'user': login}) - - -def get_check_stats(oid): - return checks_collection.find_one({'_id': oid}) - - -# LTI -class ConsumersDBManager: - - @staticmethod - def add_consumer(consumer_key, consumer_secret, timestamp_and_nonce=[]): - consumer = Consumers() - consumer.consumer_key = consumer_key - consumer.consumer_secret = consumer_secret - if consumers_collection.find_one({'consumer_key': consumer_key}) is not None: - return None - else: - consumers_collection.insert_one(consumer.pack()) - return consumer - - @staticmethod - def get_secret(key): - consumer = consumers_collection.find_one({'consumer_key': key}) - if consumer is not None: - return consumer.get('consumer_secret') - else: - return None - - @staticmethod - def is_key_valid(key): - consumer = consumers_collection.find_one({'consumer_key': key}) - if consumer is not None: - return True - else: - return False - - @staticmethod - def has_timestamp_and_nonce(key, timestamp, nonce): - consumer = consumers_collection.find_one({'consumer_key': key}) - if consumer is not None: - entries = consumer.get('timestamp_and_nonce') - return (timestamp, nonce) in entries - else: - return False - - @staticmethod - def add_timestamp_and_nonce(key, timestamp, nonce): - upd_consumer = {"$push": {'timestamp_and_nonce': (timestamp, nonce)}} - consumer = consumers_collection.update_one( - {'consumer_key': key}, upd_consumer) - return consumer if consumer else None - - -def add_log(timestamp, serviceName, levelname, levelno, message, - pathname, filename, funcName, lineno): - args = locals() - new_log = Logs(args) - logs_collection.insert_one(new_log.pack()) - return new_log - - -# criteria pack methods - -def get_criteria_pack(name): - return criteria_pack_collection.find_one({'name': name}) - - -def save_criteria_pack(pack_info): - """ - pack_info - dict, that includes name, raw_criterions, file_type, min_score - """ - pack_info['updated'] = datetime.now() - return criteria_pack_collection.update_one({'name': pack_info.get('name')}, {'$set': pack_info}, upsert=True) - - -def get_criterion_pack_list(): - return criteria_pack_collection.find() - - -# mapping celery_task to check - -def add_celery_task(celery_task_id, check_id): - return celery_check_collection.insert_one( - {'celery_task_id': celery_task_id, 'check_id': check_id, 'started_at': datetime.now()}).inserted_id - - -def mark_celery_task_as_finished(celery_task_id, finished_time=None): - celery_task = get_celery_task(celery_task_id) - if not celery_task: return - if finished_time is None: finished_time = datetime.now() - return celery_check_collection.update_one({'celery_task_id': celery_task_id}, { - '$set': {'finished_at': finished_time, - 'processing_time': (finished_time - celery_task['started_at']).total_seconds()}}) - -def get_average_processing_time(min_time=5.0): - # use only success check (failed checks processing time is more bigger than normal) - result = list(celery_check_collection.aggregate([ - {'$match': {'processing_time': {'$lt': 170}}}, - {'$group': {'_id': None, 'avg_processing_time': {'$avg': "$processing_time"}}} - ])) - if result and result[0]['avg_processing_time']: - result = result[0]['avg_processing_time'] - if result > min_time: - return round(result, 1) - return min_time - - -def get_celery_task(celery_task_id): - return celery_check_collection.find_one({'celery_task_id': celery_task_id}) - - -def get_celery_task_by_check(check_id): - return celery_check_collection.find_one({'check_id': check_id}) diff --git a/app/db/db_types.py b/app/db/db_types.py deleted file mode 100644 index eeeb26d9..00000000 --- a/app/db/db_types.py +++ /dev/null @@ -1,147 +0,0 @@ -from bson import ObjectId -from flask_login import UserMixin - -from main.check_packs import BASE_PACKS, BaseCriterionPack, DEFAULT_TYPE_INFO, DEFAULT_REPORT_TYPE_INFO - -class Packable: - def __init__(self, dictionary): - pass - - def pack(self): - return dict(vars(self)) - - def __str__(self) -> str: - return f"{self.__class__.__name__}: {', '.join([f'{key}: {value}' for key, value in vars(self).items()])}" - - -# You shouldn't create this or change username and presentations explicitly -class User(Packable, UserMixin): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - self.username = dictionary.get('username', '') - self.name = dictionary.get('name', '') - self.password_hash = dictionary.get('password_hash', '') - self.presentations = dictionary.get('presentations', []) - self.file_type = dictionary.get('file_type', DEFAULT_REPORT_TYPE_INFO) - try: - self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type['type']).name) - except: - self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type).name) - self.is_LTI = dictionary.get('is_LTI', False) - self.lms_user_id = dictionary.get('lms_user_id', None) - self.is_admin = dictionary.get('is_admin', False) - self.params_for_passback = dictionary.get('params_for_passback', None) - self.formats = dictionary.get('formats', []) - self.two_files = dictionary.get('two_files', False) - - def get_id(self): - return self.username - - -class Consumers(Packable): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - self.consumer_key = dictionary.get('consumer_key', '') - self.consumer_secret = dictionary.get('consumer_secret', '') - self.timestamp_and_nonce = dictionary.get('timestamp_and_nonce', []) - - -class PackableWithId(Packable): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - if '_id' in dictionary: - self._id = ObjectId(dictionary.get('_id')) - - def pack(self, to_str=False): - package = super().pack() - if '_id' in package: package['_id'] = self._id if not to_str else str(self._id) - return package - - -# You shouldn't create or change this explicitly -class Presentation(PackableWithId): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - self.name = dictionary.get('name', '') - self.checks = dictionary.get('checks', []) - self.file_type = dictionary.get('file_type', DEFAULT_TYPE_INFO) - - -class Logs(Packable): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - self.timestamp = dictionary.get('timestamp', None) - self.serviceName = dictionary.get('serviceName', None) - self.levelname = dictionary.get('levelname', None) - self.levelno = dictionary.get('levelno', None) - self.message = dictionary.get('message', None) - self.pathname = dictionary.get('pathname', None) - self.filename = dictionary.get('filename', None) - self.funcName = dictionary.get('funcName', None) - self.lineno = dictionary.get('lineno', None) - - -class Check(PackableWithId): - def __init__(self, dictionary=None): - super().__init__(dictionary) - dictionary = dictionary or {} - self.filename = dictionary.get('filename', '') - self.conv_pdf_fs_id = dictionary.get('conv_pdf_fs_id', '') - self.user = dictionary.get('user', '') - self.lms_user_id = dictionary.get('lms_user_id', None) - self.is_passbacked = dictionary.get('is_passbacked', None) - self.lms_passback_time = dictionary.get('lms_passback_time', None) - self.score = dictionary.get('score', -1) - self.file_type = dictionary.get('file_type', DEFAULT_TYPE_INFO) - self.enabled_checks = dictionary.get('enabled_checks', BASE_PACKS.get(self.file_type['type']).name) - self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type['type']).name) - self.params_for_passback = dictionary.get('params_for_passback', None) - self.is_failed = dictionary.get('is_failed', None) - self.is_ended = dictionary.get('is_ended', True) - self.is_passed = dictionary.get('is_passed', int(self.score) == 1) - - def calc_score(self): - # check after implementation criterion pack - if isinstance(self.enabled_checks, (list,)): - return BaseCriterionPack.calc_score(self.enabled_checks) - # old check - enabled_checks = dict((k, v) for k, v in self.enabled_checks.items() if v) - enabled_value = len([check for check in enabled_checks.values() if check]) - numerical_score = 0 - for check in enabled_checks.values(): - try: - if check.get('pass'): - numerical_score += 1 - except TypeError: - pass - - return float("{:.3f}".format(numerical_score / enabled_value)) - - def correct(self): - # check after implementation criterion pack - if isinstance(self.enabled_checks, (list,)): - return self.is_passed - # old check - return all([check == False or check['pass'] for check in self.enabled_checks.values()]) - - def pack(self, to_str=False): - package = super().pack(to_str) - package['conv_pdf_fs_id'] = self.conv_pdf_fs_id if not to_str else str(self.conv_pdf_fs_id) - package['enabled_checks'] = self.enabled_checks - return package - - def get_flags(self): - def none_to_true(x): - return x is None or bool(x) - - def none_to_false(x): - return x is not None and bool(x) - - is_ended = none_to_true(self.is_ended) # None for old checks => True, True->True, False->False - is_failed = none_to_false(self.is_failed) # None for old checks => False, True->True, False->False - return {'is_ended': is_ended, 'is_failed': is_failed} diff --git a/app/db/methods/celery_check.py b/app/db/methods/celery_check.py new file mode 100644 index 00000000..67314a01 --- /dev/null +++ b/app/db/methods/celery_check.py @@ -0,0 +1,43 @@ +from datetime import datetime + +from app.db.db_main import get_celery_check_collection +from app.db.methods.client import get_db + +db = get_db() + +celery_check_collection = get_celery_check_collection() # collection for mapping celery_task to check + + +def add_celery_task(celery_task_id, check_id): + return celery_check_collection.insert_one( + {'celery_task_id': celery_task_id, 'check_id': check_id, 'started_at': datetime.now()}).inserted_id + + +def mark_celery_task_as_finished(celery_task_id, finished_time=None): + celery_task = get_celery_task(celery_task_id) + if not celery_task: return + if finished_time is None: finished_time = datetime.now() + return celery_check_collection.update_one({'celery_task_id': celery_task_id}, { + '$set': {'finished_at': finished_time, + 'processing_time': (finished_time - celery_task['started_at']).total_seconds()}}) + + +def get_average_processing_time(min_time=5.0): + # use only success check (failed checks processing time is more bigger than normal) + result = list(celery_check_collection.aggregate([ + {'$match': {'processing_time': {'$lt': 170}}}, + {'$group': {'_id': None, 'avg_processing_time': {'$avg': "$processing_time"}}} + ])) + if result and result[0]['avg_processing_time']: + result = result[0]['avg_processing_time'] + if result > min_time: + return round(result, 1) + return min_time + + +def get_celery_task(celery_task_id): + return celery_check_collection.find_one({'celery_task_id': celery_task_id}) + + +def get_celery_task_by_check(check_id): + return celery_check_collection.find_one({'check_id': check_id}) diff --git a/app/db/methods/check.py b/app/db/methods/check.py new file mode 100644 index 00000000..ed5abeb9 --- /dev/null +++ b/app/db/methods/check.py @@ -0,0 +1,146 @@ +from gridfs import NoFile +from datetime import datetime + +import pymongo + +from app.db.db_main import get_checks_collection, get_users_collection, get_files_info_collection +from app.db.methods.client import get_db, get_fs +from app.db.types.Check import Check + + +db = get_db() +fs = get_fs() + +checks_collection = get_checks_collection() + +users_collection = get_users_collection() +files_info_collection = get_files_info_collection() + + +def add_check(file_id, check): + checks_id = checks_collection.insert_one(check.pack()).inserted_id + files_info_collection.update_one({'_id': file_id}, {"$push": {'checks': checks_id}}) + return checks_id + + +def update_check(check): + return bool(checks_collection.find_one_and_replace({'_id': check._id}, check.pack())) + + +def get_check(checks_id): + checks = checks_collection.find_one({'_id': checks_id}) + if checks is not None: + return Check(checks) + else: + return None + + +# Returns presentations parsed_file with given id or None + +def get_checks_pdf(checks_id): + checks = checks_collection.find_one({'_id': checks_id}) + pdf_id = checks.get('conv_pdf_fs_id') + try: + return fs.open_download_stream(pdf_id) + except NoFile: + return None + + +def delete_check(presentation, checks_id): + if checks_id in presentation.checks: + upd_presentation = files_info_collection.update_one( + {'_id': presentation._id}, {"$pull": {'checks': checks_id}}) + checks = Check( + checks_collection.find_one_and_delete({'_id': checks_id})) + fs.delete(checks_id) + return presentation, checks + else: + return presentation, get_check(checks_id) + + +def get_unpassed_checks(): + return checks_collection.find({'is_passbacked': False}) + + +def set_passbacked_flag(checks_id, flag): + upd_check = {"$set": {}} + upd_check['$set']['is_passbacked'] = flag + + if flag is None: + # flag = None - if user without passback + upd_check['$set']['lms_passback_time'] = None + elif flag: + upd_check['$set']['lms_passback_time'] = datetime.now() + + check = checks_collection.update_one({'_id': checks_id}, upd_check) + return check if check else None + + +def get_latest_users_check(filter=None): + local_filter = filter + user = local_filter.get('user') + username_filter = {'username': user} if user else {} + all_users = [user['username'] for user in users_collection.find(username_filter, {'username': 1})] + latest_checks = [] + for user in all_users: + local_filter['user'] = user + check, count = get_checks_cursor(local_filter, limit=1, sort='_id', order='desc') + if count: + latest_checks.append(check[0]) + return latest_checks, len(latest_checks) + + +def get_latest_user_check_by_moodle(moodle_id): + return list(db.checks.find( + {'lms_user_id': moodle_id} + ).sort('_id', -1).limit(1)) + + +def get_latest_check_cursor(filter, *args, **kwargs): + return get_latest_users_check(filter) + + +def get_all_checks(): + return checks_collection.find() + + +def get_checks(filter={}, latest=None, limit=10, offset=0, sort=None, order=None): + if latest: + return get_latest_check_cursor(filter, limit, offset, sort, order) + else: + return get_checks_cursor(filter, limit, offset, sort, order) + + +def get_checks_cursor(filter={}, limit=10, offset=0, sort=None, order=None): + sort = 'lms_passback_time' if sort == 'moodle-date' else sort + + count = checks_collection.count_documents(filter) + rows = checks_collection.find(filter) + + if sort and order in ("asc, desc"): + rows = rows.sort(sort, pymongo.ASCENDING if order == + "asc" else pymongo.DESCENDING) + + rows = rows.skip(offset) if offset else rows + rows = rows.limit(limit) if limit else rows + + return rows, count + + +# get logs cursor with specified parameters + + +def get_user_checks(login): + return checks_collection.find({'user': login}) + + +def get_check_stats(oid): + return checks_collection.find_one({'_id': oid}) + + + + + + + + diff --git a/app/db/methods/client.py b/app/db/methods/client.py new file mode 100644 index 00000000..a5019f79 --- /dev/null +++ b/app/db/methods/client.py @@ -0,0 +1,19 @@ +from gridfs import GridFSBucket, NoFile +from pymongo import MongoClient + + +client = MongoClient("mongodb://mongodb:27017") +db = client['pres-parser-db'] +fs = GridFSBucket(db) + + +def get_client(): + return client + + +def get_db(): + return db + + +def get_fs(): + return fs \ No newline at end of file diff --git a/app/db/methods/criteria_pack.py b/app/db/methods/criteria_pack.py new file mode 100644 index 00000000..0fc42e81 --- /dev/null +++ b/app/db/methods/criteria_pack.py @@ -0,0 +1,26 @@ +from datetime import datetime + +from app.db.db_main import get_criteria_pack_collection +from app.db.methods.client import get_db, get_fs + + +db = get_db() +fs = get_fs() + +criteria_pack_collection = get_criteria_pack_collection() + + +def get_criteria_pack(name): + return criteria_pack_collection.find_one({'name': name}) + + +def save_criteria_pack(pack_info): + """ + pack_info - dict, that includes name, raw_criterions, file_type, min_score + """ + pack_info['updated'] = datetime.now() + return criteria_pack_collection.update_one({'name': pack_info.get('name')}, {'$set': pack_info}, upsert=True) + + +def get_criterion_pack_list(): + return criteria_pack_collection.find() diff --git a/app/db/methods/edit_user.py b/app/db/methods/edit_user.py new file mode 100644 index 00000000..8a432cbe --- /dev/null +++ b/app/db/methods/edit_user.py @@ -0,0 +1,8 @@ +from app.db.db_main import users_collection + + +def edit_user(user): + if users_collection.find_one_and_replace({'username': user.username}, user.pack()) is not None: + return True + else: + return False diff --git a/app/db/methods/file.py b/app/db/methods/file.py new file mode 100644 index 00000000..63bb2109 --- /dev/null +++ b/app/db/methods/file.py @@ -0,0 +1,116 @@ +from app.db.db_main import get_files_info_collection, get_users_collection +from app.db.methods.client import get_client, get_db, get_fs +from os.path import basename +from bson import ObjectId +from gridfs import NoFile + +from app.utils.converter import convert_to + +from app.db.types.Presentation import Presentation +from app.db.methods.edit_user import edit_user +from app.db.methods.check import delete_check + +client = get_client() +db = get_db() +fs = get_fs() + +files_info_collection = get_files_info_collection() # actually, collection for all files (pres and reports) + +users_collection = get_users_collection() + + +def add_file_info_and_content(username, filepath, file_type, file_id=None): + if not file_id: + file_id = ObjectId() + # parsed_file's info + filename = basename(filepath) + file_info = Presentation({ + '_id': file_id, + 'name': filename, + 'file_type': file_type + }) + file_info_id = files_info_collection.insert_one(file_info.pack()).inserted_id + assert file_id == file_info_id, f"{file_id} -- {file_info_id}" + # parsed_file's content in GridFS (file_id = file_info_id) + add_file_to_db(filename, filepath, file_info_id) + # add parsed_file to user info + users_collection.update_one({'username': username}, {"$push": {'presentations': file_info_id}}) + return file_info_id + + +# Returns presentations with given id or None +def get_presentation(presentation_id): + file = files_info_collection.find_one({'_id': presentation_id}) + if file is not None: + return Presentation(file) + else: + return None + + +# Returns presentations of given user with given id or None +def find_presentation(user, presentation_name): + presentations = [] + for presentation_id in user.presentations: + presentations.append(get_presentation(presentation_id)) + presentation = next( + (x for x in presentations if x.name == presentation_name), None) + if presentation is not None: + return presentation + else: + return None + + +# Deletes presentations with given id, deleting also its checks, returns presentations +def delete_presentation(user, presentation_id): + if presentation_id in user.presentations: + user.presentations.remove(presentation_id) + edit_user(user) + presentation = get_presentation(presentation_id) + for check_id in presentation.checks: + presentation, check = delete_check(presentation, check_id) + presentation = Presentation( + files_info_collection.find_one_and_delete({'_id': presentation_id})) + return user, presentation + else: + return user, get_presentation(presentation_id) + + +def write_pdf(filename, filepath): + converted_filepath = convert_to(filepath, target_format='pdf') + return add_file_to_db(filename, converted_filepath) + + +def add_file_to_db(filename, filepath, file_id=None): + if not file_id: + file_id = ObjectId() + fs.upload_from_stream_with_id(file_id, filename, open(filepath, 'rb')) + return file_id + + +def write_file_from_db_file(file_id, abs_filepath): + with open(abs_filepath, 'wb+') as file: + fs.download_to_stream(file_id, file) + return True + + +def get_file_by_check(checks_id): + try: + return fs.open_download_stream(checks_id) + except NoFile: + return None + + +def find_pdf_by_file_id(file_id): + try: + return fs.open_download_stream(file_id) + except NoFile: + return None + + +def get_storage(): + files = db.fs.files.find() + ct = 0 + for file in files: + ct += file['length'] + + return ct diff --git a/app/db/methods/log.py b/app/db/methods/log.py new file mode 100644 index 00000000..ccf476b2 --- /dev/null +++ b/app/db/methods/log.py @@ -0,0 +1,34 @@ +import pymongo + +from app.db.db_main import get_logs_collection +from app.db.methods.client import get_db + +from app.db.types.Logs import Logs + +db = get_db() + +logs_collection = get_logs_collection() + + +def add_log(timestamp, serviceName, levelname, levelno, message, + pathname, filename, funcName, lineno): + args = locals() + new_log = Logs(args) + logs_collection.insert_one(new_log.pack()) + return new_log + + +def get_logs_cursor(filter={}, limit=10, offset=0, sort=None, order=None): + sort = 'serviceName' if sort == 'service-name' else sort + + count = logs_collection.count_documents(filter) + rows = logs_collection.find(filter) + + if sort and order in ("asc, desc"): + rows = rows.sort(sort, pymongo.ASCENDING if order == + "asc" else pymongo.DESCENDING) + + rows = rows.skip(offset) if offset else rows + rows = rows.limit(limit) if limit else rows + + return rows, count \ No newline at end of file diff --git a/app/db/methods/user.py b/app/db/methods/user.py new file mode 100644 index 00000000..3af14b4b --- /dev/null +++ b/app/db/methods/user.py @@ -0,0 +1,75 @@ +import pymongo + +from app.db.db_main import get_users_collection +from app.db.methods.client import get_db +from app.db.types.User import User + +from app.db.methods.file import delete_presentation + +db = get_db() + +users_collection = get_users_collection() + + +def add_user(username, password_hash='', is_LTI=False): + user = User() + user.username = username + user.is_LTI = is_LTI + if not is_LTI: + user.password_hash = password_hash + if users_collection.find_one({'username': username}) is not None: + return None + else: + users_collection.insert_one(user.pack()) + return user + + +def validate_user(username, password_hash): + user = users_collection.find_one( + {'username': username, 'password_hash': password_hash}) + if user is not None: + return User(user) + else: + return None + + +def get_user(username): + user = users_collection.find_one({'username': username}) + if user is not None: + return User(user) + else: + return None + + +def get_all_users(): + return users_collection.find() + + +def delete_user(username): + user = get_user(username) + for presentation_id in user.presentations: + user, presentation = delete_presentation(user, presentation_id) + user = User(users_collection.find_one_and_delete({'username': username})) + return user + + +def get_user_cursor(filter={}, limit=10, offset=0, sort=None, order=None): + sort = 'username' if sort == 'username' else sort + + count = users_collection.count_documents(filter) + rows = users_collection.find(filter) + + if sort and order in ("asc, desc"): + rows = rows.sort(sort, pymongo.ASCENDING if order == + "asc" else pymongo.DESCENDING) + + rows = rows.skip(offset) if offset else rows + rows = rows.limit(limit) if limit else rows + + return rows, count +# +# def edit_user(user): +# if users_collection.find_one_and_replace({'username': user.username}, user.pack()) is not None: +# return True +# else: +# return False \ No newline at end of file diff --git a/app/db/types/Check.py b/app/db/types/Check.py new file mode 100644 index 00000000..82f3a9c7 --- /dev/null +++ b/app/db/types/Check.py @@ -0,0 +1,63 @@ +from app.db.types.PackableWithId import PackableWithId +from app.main.check_packs import DEFAULT_TYPE_INFO, BASE_PACKS, BaseCriterionPack + + +class Check(PackableWithId): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + self.filename = dictionary.get('filename', '') + self.conv_pdf_fs_id = dictionary.get('conv_pdf_fs_id', '') + self.user = dictionary.get('user', '') + self.lms_user_id = dictionary.get('lms_user_id', None) + self.is_passbacked = dictionary.get('is_passbacked', None) + self.lms_passback_time = dictionary.get('lms_passback_time', None) + self.score = dictionary.get('score', -1) + self.file_type = dictionary.get('file_type', DEFAULT_TYPE_INFO) + self.enabled_checks = dictionary.get('enabled_checks', BASE_PACKS.get(self.file_type['type']).name) + self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type['type']).name) + self.params_for_passback = dictionary.get('params_for_passback', None) + self.is_failed = dictionary.get('is_failed', None) + self.is_ended = dictionary.get('is_ended', True) + self.is_passed = dictionary.get('is_passed', int(self.score) == 1) + + def calc_score(self): + # check after implementation criterion pack + if isinstance(self.enabled_checks, (list,)): + return BaseCriterionPack.calc_score(self.enabled_checks) + # old check + enabled_checks = dict((k, v) for k, v in self.enabled_checks.items() if v) + enabled_value = len([check for check in enabled_checks.values() if check]) + numerical_score = 0 + for check in enabled_checks.values(): + try: + if check.get('pass'): + numerical_score += 1 + except TypeError: + pass + + return float("{:.3f}".format(numerical_score / enabled_value)) + + def correct(self): + # check after implementation criterion pack + if isinstance(self.enabled_checks, (list,)): + return self.is_passed + # old check + return all([check == False or check['pass'] for check in self.enabled_checks.values()]) + + def pack(self, to_str=False): + package = super().pack(to_str) + package['conv_pdf_fs_id'] = self.conv_pdf_fs_id if not to_str else str(self.conv_pdf_fs_id) + package['enabled_checks'] = self.enabled_checks + return package + + def get_flags(self): + def none_to_true(x): + return x is None or bool(x) + + def none_to_false(x): + return x is not None and bool(x) + + is_ended = none_to_true(self.is_ended) # None for old checks => True, True->True, False->False + is_failed = none_to_false(self.is_failed) # None for old checks => False, True->True, False->False + return {'is_ended': is_ended, 'is_failed': is_failed} \ No newline at end of file diff --git a/app/db/types/ConsumerDBManager.py b/app/db/types/ConsumerDBManager.py new file mode 100644 index 00000000..f1f3b01d --- /dev/null +++ b/app/db/types/ConsumerDBManager.py @@ -0,0 +1,56 @@ +from app.db.methods.client import get_db, get_fs + + +from app.db.types.Consumers import Consumers + +db = get_db() +fs = get_fs() + +consumers_collection = db['consumers'] + + +# LTI +class ConsumersDBManager: + + @staticmethod + def add_consumer(consumer_key, consumer_secret, timestamp_and_nonce=[]): + consumer = Consumers() + consumer.consumer_key = consumer_key + consumer.consumer_secret = consumer_secret + if consumers_collection.find_one({'consumer_key': consumer_key}) is not None: + return None + else: + consumers_collection.insert_one(consumer.pack()) + return consumer + + @staticmethod + def get_secret(key): + consumer = consumers_collection.find_one({'consumer_key': key}) + if consumer is not None: + return consumer.get('consumer_secret') + else: + return None + + @staticmethod + def is_key_valid(key): + consumer = consumers_collection.find_one({'consumer_key': key}) + if consumer is not None: + return True + else: + return False + + @staticmethod + def has_timestamp_and_nonce(key, timestamp, nonce): + consumer = consumers_collection.find_one({'consumer_key': key}) + if consumer is not None: + entries = consumer.get('timestamp_and_nonce') + return (timestamp, nonce) in entries + else: + return False + + @staticmethod + def add_timestamp_and_nonce(key, timestamp, nonce): + upd_consumer = {"$push": {'timestamp_and_nonce': (timestamp, nonce)}} + consumer = consumers_collection.update_one( + {'consumer_key': key}, upd_consumer) + return consumer if consumer else None \ No newline at end of file diff --git a/app/db/types/Consumers.py b/app/db/types/Consumers.py new file mode 100644 index 00000000..267ac10d --- /dev/null +++ b/app/db/types/Consumers.py @@ -0,0 +1,10 @@ +from app.db.types.Packable import Packable + + +class Consumers(Packable): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + self.consumer_key = dictionary.get('consumer_key', '') + self.consumer_secret = dictionary.get('consumer_secret', '') + self.timestamp_and_nonce = dictionary.get('timestamp_and_nonce', []) diff --git a/app/db/types/Logs.py b/app/db/types/Logs.py new file mode 100644 index 00000000..eb837df4 --- /dev/null +++ b/app/db/types/Logs.py @@ -0,0 +1,16 @@ +from app.db.types.Packable import Packable + + +class Logs(Packable): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + self.timestamp = dictionary.get('timestamp', None) + self.serviceName = dictionary.get('serviceName', None) + self.levelname = dictionary.get('levelname', None) + self.levelno = dictionary.get('levelno', None) + self.message = dictionary.get('message', None) + self.pathname = dictionary.get('pathname', None) + self.filename = dictionary.get('filename', None) + self.funcName = dictionary.get('funcName', None) + self.lineno = dictionary.get('lineno', None) diff --git a/app/db/types/Packable.py b/app/db/types/Packable.py new file mode 100644 index 00000000..5867f039 --- /dev/null +++ b/app/db/types/Packable.py @@ -0,0 +1,9 @@ +class Packable: + def __init__(self, dictionary): + pass + + def pack(self): + return dict(vars(self)) + + def __str__(self) -> str: + return f"{self.__class__.__name__}: {', '.join([f'{key}: {value}' for key, value in vars(self).items()])}" diff --git a/app/db/types/PackableWithId.py b/app/db/types/PackableWithId.py new file mode 100644 index 00000000..23fe88c9 --- /dev/null +++ b/app/db/types/PackableWithId.py @@ -0,0 +1,17 @@ +from bson import ObjectId + +from app.db.types.Packable import Packable + + +class PackableWithId(Packable): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + if '_id' in dictionary: + self._id = ObjectId(dictionary.get('_id')) + + def pack(self, to_str=False): + package = super().pack() + if '_id' in package: + package['_id'] = self._id if not to_str else str(self._id) + return package diff --git a/app/db/types/Presentation.py b/app/db/types/Presentation.py new file mode 100644 index 00000000..09363d70 --- /dev/null +++ b/app/db/types/Presentation.py @@ -0,0 +1,11 @@ +from app.db.types.PackableWithId import PackableWithId +from app.main.check_packs import DEFAULT_TYPE_INFO + + +class Presentation(PackableWithId): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + self.name = dictionary.get('name', '') + self.checks = dictionary.get('checks', []) + self.file_type = dictionary.get('file_type', DEFAULT_TYPE_INFO) diff --git a/app/db/types/User.py b/app/db/types/User.py new file mode 100644 index 00000000..12f730a0 --- /dev/null +++ b/app/db/types/User.py @@ -0,0 +1,29 @@ +from flask_login import UserMixin + +from app.db.types.Packable import Packable +from app.main.check_packs import BASE_PACKS, DEFAULT_REPORT_TYPE_INFO + +# You shouldn't create this or change username and presentations explicitly + +class User(Packable, UserMixin): + def __init__(self, dictionary=None): + super().__init__(dictionary) + dictionary = dictionary or {} + self.username = dictionary.get('username', '') + self.name = dictionary.get('name', '') + self.password_hash = dictionary.get('password_hash', '') + self.presentations = dictionary.get('presentations', []) + self.file_type = dictionary.get('file_type', DEFAULT_REPORT_TYPE_INFO) + try: + self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type['type']).name) + except: + self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type).name) + self.is_LTI = dictionary.get('is_LTI', False) + self.lms_user_id = dictionary.get('lms_user_id', None) + self.is_admin = dictionary.get('is_admin', False) + self.params_for_passback = dictionary.get('params_for_passback', None) + self.formats = dictionary.get('formats', []) + self.two_files = dictionary.get('two_files', False) + + def get_id(self): + return self.username \ No newline at end of file diff --git a/app/lti_session_passback/lti/check_request.py b/app/lti_session_passback/lti/check_request.py index 727ad913..e58e4c91 100644 --- a/app/lti_session_passback/lti/check_request.py +++ b/app/lti_session_passback/lti/check_request.py @@ -1,4 +1,4 @@ -from db.db_methods import ConsumersDBManager +from app.db.types.ConsumerDBManager import ConsumersDBManager from lti.contrib.flask import FlaskToolProvider from utils.mock_lti_auth import mock_lti_auth diff --git a/app/lti_session_passback/lti/lti_validator.py b/app/lti_session_passback/lti/lti_validator.py index 720b475c..e8a3b6f6 100644 --- a/app/lti_session_passback/lti/lti_validator.py +++ b/app/lti_session_passback/lti/lti_validator.py @@ -1,4 +1,4 @@ -from db.db_methods import ConsumersDBManager +from app.db.types.ConsumerDBManager import ConsumersDBManager from oauthlib.oauth1 import RequestValidator diff --git a/app/lti_session_passback/lti/utils.py b/app/lti_session_passback/lti/utils.py index 44114ff8..cbcceef3 100644 --- a/app/lti_session_passback/lti/utils.py +++ b/app/lti_session_passback/lti/utils.py @@ -1,6 +1,6 @@ import logging -from db.db_methods import ConsumersDBManager +from app.db.types.ConsumerDBManager import ConsumersDBManager from main.checks_config.parser import sld_num logger = logging.getLogger('root_logger') diff --git a/app/main/checker.py b/app/main/checker.py index a8cf8474..f3174aee 100644 --- a/app/main/checker.py +++ b/app/main/checker.py @@ -1,4 +1,4 @@ -from db.db_methods import get_criteria_pack +from app.db.methods.criteria_pack import get_criteria_pack from .check_packs import BaseCriterionPack diff --git a/app/passback_grades.py b/app/passback_grades.py index 92bbca51..47067cf6 100644 --- a/app/passback_grades.py +++ b/app/passback_grades.py @@ -2,7 +2,9 @@ import urllib3 -from db.db_methods import ConsumersDBManager, get_unpassed_checks, set_passbacked_flag +from app.db.types.ConsumerDBManager import ConsumersDBManager +from app.db.methods import check as check_methods + from lti_session_passback.lti_provider import LTIProvider from root_logger import get_root_logger @@ -20,7 +22,7 @@ def check_success_response(response): def grade_passback(check): passback_params = check.get('params_for_passback', None) if not passback_params or passback_params["lis_outcome_service_url"] == "lis_outcome_service_url": - set_passbacked_flag(check.get('_id'), None) + check_methods.set_passbacked_flag(check.get('_id'), None) return consumer_secret = ConsumersDBManager.get_secret(passback_params['oauth_consumer_key']) @@ -38,19 +40,19 @@ def grade_passback(check): if check_success_response(response): logger.info('Score was successfully passed back: score = {}, check_id = {}'.format(check.get('score'), check.get('_id'))) - set_passbacked_flag(check.get('_id'), True) + check_methods.set_passbacked_flag(check.get('_id'), True) else: logger.error('Passback failed for check_id = {}'.format(check.get('_id'))) else: logger.info( 'LMS score is more then current (not passbacked): LMS_score = {}, system_score = {} check_id = {}'.format( current_lms_score, check.get('score'), check.get('_id'))) - set_passbacked_flag(check.get('_id'), None) + check_methods.set_passbacked_flag(check.get('_id'), None) def run_passback(): errors, passbacked = [], [] - for check in get_unpassed_checks(): + for check in check_methods.get_unpassed_checks(): try: grade_passback(check) passbacked.append(str(check['_id'])) diff --git a/app/root_logger.py b/app/root_logger.py index 2f9d6f61..12986477 100644 --- a/app/root_logger.py +++ b/app/root_logger.py @@ -2,7 +2,7 @@ import sys from datetime import datetime -from db.db_methods import add_log +from db.methods.log import add_log class MongoDBLoggingHandler(logging.StreamHandler): diff --git a/app/routes/api.py b/app/routes/api.py index fac54056..597b39a5 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -6,7 +6,8 @@ from flask_login import login_required, current_user from datetime import datetime -from app.db import db_methods +from app.db.methods import check as check_methods +from app.db.methods import criteria_pack as criteria_pack_methods from app.root_logger import get_root_logger from app.main.check_packs import BASE_PACKS, DEFAULT_REPORT_TYPE_INFO, REPORT_TYPES, init_criterions @@ -22,7 +23,7 @@ def ready_result(_id): except bson.errors.InvalidId: logger.error('_id exception:', exc_info=True) return {} - check = db_methods.get_check(oid) + check = check_methods.get_check(oid) if check is not None: return {"is_ended": check.is_ended} @@ -65,7 +66,7 @@ def api_criteria_pack(): msg = f"При инициализации набора {pack_name} возникли ошибки. JSON-конфигурация: '{raw_criterions}'. Успешно инициализированные: {inited}. Возникшие ошибки: {err}." return {'data': msg, 'time': datetime.now()}, 400 # if ok - save to DB - db_methods.save_criteria_pack({ + criteria_pack_methods.save_criteria_pack({ 'name': pack_name, 'raw_criterions': raw_criterions, 'file_type': file_type_info, diff --git a/app/routes/capacity.py b/app/routes/capacity.py index e17b160d..e8d32fe8 100644 --- a/app/routes/capacity.py +++ b/app/routes/capacity.py @@ -1,6 +1,6 @@ from flask import Blueprint, request, current_app -from app.db import db_methods +from app.db.methods import file as file_methods capacity = Blueprint('capacity', __name__, template_folder='templates', static_folder='static') @@ -10,10 +10,10 @@ def system_capacity(): units = {'b': 1, 'mb': 1024 ** 2, 'gb': 1024 ** 3} unit = units.get(request.args.get('unit', 'gb').lower(), units['gb']) - current_size = db_methods.get_storage() + current_size = file_methods.get_storage() ratio = current_size / current_app.config['MAX_SYSTEM_STORAGE'] return { 'size': current_size / unit, 'max_size': current_app.config['MAX_SYSTEM_STORAGE'] / unit, 'ratio': ratio - } \ No newline at end of file + } diff --git a/app/routes/check_list.py b/app/routes/check_list.py index bbb67eff..c1a8ee9e 100644 --- a/app/routes/check_list.py +++ b/app/routes/check_list.py @@ -1,11 +1,13 @@ from flask import Blueprint, render_template, jsonify, request from app.utils import format_check_for_table, checklist_filter -from app.db import db_methods +from app.db.methods import check as check_methods + from flask_login import login_required, current_user check_list = Blueprint('check_list', __name__, template_folder='templates', static_folder='static') + @check_list.route("/") @login_required def check_list_main(): @@ -35,10 +37,10 @@ def check_list_data(): query = dict(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) if data.get("latest"): - rows, count = db_methods.get_latest_check_cursor(**query) + rows, count = check_methods.get_latest_check_cursor(**query) else: # get data and records count - rows, count = db_methods.get_checks_cursor(**query) + rows, count = check_methods.get_checks_cursor(**query) # construct response response = { diff --git a/app/routes/checks.py b/app/routes/checks.py index 677960e2..bb26730c 100644 --- a/app/routes/checks.py +++ b/app/routes/checks.py @@ -4,7 +4,7 @@ from flask import Blueprint, render_template, Response from flask_login import login_required -from app.db import db_methods +from app.db.methods import file as file_methods from app.root_logger import get_root_logger @@ -16,7 +16,7 @@ @login_required def checks_main(_id): try: - f = db_methods.get_file_by_check(ObjectId(_id)) + f = file_methods.get_file_by_check(ObjectId(_id)) except bson.errors.InvalidId: logger.error('_id exception in checks occured:', exc_info=True) return render_template("./404.html") diff --git a/app/routes/criterion_pack.py b/app/routes/criterion_pack.py index 824755bb..7c520f4f 100644 --- a/app/routes/criterion_pack.py +++ b/app/routes/criterion_pack.py @@ -3,7 +3,7 @@ from flask import Blueprint, render_template, abort from flask_login import login_required, current_user -from app.db import db_methods +from app.db.methods import criteria_pack as criteria_pack_methods criterion_pack = Blueprint('criterion_pack', __name__, template_folder='templates', static_folder='static') @@ -16,14 +16,13 @@ def criteria_pack_new(): return render_template('./criteria_pack.html', name=current_user.name, navi_upload=True) - @criterion_pack.route("/", methods=["GET"]) @login_required def criteria_pack(name): if not current_user.is_admin: abort(403) - pack = db_methods.get_criteria_pack(name) + pack = criteria_pack_methods.get_criteria_pack(name) if not pack: abort(404) pack['raw_criterions'] = json.dumps(pack['raw_criterions'], indent=4, ensure_ascii=False) diff --git a/app/routes/criterion_packs.py b/app/routes/criterion_packs.py index 86650a9d..9cb43705 100644 --- a/app/routes/criterion_packs.py +++ b/app/routes/criterion_packs.py @@ -1,8 +1,7 @@ - from flask import Blueprint, render_template, abort from flask_login import login_required, current_user -from app.db import db_methods +from app.db.methods import criteria_pack as criteria_pack_methods criterion_packs = Blueprint('criterion_packs', __name__, template_folder='templates', static_folder='static') @@ -13,6 +12,5 @@ def criteria_packs(): if not current_user.is_admin: abort(403) - packs = db_methods.get_criterion_pack_list() + packs = criteria_pack_methods.get_criterion_pack_list() return render_template('./pack_list.html', packs=packs, name=current_user.name, navi_upload=True) - diff --git a/app/routes/get_last_check_results.py b/app/routes/get_last_check_results.py index 42d6a886..09bc95be 100644 --- a/app/routes/get_last_check_results.py +++ b/app/routes/get_last_check_results.py @@ -4,7 +4,7 @@ from flask import Blueprint, render_template, redirect, url_for from flask_login import login_required -from app.db import db_methods +from app.db.methods import check as check_methods from app.root_logger import get_root_logger @@ -15,7 +15,7 @@ @get_last_check_results.route("/", methods=["GET"]) @login_required def get_latest_user_check(moodle_id): - check = db_methods.get_latest_user_check_by_moodle(moodle_id) + check = check_methods.get_latest_user_check_by_moodle(moodle_id) logger.error(str(check)) if check: check = check[0] diff --git a/app/routes/get_pdf.py b/app/routes/get_pdf.py index 6dd3ae55..c653e140 100644 --- a/app/routes/get_pdf.py +++ b/app/routes/get_pdf.py @@ -4,7 +4,8 @@ from flask import Blueprint, Response, render_template from flask_login import login_required -from app.db import db_methods +from app.db.methods import file as file_methods + from app.root_logger import get_root_logger get_pdf = Blueprint('get_pdf', __name__, template_folder='templates', static_folder='static') @@ -15,7 +16,7 @@ @login_required def get_pdf_main(_id): try: - file = db_methods.find_pdf_by_file_id(ObjectId(_id)) + file = file_methods.find_pdf_by_file_id(ObjectId(_id)) except bson.errors.InvalidId: logger.error('_id exception in fetching pdf occured:', exc_info=True) return render_template("./404.html") diff --git a/app/routes/get_zip.py b/app/routes/get_zip.py index 9650b33f..4bfa4c73 100644 --- a/app/routes/get_zip.py +++ b/app/routes/get_zip.py @@ -3,16 +3,17 @@ import shutil import tempfile from io import StringIO - import pandas as pd +from flask import Blueprint, abort, request, Response from app.routes.utils import get_query, get_stats, check_export_access -from flask import Blueprint, abort, request, Response +from app.db.methods import file as file_methods +from app.db.methods import check as check_methods -from app.db import db_methods get_zip = Blueprint('get_zip', __name__, template_folder='templates', static_folder='static') + @get_zip.route("/") def get_zip_main(): if not check_export_access(): @@ -24,10 +25,10 @@ def get_zip_main(): dirpath = tempfile.TemporaryDirectory() # write files - checks_list, _ = db_methods.get_checks(**get_query(request)) + checks_list, _ = check_methods.get_checks(**get_query(request)) for check in checks_list: - db_file = db_methods.find_pdf_by_file_id(check['_id']) - original_name = db_methods.get_check(check['_id']).filename #get a filename from every check + db_file = file_methods.find_pdf_by_file_id(check['_id']) + original_name = check_methods.get_check(check['_id']).filename #get a filename from every check if db_file is not None: final_name = original_name if (original_name and original_names) else db_file.filename # to avoid overwriting files with one name and different content: now we save only last version of pres (from last check) diff --git a/app/routes/login.py b/app/routes/login.py index a36aa27a..ea12ec9a 100644 --- a/app/routes/login.py +++ b/app/routes/login.py @@ -12,4 +12,3 @@ def login_main(): elif request.method == "POST": u = user.login(request.json) return u.username if u is not None and login_user(u, remember=True) else "" - diff --git a/app/routes/logs.py b/app/routes/logs.py index 28346c42..674712cb 100644 --- a/app/routes/logs.py +++ b/app/routes/logs.py @@ -3,15 +3,11 @@ from flask_login import current_user, login_required from app.root_logger import get_root_logger from datetime import datetime, timedelta -from app.db import db_methods -# from app.server_consts import logger +from app.db.methods.log import get_logs_cursor logs = Blueprint('logs', __name__, template_folder='templates', static_folder='static') logger = get_root_logger('web') -# def get_logger(): -# return logger - @logs.route("/") @login_required def logs_main(): @@ -87,7 +83,7 @@ def logs_data(): order = 'desc' if not order else order # get data and records count - rows, count = db_methods.get_logs_cursor(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) + rows, count = get_logs_cursor(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) # construct response response = { diff --git a/app/routes/lti.py b/app/routes/lti.py index f2a303ae..60dac7c7 100644 --- a/app/routes/lti.py +++ b/app/routes/lti.py @@ -1,18 +1,20 @@ -from flask import Blueprint, render_template, jsonify, request, redirect, url_for, abort -from app.utils import format_check_for_table, checklist_filter -from app.db import db_methods -from flask_login import login_required, current_user, logout_user, login_user +from flask import Blueprint, request, redirect, url_for, abort +from flask_login import logout_user, login_user from app.lti_session_passback.lti import utils from app.lti_session_passback.lti.check_request import check_request from app.main.check_packs import BASE_PACKS, BaseCriterionPack, DEFAULT_TYPE from app.root_logger import get_root_logger +from app.db.methods import user as user_methods +from app.db.methods.edit_user import edit_user +from app.db.methods import criteria_pack as criteria_pack_methods from app.server_consts import ALLOWED_EXTENSIONS lti = Blueprint('lti', __name__, template_folder='templates', static_folder='static') logger = get_root_logger('web') + @lti.route('/lti', methods=['POST']) def lti_main(): if check_request(request): @@ -27,14 +29,14 @@ def lti_main(): # task settings # pack name custom_criterion_pack = custom_params.get('pack', BASE_PACKS.get(DEFAULT_TYPE).name) - criterion_pack_info = db_methods.get_criteria_pack(custom_criterion_pack) + criterion_pack_info = criteria_pack_methods.get_criteria_pack(custom_criterion_pack) if not criterion_pack_info: default_criterion_pack = BASE_PACKS.get(DEFAULT_TYPE).name logger.error( f"Ошибка при lti-авторизации. Несуществующий набор {custom_criterion_pack} пользователя {username}. Установлен набор по умолчанию: {default_criterion_pack}") logger.debug(f"lti-параметры: {temporary_user_params}") custom_criterion_pack = default_criterion_pack - criterion_pack_info = db_methods.get_criteria_pack(custom_criterion_pack) + criterion_pack_info = criteria_pack_methods.get_criteria_pack(custom_criterion_pack) custom_criterion_pack_obj = BaseCriterionPack(**criterion_pack_info) # get file type and formats from pack file_type_info = custom_criterion_pack_obj.file_type @@ -47,12 +49,12 @@ def lti_main(): logout_user() - lti_user = db_methods.add_user(user_id, is_LTI=True) + lti_user = user_methods.add_user(user_id, is_LTI=True) if lti_user: lti_user.name = person_name lti_user.is_admin = role else: - lti_user = db_methods.get_user(user_id) + lti_user = user_methods.get_user(user_id) # task settings lti_user.file_type = file_type_info @@ -63,9 +65,10 @@ def lti_main(): lti_user.params_for_passback = params_for_passback lti_user.lms_user_id = lms_user_id - db_methods.edit_user(lti_user) + edit_user(lti_user) login_user(lti_user) return redirect(url_for('upload')) else: - abort(403) \ No newline at end of file + abort(403) + \ No newline at end of file diff --git a/app/routes/recheck.py b/app/routes/recheck.py index 2e2113da..b3fe8bd3 100644 --- a/app/routes/recheck.py +++ b/app/routes/recheck.py @@ -5,8 +5,10 @@ from flask import Blueprint, request, abort, redirect, url_for from flask_login import login_required, current_user -from app.db import db_methods from app.tasks import create_task +from app.db.methods import file as file_methods +from app.db.methods import check as check_methods +from app.db.methods import celery_check as celery_check_methods from app.server_consts import UPLOAD_FOLDER @@ -20,7 +22,7 @@ def recheck_main(check_id): if not current_user.is_admin: abort(403) oid = ObjectId(check_id) - check = db_methods.get_check(oid) + check = check_methods.get_check(oid) if not check: abort(404) @@ -28,13 +30,13 @@ def recheck_main(check_id): # write files (original and pdf) to filestorage filepath = join(UPLOAD_FOLDER, f"{check_id}.{check.filename.rsplit('.', 1)[-1]}") pdf_filepath = join(UPLOAD_FOLDER, f"{check_id}.pdf") - db_methods.write_file_from_db_file(oid, filepath) - db_methods.write_file_from_db_file(ObjectId(check.conv_pdf_fs_id), pdf_filepath) + file_methods.write_file_from_db_file(oid, filepath) + file_methods.write_file_from_db_file(ObjectId(check.conv_pdf_fs_id), pdf_filepath) check.is_ended = False - db_methods.update_check(check) + check_methods.update_check(check) task = create_task.delay(check.pack(to_str=True)) # add check to queue - db_methods.add_celery_task(task.id, check_id) # mapping celery_task to check (check_id = file_id) + celery_check_methods.add_celery_task(task.id, check_id) # mapping celery_task to check (check_id = file_id) if request.args.get('api'): return {'task_id': task.id, 'check_id': check_id} else: diff --git a/app/routes/results.py b/app/routes/results.py index cc203676..d6d17cdd 100644 --- a/app/routes/results.py +++ b/app/routes/results.py @@ -3,7 +3,9 @@ from flask import Blueprint, render_template -from app.db import db_methods +from app.db.methods import check as check_methods +from app.db.methods import celery_check as celery_check_methods + from app.utils import format_check from app.root_logger import get_root_logger @@ -20,13 +22,13 @@ def results_main(_id): except bson.errors.InvalidId: logger.error('_id exception:', exc_info=True) return render_template("./404.html") - check = db_methods.get_check(oid) + check = check_methods.get_check(oid) if check is not None: # show processing time for user - avg_process_time = None if check.is_ended else db_methods.get_average_processing_time() + avg_process_time = None if check.is_ended else celery_check_methods.get_average_processing_time() return render_template("./results.html", navi_upload=True, results=check, columns=TABLE_COLUMNS, avg_process_time=avg_process_time, stats=format_check(check.pack())) else: logger.info("Запрошенная проверка не найдена: " + _id) - return render_template("./404.html") \ No newline at end of file + return render_template("./404.html") diff --git a/app/routes/tasks.py b/app/routes/tasks.py index 8eedb8ad..29b75600 100644 --- a/app/routes/tasks.py +++ b/app/routes/tasks.py @@ -7,8 +7,10 @@ from app.root_logger import get_root_logger from app.utils import get_file_len, check_file -from app.db import db_methods -from app.db.db_types import Check +from app.db.methods import file as file_methods +from app.db.methods import check as check_methods +from app.db.methods import celery_check as celery_check_methods +from app.db.types.Check import Check from app.server_consts import ALLOWED_EXTENSIONS, UPLOAD_FOLDER from app.tasks import create_task @@ -29,7 +31,7 @@ def run_task(): if not file: logger.critical("request doesn't include file") return "request doesn't include file" - if get_file_len(file) * 2 + db_methods.get_storage() > current_app.config['MAX_SYSTEM_STORAGE']: + if get_file_len(file) * 2 + file_methods.get_storage() > current_app.config['MAX_SYSTEM_STORAGE']: logger.critical('Storage overload has occured') return 'storage_overload' file_check_response = check_file(file, extension, ALLOWED_EXTENSIONS[file_ext_type], check_mime=True) @@ -52,10 +54,10 @@ def run_task(): filepath = join(UPLOAD_FOLDER, f"{file_id}.{extension}") file.save(filepath) # add file and file's info to db - file_id = db_methods.add_file_info_and_content(current_user.username, filepath, file_type, file_id) + file_id = file_methods.add_file_info_and_content(current_user.username, filepath, file_type, file_id) # use pdf from response or convert to pdf and save on disk and db if current_user.two_files and pdf_file: - if get_file_len(pdf_file) * 2 + db_methods.get_storage() > current_app.config['MAX_SYSTEM_STORAGE']: + if get_file_len(pdf_file) * 2 + file_methods.get_storage() > current_app.config['MAX_SYSTEM_STORAGE']: logger.critical('Storage overload has occured') return 'storage_overload' logger.info( @@ -63,9 +65,9 @@ def run_task(): filenamepdf, extension = pdf_file.filename.rsplit('.', 1) filepathpdf = join(UPLOAD_FOLDER, f"{file_id}.{extension}") pdf_file.save(filepathpdf) - converted_id = db_methods.add_file_to_db(filenamepdf, filepathpdf) + converted_id = file_methods.add_file_to_db(filenamepdf, filepathpdf) else: - converted_id = db_methods.write_pdf(filename, filepath) + converted_id = file_methods.write_pdf(filename, filepath) check = Check({ '_id': file_id, 'conv_pdf_fs_id': converted_id, @@ -80,11 +82,12 @@ def run_task(): 'is_failed': False, 'params_for_passback': current_user.params_for_passback }) - db_methods.add_check(file_id, check) # add check for parsed_file to db + check_methods.add_check(file_id, check) # add check for parsed_file to db task = create_task.delay(check.pack(to_str=True)) # add check to queue - db_methods.add_celery_task(task.id, file_id) # mapping celery_task to check (check_id = file_id) + celery_check_methods.add_celery_task(task.id, file_id) # mapping celery_task to check (check_id = file_id) return {'task_id': task.id, 'check_id': str(file_id)} + @tasks.route("/", methods=["GET"]) @login_required def get_status(task_id): diff --git a/app/routes/upload.py b/app/routes/upload.py index 521b3d16..94896244 100644 --- a/app/routes/upload.py +++ b/app/routes/upload.py @@ -1,7 +1,7 @@ from flask import Blueprint, request, abort, render_template from flask_login import login_required, current_user -from app.db import db_methods +from app.db.methods import criteria_pack as criteria_pack_methods from app.server_consts import ALLOWED_EXTENSIONS from app.main.checks import CRITERIA_INFO @@ -20,11 +20,10 @@ def upload_main(): else: abort(401) elif request.method == "GET": - pack = db_methods.get_criteria_pack(current_user.criteria) + pack = criteria_pack_methods.get_criteria_pack(current_user.criteria) list_of_check = pack['raw_criterions'] file_type = current_user.file_type['type'] check_labels_and_discrpt = {CRITERIA_INFO[file_type][check[0]]['label']: CRITERIA_INFO[file_type][check[0]]['description'] for check in list_of_check} formats = set(current_user.formats) formats = formats & ALLOWED_EXTENSIONS[file_type] if formats else ALLOWED_EXTENSIONS[file_type] return render_template("./upload.html", navi_upload=False, formats=sorted(formats), list_of_check=check_labels_and_discrpt) - diff --git a/app/routes/users.py b/app/routes/users.py index d024063a..fd4399fe 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -2,9 +2,8 @@ from flask import abort, Blueprint, render_template, request, jsonify from flask_login import current_user from functools import wraps -from app.db.db_methods import get_all_users, get_user +from app.db.methods.user import get_user, get_user_cursor from utils import checklist_filter, format_check_for_table -from db import db_methods users = Blueprint('users', __name__, template_folder='templates', static_folder='static') @@ -17,6 +16,7 @@ def my_wrapper(*args, **kwargs): abort(403) return my_wrapper + @users.route("/data") @admin_required def users_data(): @@ -63,7 +63,7 @@ def users_data(): order = request.args.get("order", "") order = 'username' if not order else order - rows, count = db_methods.get_user_cursor(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) + rows, count = get_user_cursor(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) response = { "total": count, @@ -84,6 +84,7 @@ def users_data(): def index(): return render_template('user_list.html') + @users.route('/', methods=["GET"]) @admin_required def user_info(username): diff --git a/app/servants/pre_luncher.py b/app/servants/pre_luncher.py index 400e287e..e420a784 100644 --- a/app/servants/pre_luncher.py +++ b/app/servants/pre_luncher.py @@ -1,7 +1,11 @@ import hashlib import logging -from db.db_methods import add_user, get_user, get_client, edit_user, save_criteria_pack +from app.db.methods.user import add_user, get_user +from app.db.methods.edit_user import edit_user +from app.db.methods.criteria_pack import save_criteria_pack +from app.db.methods.client import get_client + from main.check_packs.pack_config import BASE_PACKS, DEFAULT_REPORT_TYPE_INFO from pymongo.errors import ConnectionFailure diff --git a/app/servants/user.py b/app/servants/user.py index f3e2ea91..73e93b7f 100644 --- a/app/servants/user.py +++ b/app/servants/user.py @@ -1,23 +1,24 @@ import logging -from db import db_methods from flask_login import logout_user, current_user +from app.db.methods.user import add_user, validate_user +from app.db.methods.edit_user import edit_user logger = logging.getLogger('root_logger') def login(args): - validation = db_methods.validate_user(args['username'], args['password_hash']) + validation = validate_user(args['username'], args['password_hash']) logger.info("Запрошен вход пользователя " + args['username'] + ", " + ("вход разрешен" if validation else "во входе отказано")) return validation def signup(args): - user = db_methods.add_user(args['username'], args['password_hash']) + user = add_user(args['username'], args['password_hash']) if user is not None: user.name = args['name'] - if db_methods.edit_user(user): + if edit_user(user): logger.info("Пользователь " + args['username'] + " зарегистрирован") return user logger.info("Не удалось зарегистрировать пользователя " + args['username']) @@ -32,7 +33,7 @@ def logout(): def edit(json): current_user.name = json['name'] - edited = db_methods.edit_user(current_user) + edited = edit_user(current_user) logger.info( "Пользователь " + current_user.username + ("" if edited else " не") + " изменил имя на " + current_user.name) return 'OK' if edited else 'Not OK' diff --git a/app/server.py b/app/server.py index 2317e47e..d0dd1b8f 100644 --- a/app/server.py +++ b/app/server.py @@ -1,35 +1,16 @@ -import json import os -import shutil -import tempfile -from datetime import datetime, timedelta -from os.path import join from sys import argv -from io import StringIO - -import bson -import pandas as pd -from bson import ObjectId -from celery.result import AsyncResult -from flask import (Flask, Response, abort, jsonify, redirect, render_template, +from flask import (Flask, redirect, render_template, request, url_for) -from flask_login import (LoginManager, current_user, login_required, - login_user, logout_user) +from flask_login import (LoginManager, current_user, login_user) from flask_recaptcha import ReCaptcha -import servants.user as user -from app.utils import format_check_for_table, check_file -from db import db_methods -from db.db_types import Check +import servants.user as servants_user +from app.db.methods.user import get_user from lti_session_passback.lti import utils -from lti_session_passback.lti.check_request import check_request -from main.check_packs import BASE_PACKS, BaseCriterionPack, DEFAULT_REPORT_TYPE_INFO, DEFAULT_TYPE, REPORT_TYPES, \ - init_criterions, BASE_PRES_CRITERION, BASE_REPORT_CRITERION from root_logger import get_logging_stdout_handler, get_root_logger from servants import pre_luncher -from tasks import create_task -from utils import checklist_filter, decorator_assertion, get_file_len, format_check -from app.main.checks import CRITERIA_INFO +from utils import decorator_assertion from routes.admin import admin from routes.users import users from routes.check_list import check_list @@ -97,7 +78,7 @@ @login_manager.user_loader def load_user(user_id): - return db_methods.get_user(user_id) + return get_user(user_id) # User chapters req handlers: @@ -107,7 +88,7 @@ def signup(): if request.method == "GET": return render_template("./signup.html", navi_upload=False) elif request.method == "POST": - u = user.signup(request.json) + u = servants_user.signup(request.json) return u.username if u is not None and login_user(u, remember=True) else "" @@ -124,12 +105,14 @@ def request_entity_too_large(error=None): def unauthorized_callback(): return redirect(url_for("login.login_main")) + @app.route('/', defaults={'path': ''}) @app.route('/') def catch_all(path): logger.info("Страница /" + path + " не найдена!") return render_template("./404.html") + @app.route("/") def default(): if current_user.is_authenticated: diff --git a/app/server_consts.py b/app/server_consts.py index 04ded62d..bfab3880 100644 --- a/app/server_consts.py +++ b/app/server_consts.py @@ -1,5 +1,4 @@ import os -from root_logger import get_root_logger UPLOAD_FOLDER = '/usr/src/project/files' ALLOWED_EXTENSIONS = { diff --git a/app/tasks.py b/app/tasks.py index c7ba47df..41e64dab 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -6,12 +6,12 @@ from celery.signals import worker_ready from passback_grades import run_passback -from db import db_methods -from db.db_types import Check +from db.types.Check import Check from main.checker import check from main.parser import parse -from main.check_packs import BASE_PACKS from root_logger import get_root_logger +from app.db.methods import check as check_methods +from app.db.methods import celery_check as celery_check_methods config = ConfigParser() config.read('app/config.ini') @@ -55,8 +55,8 @@ def create_task(self, check_info): updated_check = check(parse(original_filepath, pdf_filepath), check_obj) updated_check.is_ended = True updated_check.is_failed = False - db_methods.update_check(updated_check) # save to db - db_methods.mark_celery_task_as_finished(self.request.id) + check_methods.update_check(updated_check) # save to db + celery_check_methods.mark_celery_task_as_finished(self.request.id) # remove files from FILES_FOLDER after checking remove_files((original_filepath, pdf_filepath)) @@ -66,11 +66,11 @@ def create_task(self, check_info): if self.request.retries == self.max_retries: logger.error(f"\tДостигнуто максимальное количество попыток перезапуска. Удаление задачи из очереди", exc_info=True) - db_methods.mark_celery_task_as_finished(self.request.id) + celery_check_methods.mark_celery_task_as_finished(self.request.id) updated_check = Check(check_info) updated_check.is_failed = True updated_check.is_ended = True - db_methods.update_check(updated_check) # save to db + check_methods.update_check(updated_check) # save to db # remove files from FILES_FOLDER after checking remove_files((original_filepath, pdf_filepath)) return 'Not OK, error: {}'.format(e) diff --git a/app/templates/404.html b/app/templates/404.html index 86940cb3..bf8140dd 100644 --- a/app/templates/404.html +++ b/app/templates/404.html @@ -6,4 +6,6 @@
Страница не найдена!
+ + {% endblock %} diff --git a/app/templates/admin_criterions.html b/app/templates/admin_criterions.html index 577cef29..6d219cf5 100644 --- a/app/templates/admin_criterions.html +++ b/app/templates/admin_criterions.html @@ -47,6 +47,7 @@

+ diff --git a/app/templates/admin_pages_list.html b/app/templates/admin_pages_list.html index 69869528..4aba4ece 100644 --- a/app/templates/admin_pages_list.html +++ b/app/templates/admin_pages_list.html @@ -12,5 +12,6 @@

Список страниц для администра
  • Таблица с информацией о пользователях
    • + {% endblock main %} diff --git a/app/templates/check_list.html b/app/templates/check_list.html index d0d0d671..9a497e79 100644 --- a/app/templates/check_list.html +++ b/app/templates/check_list.html @@ -69,9 +69,10 @@ {% endblock main %} - {% block script %} + {% endblock %} + diff --git a/app/templates/criteria_pack.html b/app/templates/criteria_pack.html index e923e565..6e889a4d 100644 --- a/app/templates/criteria_pack.html +++ b/app/templates/criteria_pack.html @@ -55,4 +55,7 @@ + + + {% endblock %} \ No newline at end of file diff --git a/app/templates/intro_page.html b/app/templates/intro_page.html index b47ee953..a154d8fc 100644 --- a/app/templates/intro_page.html +++ b/app/templates/intro_page.html @@ -19,4 +19,6 @@

    Добро пожаловать в Doc + + {% endblock %} diff --git a/app/templates/login.html b/app/templates/login.html index 700de321..dbba99a2 100644 --- a/app/templates/login.html +++ b/app/templates/login.html @@ -44,4 +44,6 @@

    Вход

    + + {% endblock %} diff --git a/app/templates/logs.html b/app/templates/logs.html index 0ef51a0a..44ed6bb3 100644 --- a/app/templates/logs.html +++ b/app/templates/logs.html @@ -62,4 +62,6 @@ + + {% endblock main %} diff --git a/app/templates/one_user_info.html b/app/templates/one_user_info.html index 4da93383..211ca0d7 100644 --- a/app/templates/one_user_info.html +++ b/app/templates/one_user_info.html @@ -61,5 +61,6 @@

    + {% endblock main %} diff --git a/app/templates/pack_list.html b/app/templates/pack_list.html index 5a9e7731..67ad43a1 100644 --- a/app/templates/pack_list.html +++ b/app/templates/pack_list.html @@ -77,5 +77,6 @@ + {% endblock %} \ No newline at end of file diff --git a/app/templates/profile.html b/app/templates/profile.html index 519a2eb2..0f6c0d58 100644 --- a/app/templates/profile.html +++ b/app/templates/profile.html @@ -40,5 +40,5 @@

    {{ user.name }}

    {% endif %} - + {% endblock %} diff --git a/app/templates/results.html b/app/templates/results.html index 940583c9..e3301697 100644 --- a/app/templates/results.html +++ b/app/templates/results.html @@ -113,5 +113,6 @@

    {% endif %} + {% endblock %} diff --git a/app/templates/root.html b/app/templates/root.html index 9ccf6b8f..cd96f014 100644 --- a/app/templates/root.html +++ b/app/templates/root.html @@ -14,6 +14,8 @@ - + + + {% block script %}{% endblock %} diff --git a/app/templates/signup.html b/app/templates/signup.html index f73af1fc..5ee72c07 100644 --- a/app/templates/signup.html +++ b/app/templates/signup.html @@ -55,4 +55,6 @@

    Создать аккаунт

    + + {% endblock %} diff --git a/app/templates/upload.html b/app/templates/upload.html index 2a316f01..2a0053dc 100644 --- a/app/templates/upload.html +++ b/app/templates/upload.html @@ -74,7 +74,7 @@
    {{ uploading_label }}
    - Критерии: + Критерии: {% for criterion_name, criterion_descrpt in list_of_check.items() %} @@ -92,4 +92,6 @@
    {{ uploading_label }}
    + + {% endblock %} diff --git a/app/templates/user_list.html b/app/templates/user_list.html index d6c0baf1..9cf40915 100644 --- a/app/templates/user_list.html +++ b/app/templates/user_list.html @@ -59,6 +59,7 @@

    + {% endblock main %} diff --git a/app/templates/version.html b/app/templates/version.html index 825c39fd..6623a70b 100644 --- a/app/templates/version.html +++ b/app/templates/version.html @@ -46,5 +46,6 @@ {% endif %} + {% endblock main %} diff --git a/assets/scripts/check_list.js b/assets/scripts/check_list.js index 39d5e4b3..48161f6c 100644 --- a/assets/scripts/check_list.js +++ b/assets/scripts/check_list.js @@ -1,3 +1,5 @@ +import {collect_values_if_possible, hash} from "./general"; + import { debounce, isFloat, resetTable, ajaxRequest, onPopState } from "./utils" let $table; @@ -218,7 +220,7 @@ function buttons() { event: function () { const params = window.location.search $("[name=FetchZip]")[0].innerHTML = " Архивирование..." - fetch('get_zip' + '?' + params) + fetch('../get_zip' + '?' + params) .then(response => response.ok ? response.blob() : false) .then(blob => { $("[name=FetchZip]")[0].textContent = "Скачать архив" diff --git a/assets/scripts/general.js b/assets/scripts/general.js new file mode 100644 index 00000000..2a94488f --- /dev/null +++ b/assets/scripts/general.js @@ -0,0 +1,21 @@ +import * as md5 from "md5"; + +export function hash(password) { + return md5(password) +} + +export function collect_values_if_possible(...ids) { + const id_array = [...ids]; + const necessary_fields = $(id_array.map(el => "#" + el).join(", ")); + let valid = true; + necessary_fields.map(function () { + $(this).toggleClass("is-invalid", this.value === ""); + valid &= (this.value !== ""); + return this; + }); + if (valid) { + const result = Object(); + for (const field of necessary_fields) result[field.id] = field.value; + return result; + } +} diff --git a/assets/scripts/general_imports.js b/assets/scripts/general_imports.js new file mode 100644 index 00000000..cfe1c19f --- /dev/null +++ b/assets/scripts/general_imports.js @@ -0,0 +1,21 @@ +import 'bootstrap'; +import 'bootstrap/dist/css/bootstrap.min.css'; + +import 'bootstrap-table'; +import 'bootstrap-table/dist/bootstrap-table.min.css' + +import 'bootstrap-table/dist/extensions/filter-control/bootstrap-table-filter-control' +import 'bootstrap-table/dist/extensions/filter-control/bootstrap-table-filter-control.min.css' + +import 'bootstrap-table/dist/extensions/auto-refresh/bootstrap-table-auto-refresh.js' + +import 'bootstrap-icons/font/bootstrap-icons.css' + +import 'bootstrap-datepicker'; +import 'bootstrap-datepicker/dist/css/bootstrap-datepicker.min.css' + + +import '../styles/main.css'; + +import '../favicon.ico'; +import '../styles/404.css'; diff --git a/assets/scripts/login.js b/assets/scripts/login.js index 73614386..49485b49 100644 --- a/assets/scripts/login.js +++ b/assets/scripts/login.js @@ -1,8 +1,7 @@ -import {collect_values_if_possible, hash} from "./main"; +import {collect_values_if_possible, hash} from "./general"; import '../styles/login.css'; - $("#login_button").click(async () => { const params = collect_values_if_possible("login_text_field", "password_text_field"); diff --git a/assets/scripts/main.js b/assets/scripts/main.js-bat similarity index 100% rename from assets/scripts/main.js rename to assets/scripts/main.js-bat diff --git a/assets/scripts/signup.js b/assets/scripts/signup.js index 2c95c43c..5cbb58ec 100644 --- a/assets/scripts/signup.js +++ b/assets/scripts/signup.js @@ -1,4 +1,4 @@ -import {collect_values_if_possible, hash} from "./main"; +import {collect_values_if_possible, hash} from "./general"; import '../styles/signup.css'; diff --git a/db_versioning/versions.py b/db_versioning/versions.py index 800e4b92..bd953313 100644 --- a/db_versioning/versions.py +++ b/db_versioning/versions.py @@ -1,4 +1,4 @@ -from app.db.db_types import Check +from app.db.types.Check import Check class Version: diff --git a/webpack.config.js b/webpack.config.js index 77928871..161f4314 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -4,7 +4,25 @@ const { CleanWebpackPlugin } = require('clean-webpack-plugin'); module.exports = { mode: 'production', - entry: ['core-js/stable', 'regenerator-runtime/runtime', "./assets/scripts/main.js"], +// entry: ['core-js/stable', 'regenerator-runtime/runtime', "./assets/scripts/main.js"], + entry: { +// stable: 'core-js/stable', +// runtime: 'regenerator-runtime/runtime', +// main: ['core-js/stable', 'regenerator-runtime/runtime',"./assets/scripts/main.js"], + admin_criterions: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/admin_criterions.js'], + check_list: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/check_list.js'], + criterion_pack: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/criterion_pack.js'], + general: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/general.js'], + login: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/login.js'], + logs: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/logs.js'], + one_user_info: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/one_user_info.js'], + profile: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/profile.js'], + results: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/results.js'], + signup: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/signup.js'], + upload: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/upload.js'], + user_list: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/user_list.js'], + version: ['core-js/stable', 'regenerator-runtime/runtime', './assets/scripts/general_imports.js', './assets/scripts/version.js'], + }, output: { path: path.join(__dirname, './src/'), filename: "./[name].js"