From c00de7fd09341860efc18d8ea2edacd438871529 Mon Sep 17 00:00:00 2001 From: comasqw <138010940+comasqw@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:09:14 +0400 Subject: [PATCH] refactor --- .../LocalizationHelper/__init__.py | 5 + .../LocalizationHelper/exceptions.py | 6 + .../LocalizationHelper/localization_helper.py | 180 ++++++++++++++++++ .../LocalizationHelper/logger.py | 43 +++++ .../LocalizationHelper/parsers/__init__.py | 3 + .../LocalizationHelper/parsers/base_parser.py | 28 +++ .../parsers/fluent/__init__.py | 3 + .../parsers/fluent/ftl_parser.py | 28 +++ .../parsers}/fluent/ftl_reader.py | 35 ++-- .../parsers/fluent/ftl_writer.py | 23 +++ .../parsers/yaml/__init__.py | 2 + .../parsers/yaml/yaml_parser.py | 29 +++ .../parsers/yaml/yaml_reader.py | 31 +++ .../LocalizationHelper/prototype.py | 39 ++++ Tools/_CP14/LocalizationHelper/README.md | 14 +- Tools/_CP14/LocalizationHelper/base_parser.py | 38 ---- Tools/_CP14/LocalizationHelper/config.json | 8 - .../LocalizationHelper/fluent/__init__.py | 0 .../LocalizationHelper/fluent/ftl_writer.py | 17 -- .../LocalizationHelper/ftl_parser/__init__.py | 1 - .../ftl_parser/ftl_parser.py | 26 --- .../LocalizationHelper/localization_helper.py | 136 ------------- .../LocalizationHelper/logs/errors_log.txt | 0 Tools/_CP14/LocalizationHelper/main.py | 11 ++ .../_CP14/LocalizationHelper/requirements.txt | 2 +- Tools/_CP14/LocalizationHelper/run.bat | 2 +- .../LocalizationHelper/yml_parser/__init__.py | 1 - .../yml_parser/yml_parser.py | 76 -------- 28 files changed, 462 insertions(+), 325 deletions(-) create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/__init__.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/exceptions.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/localization_helper.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/logger.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/__init__.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/base_parser.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/__init__.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_parser.py rename Tools/_CP14/LocalizationHelper/{ => LocalizationHelper/parsers}/fluent/ftl_reader.py (50%) create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_writer.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/__init__.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_parser.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_reader.py create mode 100644 Tools/_CP14/LocalizationHelper/LocalizationHelper/prototype.py delete mode 100644 Tools/_CP14/LocalizationHelper/base_parser.py delete mode 100644 Tools/_CP14/LocalizationHelper/config.json delete mode 100644 Tools/_CP14/LocalizationHelper/fluent/__init__.py delete mode 100644 Tools/_CP14/LocalizationHelper/fluent/ftl_writer.py delete mode 100644 Tools/_CP14/LocalizationHelper/ftl_parser/__init__.py delete mode 100644 Tools/_CP14/LocalizationHelper/ftl_parser/ftl_parser.py delete mode 100644 Tools/_CP14/LocalizationHelper/localization_helper.py delete mode 100644 Tools/_CP14/LocalizationHelper/logs/errors_log.txt create mode 100644 Tools/_CP14/LocalizationHelper/main.py delete mode 100644 Tools/_CP14/LocalizationHelper/yml_parser/__init__.py delete mode 100644 Tools/_CP14/LocalizationHelper/yml_parser/yml_parser.py diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/__init__.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/__init__.py new file mode 100644 index 0000000000..949488486e --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/__init__.py @@ -0,0 +1,5 @@ +from .logger import get_logger, LogText +from .exceptions import * +from .parsers import YamlParser, FtlParser +from .prototype import Prototype, check_prototype_attrs +from .localization_helper import LocalizationHelper diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/exceptions.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/exceptions.py new file mode 100644 index 0000000000..97abf5c2ba --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/exceptions.py @@ -0,0 +1,6 @@ +class ErrorWhileWritingToFile(Exception): + pass + + +class ErrorWhileReadingFromFile(Exception): + pass diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/localization_helper.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/localization_helper.py new file mode 100644 index 0000000000..bc2e21feeb --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/localization_helper.py @@ -0,0 +1,180 @@ +import json +import os + +from . import Prototype, check_prototype_attrs, get_logger, LogText, ErrorWhileWritingToFile, ErrorWhileReadingFromFile +from .parsers import FtlParser, YamlParser, create_ftl + +logger = get_logger(__name__) + +SAVE_RESULT_TO = "entities.ftl" +YAML_PARSER_LAST_LAUNCH_RESULT_PATH = "last_launch_result/result.json" + + +class LocalizationHelper: + + def __init__(self): + logger.debug("%s LocalizationHelper ", LogText.CLASS_INITIALIZATION) + + @staticmethod + def _save_to_json(path: str, data: dict): + os.makedirs(os.path.dirname(path), exist_ok=True) + try: + logger.debug("%s: %s", LogText.SAVING_DATA_TO_FILE, path) + with open(path, "w", encoding="utf-8") as file: + json.dump(data, file, ensure_ascii=False, indent=4) + except Exception as e: + raise ErrorWhileWritingToFile(e) + + @staticmethod + def _read_from_json(path: str) -> dict: + if os.path.exists(path): + try: + logger.debug("%s: %s", LogText.READING_DATA_FROM_FILE, path) + with open(path, encoding="utf-8") as file: + return json.load(file) + except Exception as e: + raise ErrorWhileReadingFromFile(e) + return {} + + def _save_yaml_parser_last_launch_result(self, last_launch_result: dict[str, Prototype]): + logger.debug("%s %s", LogText.SAVING_LAST_LAUNCH_RESULT, YAML_PARSER_LAST_LAUNCH_RESULT_PATH) + + prototypes_dict = {} + for prototype_id, prototype_obj in last_launch_result.items(): + prototypes_dict[prototype_id] = prototype_obj.attrs_dict + + self._save_to_json(YAML_PARSER_LAST_LAUNCH_RESULT_PATH, prototypes_dict) + + def _read_prototypes_from_last_launch_result(self) -> dict[str, Prototype] | None: + if os.path.isfile(YAML_PARSER_LAST_LAUNCH_RESULT_PATH): + last_launch_result = self._read_from_json(YAML_PARSER_LAST_LAUNCH_RESULT_PATH) + last_launch_result_dict = {} + for prototype_id, prototype_attrs in last_launch_result.items(): + last_launch_result_dict[prototype_id] = Prototype(prototype_attrs) + + return last_launch_result_dict + return None + + @staticmethod + def _update_prototype_if_attrs_has_been_changed(yaml_prototype_obj: Prototype, last_launch_prototype_obj: Prototype, + final_prototype_obj: Prototype): + if yaml_prototype_obj.attrs_dict != last_launch_prototype_obj.attrs_dict: + log_text = f"Has been updated from: {final_prototype_obj.attrs_dict}, to: " + + final_prototype_obj.attrs_dict.update(yaml_prototype_obj.attrs_dict) + final_prototype_obj.reinitialize_with_attrs_dict() + + log_text += f"{final_prototype_obj.attrs_dict}" + logger.debug(log_text) + + return final_prototype_obj + + def _merge_yaml_parser_prototypes_and_ftl_parser_prototypes(self, yaml_parser_prototypes: dict[str, Prototype], + ftl_parser_prototypes: dict[str, Prototype]) -> dict[str, Prototype]: + + general_prototypes_dict = {} + + last_launch_result = self._read_prototypes_from_last_launch_result() + + for prototype_id, yaml_prototype_obj in yaml_parser_prototypes.items(): + final_prototype_obj = yaml_prototype_obj + if prototype_id in ftl_parser_prototypes: + final_prototype_obj = ftl_parser_prototypes[prototype_id] + + if last_launch_result and prototype_id in last_launch_result: + last_launch_prototype_obj = last_launch_result[prototype_id] + final_prototype_obj = self._update_prototype_if_attrs_has_been_changed(yaml_prototype_obj, + last_launch_prototype_obj, + final_prototype_obj) + general_prototypes_dict[prototype_id] = final_prototype_obj + + return general_prototypes_dict + + @staticmethod + def _set_parent_attrs(prototype_parent_id: str, prototype_obj: Prototype, parent_prototype_obj: Prototype): + for attr_name, attr_value in prototype_obj.attrs_dict.items(): + if attr_value or attr_name in ("parent", "id"): + continue + + parent_prototype_attr_value = parent_prototype_obj.attrs_dict.get(attr_name) + if parent_prototype_attr_value: + if attr_name == "name": + prototype_obj.name = f"{{ ent-{prototype_parent_id} }}" + elif attr_name == "description": + prototype_obj.description = f"{{ ent-{prototype_parent_id}.desc }}" + elif attr_name == "suffix": + prototype_obj.suffix = parent_prototype_attr_value + + return prototype_obj + + def _parent_checks(self, general_prototypes_dict: dict[str, Prototype]): + to_delete = [] + for prototype_id, prototype_obj in general_prototypes_dict.items(): + prototype_parent_id = prototype_obj.parent + if isinstance(prototype_parent_id, list): + continue + + parent_prototype_obj = general_prototypes_dict.get(prototype_parent_id) + + if parent_prototype_obj and check_prototype_attrs(parent_prototype_obj, False): + self._set_parent_attrs(prototype_parent_id, prototype_obj, parent_prototype_obj) + else: + if not check_prototype_attrs(prototype_obj, False): + to_delete.append(prototype_id) + + for prototype_id in to_delete: + logger.debug("%s %s: %s", prototype_id, LogText.HAS_BEEN_DELETED, general_prototypes_dict[prototype_id]) + del general_prototypes_dict[prototype_id] + + return general_prototypes_dict + + def _create_general_prototypes_dict(self, yaml_parser_prototypes: dict[str, Prototype], + ftl_parser_prototypes: dict[str, Prototype]) -> dict[str, Prototype]: + general_prototypes_dict = self._merge_yaml_parser_prototypes_and_ftl_parser_prototypes(yaml_parser_prototypes, + ftl_parser_prototypes) + general_prototypes_dict = self._parent_checks(general_prototypes_dict) + + return general_prototypes_dict + + @staticmethod + def _create_result_ftl(general_prototypes_dict: dict[str, Prototype]) -> str: + result = "" + for prototype_obj in general_prototypes_dict.values(): + result += create_ftl(prototype_obj) + return result + + def _save_result(self, general_prototypes_dict: dict[str, Prototype]): + logger.debug("%s: %s", LogText.SAVING_FINAL_RESULT, SAVE_RESULT_TO) + result = self._create_result_ftl(general_prototypes_dict) + try: + with open(SAVE_RESULT_TO, "w", encoding="utf-8") as file: + file.write(result) + except Exception as e: + raise ErrorWhileWritingToFile(e) + + @staticmethod + def _print_info(general_prototypes_dict): + logger.info("%s: %s prototypes", LogText.HAS_BEEN_PROCESSED, len(general_prototypes_dict)) + logger.info("Logs in: 'logs/helper.log'") + + def main(self, yaml_prototypes_path: str, ftl_prototypes_path: str): + try: + logger.debug("%s: %s", LogText.GETTING_PROTOTYPES_FROM_YAML, yaml_prototypes_path) + prototypes_list_parsed_from_yaml = YamlParser().get_prototypes(yaml_prototypes_path) + + logger.debug("%s: %s", LogText.GETTING_PROTOTYPES_FROM_FTL, ftl_prototypes_path) + prototypes_list_parsed_from_ftl = FtlParser().get_prototypes(ftl_prototypes_path) + + logger.debug(LogText.FORMING_FINAL_DICTIONARY) + general_prototypes_dict = self._create_general_prototypes_dict(prototypes_list_parsed_from_yaml, + prototypes_list_parsed_from_ftl) + + self._save_yaml_parser_last_launch_result(prototypes_list_parsed_from_yaml) + self._save_result(general_prototypes_dict) + self._print_info(general_prototypes_dict) + except ErrorWhileWritingToFile as e: + logger.error("%s: %s", LogText.ERROR_WHILE_WRITING_DATA_TO_FILE, e, exc_info=True) + except ErrorWhileReadingFromFile as e: + logger.error("%s: %s", LogText.ERROR_WHILE_READING_DATA_FROM_FILE, e, exc_info=True) + except Exception as e: + logger.error("%s: %s", LogText.UNKNOWN_ERROR, e, exc_info=True) diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/logger.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/logger.py new file mode 100644 index 0000000000..5451f431a6 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/logger.py @@ -0,0 +1,43 @@ +import logging +import os + + +LOG_FILE_PATH = os.path.join("logs", "helper.log") +os.makedirs(os.path.dirname(LOG_FILE_PATH), exist_ok=True) + +file_handler = logging.FileHandler(LOG_FILE_PATH, mode='w', encoding="utf-8") +file_handler.setLevel(logging.DEBUG) + +console_handler = logging.StreamHandler() +console_handler.setLevel(logging.INFO) + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[ + file_handler, + console_handler + ] +) + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(name) + + +class LogText: + CLASS_INITIALIZATION = "initializing" + GETTING_PROTOTYPES_FROM_YAML = "Getting prototypes from YAML files" + GETTING_PROTOTYPES_FROM_FTL = "Getting prototypes from FTL files" + FORMING_FINAL_DICTIONARY = "Forming the final prototypes dictionary" + SAVING_LAST_LAUNCH_RESULT = "Saving yaml_parser result to file" + READING_LAST_LAUNCH_RESULT = "Reading data from last launch" + SAVING_FINAL_RESULT = "Saving the final prototypes dictionary to FTL file" + SAVING_DATA_TO_FILE = "Saving data to file" + READING_DATA_FROM_FILE = "Reading data from file" + ERROR_WHILE_WRITING_DATA_TO_FILE = "Error while writing data to a file" + ERROR_WHILE_READING_DATA_FROM_FILE = "Error while writing data to a file" + UNKNOWN_ERROR = "An error occurred during execution" + HAS_BEEN_DELETED = "Has been deleted due to lack of relevant data" + HAS_BEEN_PROCESSED = "Has been processed" + FORMING_FTL_FOR_PROTOTYPE = "Forming FTL for Prototype" diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/__init__.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/__init__.py new file mode 100644 index 0000000000..7ce3caba88 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/__init__.py @@ -0,0 +1,3 @@ +from .base_parser import BaseParser +from .yaml import YamlParser +from .fluent import FtlParser, create_ftl diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/base_parser.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/base_parser.py new file mode 100644 index 0000000000..d6ea34ec5a --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/base_parser.py @@ -0,0 +1,28 @@ +import os +from abc import ABC, abstractmethod + + +from LocalizationHelper.prototype import Prototype + + +class BaseParser(ABC): + @staticmethod + def _get_files_paths_in_dir(path: str) -> list[str]: + files_paths_lst = [] + + for dirpath, _, filenames in os.walk(path): + for filename in filenames: + file_path = f"{dirpath}\\{filename}" + files_paths_lst.append(file_path) + + return files_paths_lst + + @staticmethod + def _check_file_extension(path: str, extension: str) -> bool: + if path.endswith(extension): + return True + return False + + @abstractmethod + def get_prototypes(self, prototypes_files_path: str) -> list[Prototype]: + pass diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/__init__.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/__init__.py new file mode 100644 index 0000000000..0bde5d4edb --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/__init__.py @@ -0,0 +1,3 @@ +from .ftl_reader import read_ftl +from .ftl_parser import FtlParser +from .ftl_writer import create_ftl diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_parser.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_parser.py new file mode 100644 index 0000000000..3deab4f0b6 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_parser.py @@ -0,0 +1,28 @@ +from . import read_ftl +from LocalizationHelper import get_logger, LogText +from LocalizationHelper.prototype import Prototype +from LocalizationHelper.parsers import BaseParser + +logger = get_logger(__name__) + + +class FtlParser(BaseParser): + def __init__(self): + logger.debug("%s FtlParser", LogText.CLASS_INITIALIZATION) + + def get_prototypes(self, ftl_prototypes_path: str) -> dict[str, Prototype]: + prototypes = {} + ftl_prototypes_files_path = self._get_files_paths_in_dir(ftl_prototypes_path) + + for prototype_file_path in ftl_prototypes_files_path: + if not self._check_file_extension(prototype_file_path, "ftl"): + continue + + file_prototypes_dict = read_ftl(prototype_file_path) + + for prototype_dict in file_prototypes_dict.values(): + prototype_obj = Prototype(prototype_dict) + logger.debug("%s: %s", LogText.HAS_BEEN_PROCESSED, prototype_obj) + prototypes[prototype_obj.id] = prototype_obj + + return prototypes diff --git a/Tools/_CP14/LocalizationHelper/fluent/ftl_reader.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_reader.py similarity index 50% rename from Tools/_CP14/LocalizationHelper/fluent/ftl_reader.py rename to Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_reader.py index 8935763cc7..999cfc1119 100644 --- a/Tools/_CP14/LocalizationHelper/fluent/ftl_reader.py +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_reader.py @@ -1,14 +1,15 @@ -def read_ftl(paths: tuple) -> dict: - """ - The function looks at each line of the ftl - file and determines by the indentation in the line whether - it is a new prototype or an attribute of an old one. - """ +from LocalizationHelper import get_logger, LogText + +logger = get_logger(__name__) + + +def read_ftl(path: str) -> dict: + prototypes = {} last_prototype = "" - path, error_log_path = paths try: + logger.debug("%s: %s", LogText.READING_DATA_FROM_FILE, path) with open(path, encoding="utf-8") as file: for line in file.readlines(): if line.startswith("#") or line.startswith("\n"): @@ -19,19 +20,19 @@ def read_ftl(paths: tuple) -> dict: proto_id = proto_id.replace("ent-", "") last_prototype = proto_id prototypes[proto_id] = { - "name": proto_name.strip(), - "desc": None, - "suffix": None - } + "id": proto_id, + "name": proto_name.strip(), + "description": None, + "suffix": None + } else: if "desc" in line: - attr = "desc" + attr = "description" elif "suffix" in line: attr = "suffix" - prototypes[last_prototype][attr] = line.split(" = ")[-1].strip() + prototypes[last_prototype][attr] = line.split(" = ", 1)[1].strip() except Exception as e: - with open(error_log_path, "a") as file: - file.write(f"FTL-ERROR:\nAn error occurred while reading a file {path}, error - {e}\n") - - return prototypes + logger.error("%s: %s - %s", LogText.ERROR_WHILE_READING_DATA_FROM_FILE, path, e, exc_info=True) + else: + return prototypes diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_writer.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_writer.py new file mode 100644 index 0000000000..16bac25945 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/fluent/ftl_writer.py @@ -0,0 +1,23 @@ +from LocalizationHelper import get_logger, LogText +from LocalizationHelper.prototype import Prototype + +logger = get_logger(__name__) + +INDENT = " " + + +def create_ftl(prototype: Prototype) -> str: + logger.debug("%s: %s", LogText.FORMING_FTL_FOR_PROTOTYPE, prototype.attrs_dict) + ftl = "" + + ftl += f"ent-{prototype.id} = {prototype.name}\n" + + if prototype.description: + ftl += f"{INDENT}.desc = {prototype.description}\n" + + if prototype.suffix: + ftl += f"{INDENT}.suffix = {prototype.suffix}\n" + + ftl += "\n" + + return ftl diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/__init__.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/__init__.py new file mode 100644 index 0000000000..2064a8eda0 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/__init__.py @@ -0,0 +1,2 @@ +from .yaml_reader import read_yaml +from .yaml_parser import YamlParser diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_parser.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_parser.py new file mode 100644 index 0000000000..cbe9a17bd0 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_parser.py @@ -0,0 +1,29 @@ +from . import read_yaml +from LocalizationHelper import get_logger, LogText +from LocalizationHelper.prototype import Prototype, check_prototype_attrs +from LocalizationHelper.parsers import BaseParser + +logger = get_logger(__name__) + + +class YamlParser(BaseParser): + def __init__(self): + logger.debug("%s YamlParser", LogText.CLASS_INITIALIZATION) + + def get_prototypes(self, yaml_prototypes_path: str) -> dict[str, Prototype]: + prototypes = {} + yaml_prototypes_files_path = self._get_files_paths_in_dir(yaml_prototypes_path) + + for prototype_file_path in yaml_prototypes_files_path: + if not self._check_file_extension(prototype_file_path, "yml"): + continue + + file_prototypes_list: list[dict] = read_yaml(prototype_file_path) + if file_prototypes_list: + for prototype_dict in file_prototypes_list: + prototype_obj = Prototype(prototype_dict) + if check_prototype_attrs(prototype_obj): + logger.debug("%s: %s", LogText.HAS_BEEN_PROCESSED, prototype_obj) + prototypes[prototype_obj.id] = prototype_obj + + return prototypes diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_reader.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_reader.py new file mode 100644 index 0000000000..ebc596d954 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/parsers/yaml/yaml_reader.py @@ -0,0 +1,31 @@ +import yaml + +from LocalizationHelper import get_logger, LogText + +logger = get_logger(__name__) + +# taken from - https://github.com/poeMota/map-converter-ss14/blob/master/src/Yaml/yaml.py +def unknown_tag_constructor(loader, tag_suffix, node): + if isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) # Node + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node) # List + elif isinstance(node, yaml.MappingNode): + value = loader.construct_mapping(node) # Dict + else: + raise TypeError(f"Unknown node type: {type(node)}") + + return {tag_suffix: value} + + +yaml.add_multi_constructor('!', unknown_tag_constructor) + + +def read_yaml(path: str) -> list[dict]: + try: + logger.debug("%s: %s", LogText.READING_DATA_FROM_FILE, path) + with open(path, encoding="utf-8") as file: + data = yaml.full_load(file) + return data + except Exception as e: + logger.error("%s: %s - %s", LogText.ERROR_WHILE_READING_DATA_FROM_FILE, path, e, exc_info=True) diff --git a/Tools/_CP14/LocalizationHelper/LocalizationHelper/prototype.py b/Tools/_CP14/LocalizationHelper/LocalizationHelper/prototype.py new file mode 100644 index 0000000000..ffdb17d75f --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/LocalizationHelper/prototype.py @@ -0,0 +1,39 @@ +class Prototype: + def __init__(self, prototype: dict): + self._initialize_attrs(prototype) + + def _initialize_attrs(self, prototype: dict): + self.name = prototype.get("name") + self.description = prototype.get("description") + self.parent = prototype.get("parent") + self.id = prototype.get("id") + self.suffix = prototype.get("suffix") + self.attrs_dict = { + "id": self.id, + "name": self.name, + "description": self.description, + "parent": self.parent, + "suffix": self.suffix + } + + def reinitialize_with_attrs_dict(self): + self._initialize_attrs(self.attrs_dict) + + def __repr__(self): + return str(self.attrs_dict) + + +def check_prototype_attrs(prototype: Prototype, with_parent_check: bool = True) -> bool: + + if prototype.name: + return True + elif prototype.description: + return True + elif prototype.suffix: + return True + # In some cases a parent can be a list (because of multiple parents), + # the game will not be able to handle such cases in ftl files. + elif with_parent_check and prototype.parent and not isinstance(prototype.parent, list): + return True + + return False diff --git a/Tools/_CP14/LocalizationHelper/README.md b/Tools/_CP14/LocalizationHelper/README.md index c55bdd79a1..fa1f23df30 100644 --- a/Tools/_CP14/LocalizationHelper/README.md +++ b/Tools/_CP14/LocalizationHelper/README.md @@ -6,7 +6,7 @@ A script to help support game translation ## The right version of python: 3.10+ -## Deployment +## Dependencies Install in console @@ -31,10 +31,18 @@ or just run run.bat ``` -## Demo +## Tests +in console +```bash +python run_tests.py +``` +or just run + +```bash +tests.bat +``` -https://youtu.be/5HfDjLzhjA4 ## License Author: asqw: Discord - .asqw, GitHub - comasqw diff --git a/Tools/_CP14/LocalizationHelper/base_parser.py b/Tools/_CP14/LocalizationHelper/base_parser.py deleted file mode 100644 index f9b2d7ddb8..0000000000 --- a/Tools/_CP14/LocalizationHelper/base_parser.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import json - - -class BaseParser: - """ - BaseParser, contains the basic functions for the yml_parser module in the yml_parser package - and for the ftl_parser module in the ftl_parser package - """ - def __init__(self, paths: tuple): - self.path, self.errors_path = paths - - def _get_files_paths(self) -> list: - """ - The method gets the path to the yml folder of localization prototypes/files, e.g. "ftl", - then with the help of os library goes through each file in - the folder and creates a path for it, e.g. "ftl/objects.ftl". - """ - files_paths_lst = [] - - for dirpath, _, filenames in os.walk(self.path): - for filename in filenames: - path = f"{dirpath}\\{filename}" - files_paths_lst.append(path) - - return files_paths_lst - - @staticmethod - def save_to_json(prototypes: dict, path: str) -> None: - with open(path, 'w') as json_file: - json.dump(prototypes, json_file, indent=4) - - @staticmethod - def _check_file_extension(path: str, extension: str) -> bool: - if path.endswith(extension): - return True - return False - diff --git a/Tools/_CP14/LocalizationHelper/config.json b/Tools/_CP14/LocalizationHelper/config.json deleted file mode 100644 index 9e2e0f6d81..0000000000 --- a/Tools/_CP14/LocalizationHelper/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "paths": { - "prototypes": "../../../Resources/Prototypes/_CP14/Entities", - "localization": "../../../Resources/Locale/ru-RU/_CP14/_PROTO/entities", - "error_log_path": "logs/errors.log", - "yml_parser_last_launch": "last_launch/yml_parser.json" - } -} \ No newline at end of file diff --git a/Tools/_CP14/LocalizationHelper/fluent/__init__.py b/Tools/_CP14/LocalizationHelper/fluent/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Tools/_CP14/LocalizationHelper/fluent/ftl_writer.py b/Tools/_CP14/LocalizationHelper/fluent/ftl_writer.py deleted file mode 100644 index 920db10693..0000000000 --- a/Tools/_CP14/LocalizationHelper/fluent/ftl_writer.py +++ /dev/null @@ -1,17 +0,0 @@ -import json - - -def create_ftl(key: str, prototype: dict) -> str: - ftl = "" - - ftl += f"ent-{key} = {prototype["name"]}\n" - - if prototype["desc"] is not None: - ftl += f" .desc = {prototype["desc"]}\n" - - if prototype["suffix"] is not None: - ftl += f" .suffix = {prototype["suffix"]}\n" - - ftl += "\n" - - return ftl diff --git a/Tools/_CP14/LocalizationHelper/ftl_parser/__init__.py b/Tools/_CP14/LocalizationHelper/ftl_parser/__init__.py deleted file mode 100644 index 1e22c86d9d..0000000000 --- a/Tools/_CP14/LocalizationHelper/ftl_parser/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .ftl_parser import FTLParser diff --git a/Tools/_CP14/LocalizationHelper/ftl_parser/ftl_parser.py b/Tools/_CP14/LocalizationHelper/ftl_parser/ftl_parser.py deleted file mode 100644 index 1040986e94..0000000000 --- a/Tools/_CP14/LocalizationHelper/ftl_parser/ftl_parser.py +++ /dev/null @@ -1,26 +0,0 @@ -from fluent import ftl_reader -from base_parser import BaseParser - - -class FTLParser(BaseParser): - """ - The class inherits from the "BaseParser" class, parses ftl files of localization. - """ - - def ftl_parser(self) -> dict: - """ - The function gets the path, then with the help of the os library - goes through each file,checks that the file extension is "ftl", - then reads it through the "ftl_reader" module of the "fluent" package. - """ - prototypes = {} - - for path in self._get_files_paths(): - - if not self._check_file_extension(path, ".ftl"): - continue - - file = ftl_reader.read_ftl((path, self.errors_path)) - prototypes.update(file) - - return prototypes diff --git a/Tools/_CP14/LocalizationHelper/localization_helper.py b/Tools/_CP14/LocalizationHelper/localization_helper.py deleted file mode 100644 index 2f633654a6..0000000000 --- a/Tools/_CP14/LocalizationHelper/localization_helper.py +++ /dev/null @@ -1,136 +0,0 @@ -import json -import os -from yml_parser import YMLParser -from ftl_parser import FTLParser -from fluent import ftl_writer - -# Config constants -CONFIG_PATH = "config.json" -CONFIG_PATHS_KEY_NAME = "paths" -PROTOTYPES_PATH_IN_CONFIG = "prototypes" -FTL_PATH_IN_CONFIG = "localization" -ERRORS_LOG_PATH_IN_CONFIG = "error_log_path" -PARSED_PROTOTYPES_PATH_IN_LAST_LAUNCH = "yml_parser_last_launch" - -LAST_LAUNCH_PROTOTYPES_DIR_NAME = "last_launch" -NAME_OF_FILE_TO_SAVE = "entities.ftl" - - -class LocalizationHelper: - - if not os.path.isdir(LAST_LAUNCH_PROTOTYPES_DIR_NAME): - os.mkdir(LAST_LAUNCH_PROTOTYPES_DIR_NAME) - - def __init__(self, config_path: str): - self._config = self._read_config(config_path) - self._prototypes_path, self._localization_path, self._errors_log_path, self._yml_parser_last_launch = self._get_paths() - self._clear_logs() - self.prototypes_dict_yml = YMLParser((self._prototypes_path, self._errors_log_path)).yml_parser() - self.prototypes_dict_ftl = FTLParser((self._localization_path, self._errors_log_path)).ftl_parser() - self._check_changed_attrs() - self.prototypes = {**self.prototypes_dict_yml, **self.prototypes_dict_ftl} - - def _clear_logs(self): - with open(self._errors_log_path, "w") as file: - file.write("") - - @staticmethod - def _read_config(config_path: str) -> dict: - with open(config_path, "r", encoding="utf-8") as file: - return json.load(file) - - def _get_paths(self) -> tuple: - paths_dict = self._config[CONFIG_PATHS_KEY_NAME] - prototypes_path = paths_dict[PROTOTYPES_PATH_IN_CONFIG] - localization_path = paths_dict[FTL_PATH_IN_CONFIG] - errors_log_path = paths_dict[ERRORS_LOG_PATH_IN_CONFIG] - yml_parser_last_launch = paths_dict[PARSED_PROTOTYPES_PATH_IN_LAST_LAUNCH] - return prototypes_path, localization_path, errors_log_path, yml_parser_last_launch - - def _check_changed_attrs(self): - """ - - What error it fixes - without this function, changed attributes of prototypes that have not been changed in - localization files will simply not be added to the original ftl file, because the script first of all takes data - from localization files, if they exist, of course - - The function gets the data received during the last run of the script, and checks if some attribute from - the last run has been changed,then simply replaces with this attribute the attribute - of the prototype received during parsing of localization files. - """ - if os.path.isfile(self._yml_parser_last_launch): - with open(self._yml_parser_last_launch, 'r', encoding='utf-8') as file: - last_launch_prototypes = json.load(file) - - if last_launch_prototypes: - for prototype, last_launch_attrs in last_launch_prototypes.items(): - if prototype in self.prototypes_dict_yml: - if prototype in self.prototypes_dict_ftl: - attrs = self.prototypes_dict_ftl[prototype] - proto_attrs_in_yml = self.prototypes_dict_yml[prototype] - - for key, value in proto_attrs_in_yml.items(): - if value != last_launch_attrs.get(key): - attrs[key] = value - - self.prototypes_dict_ftl[prototype] = attrs - else: - if prototype in self.prototypes_dict_ftl: - del self.prototypes_dict_ftl[prototype] - - @staticmethod - def _save_result(entities: str) -> None: - with open(NAME_OF_FILE_TO_SAVE, "w", encoding="utf-8") as file: - file.write(entities) - - print(f"{NAME_OF_FILE_TO_SAVE} has been created\n") - - @staticmethod - def _print_errors_log_info(errors_log_path: str, prototypes: dict) -> None: - with open(errors_log_path, "r") as file: - errors = file.read() - - successful_count = len(prototypes) - errors.count("ERROR") - print(f"""Of the {len(prototypes)} prototypes, {successful_count} were successfully processed. - - Errors can be found in {errors_log_path} - Number of errors during YML processing - {errors.count("YML-ERROR")} - Number of errors during FTL processing - {errors.count("FTL-ERROR")} - Number of errors during data extraction and creation of new FTL - {errors.count("RETRIEVING-ERROR")}""") - - def main(self): - entities_ftl = "" - for prototype, prototype_attrs in self.prototypes.items(): - try: - # This fragment is needed to restore some attributes after connecting the dictionary of - # prototypes parsed from ftl with the dictionary of prototypes parsed from yml. - if prototype in self.prototypes_dict_yml: - parent = self.prototypes_dict_yml[prototype]["parent"] - - if parent and not isinstance(parent, list) and parent in self.prototypes_dict_yml: - if not prototype_attrs.get("name"): - prototype_attrs["name"] = f"{{ ent-{parent} }}" - - if not prototype_attrs.get("desc"): - prototype_attrs["desc"] = f"{{ ent-{parent}.desc }}" - - if not prototype_attrs.get("suffix"): - if self.prototypes_dict_yml[prototype].get("suffix"): - prototype_attrs["suffix"] = self.prototypes_dict_yml[prototype]["suffix"] - - if any(prototype_attrs[attr] is not None for attr in ("name", "desc", "suffix")): - proto_ftl = ftl_writer.create_ftl(prototype, self.prototypes[prototype]) - entities_ftl += proto_ftl - except Exception as e: - with open(self._errors_log_path, "a") as file: - print(prototype, prototype_attrs) - file.write( - f"RETRIEVING-ERROR:\nAn error occurred while retrieving data to be written to the file - {e}\n") - - self._save_result(entities_ftl) - self._print_errors_log_info(self._errors_log_path, self.prototypes) - - -if __name__ == '__main__': - helper = LocalizationHelper(CONFIG_PATH) - helper.main() diff --git a/Tools/_CP14/LocalizationHelper/logs/errors_log.txt b/Tools/_CP14/LocalizationHelper/logs/errors_log.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Tools/_CP14/LocalizationHelper/main.py b/Tools/_CP14/LocalizationHelper/main.py new file mode 100644 index 0000000000..3f0ea70bb1 --- /dev/null +++ b/Tools/_CP14/LocalizationHelper/main.py @@ -0,0 +1,11 @@ +from LocalizationHelper import get_logger, LocalizationHelper + +YAML_FILES_PATH = "../../../Resources/Prototypes/_CP14/Entities" +FTL_FILES_PATH = "../../../Resources/Locale/ru-RU/_CP14/_PROTO/entities" + +if __name__ == '__main__': + logger = get_logger(__name__) + + logger.info("Starting...") + helper = LocalizationHelper() + helper.main(YAML_FILES_PATH, FTL_FILES_PATH) diff --git a/Tools/_CP14/LocalizationHelper/requirements.txt b/Tools/_CP14/LocalizationHelper/requirements.txt index 036b1ca276..297ff96869 100644 --- a/Tools/_CP14/LocalizationHelper/requirements.txt +++ b/Tools/_CP14/LocalizationHelper/requirements.txt @@ -1 +1 @@ -PyYAML==6.0.1 \ No newline at end of file +PyYAML~=6.0.2 \ No newline at end of file diff --git a/Tools/_CP14/LocalizationHelper/run.bat b/Tools/_CP14/LocalizationHelper/run.bat index e8650b26f5..c083a40942 100644 --- a/Tools/_CP14/LocalizationHelper/run.bat +++ b/Tools/_CP14/LocalizationHelper/run.bat @@ -1,3 +1,3 @@ @echo off -python localization_helper.py +python main.py pause diff --git a/Tools/_CP14/LocalizationHelper/yml_parser/__init__.py b/Tools/_CP14/LocalizationHelper/yml_parser/__init__.py deleted file mode 100644 index 0fbb5dfb7f..0000000000 --- a/Tools/_CP14/LocalizationHelper/yml_parser/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .yml_parser import YMLParser diff --git a/Tools/_CP14/LocalizationHelper/yml_parser/yml_parser.py b/Tools/_CP14/LocalizationHelper/yml_parser/yml_parser.py deleted file mode 100644 index 34be5fb07c..0000000000 --- a/Tools/_CP14/LocalizationHelper/yml_parser/yml_parser.py +++ /dev/null @@ -1,76 +0,0 @@ -import yaml -from base_parser import BaseParser -import re - - -class YMLParser(BaseParser): - """ - The class inherits from the "BaseParser" class, parses yml prototypes. - """ - - @staticmethod - def _check_proto_attrs(prototype: dict) -> bool: - """ - The function checks that the prototype at least has some attribute from the "attrs_lst". - """ - attrs_lst = ["name", "description", "suffix"] - # In some cases a parent can be a list (because of multiple parents), - # the game will not be able to handle such cases in ftl files. - if not isinstance(prototype.get("parent"), list): - attrs_lst.append("parent") - - return any(prototype.get(attr) is not None for attr in attrs_lst) - - @staticmethod - def _get_proto_attrs(prototypes: dict, prototype: dict) -> None: - prototypes[prototype.get("id")] = { - "parent": prototype.get("parent"), - "name": prototype.get("name"), - "desc": prototype.get("description"), - "suffix": prototype.get("suffix") - } - - def _load_proto(self, file, path) -> list[dict]: - content_str = file.read() - prototypes_lst = re.split(r"\n(?=- type:)", content_str) - - prototypes = [] - for proto in prototypes_lst: - try: - prototype_str = "" - for line in proto.splitlines(): - if "components:" in line: - break - prototype_str += f"{line}\n" - prototype = yaml.safe_load(prototype_str) - if prototype is None: - continue - prototypes.append(prototype[0]) - except Exception as e: - with open(self.errors_path, "a") as error_file: - error_file.write( - f"YML-ERROR:\nAn error occurred during prototype processing {path}, error - {e}\n") - - return prototypes - - def yml_parser(self) -> dict: - """ - The function gets the path, then with the help of the os library - goes through each file,checks that the file extension is "yml", - then processes the file using the "PyYaml" library - """ - prototypes = {} - - for path in self._get_files_paths(): - if not self._check_file_extension(path, ".yml"): - continue - - with open(path, encoding="utf-8") as file: - content = self._load_proto(file, path) - - if content is not None: - for prototype in content: - if self._check_proto_attrs(prototype): - self._get_proto_attrs(prototypes, prototype) - - return prototypes