juni/crystall-punk-14
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Local helper update (#420)

Browse Source 
* local helper update

* Delete entities.ftl

* Helper Refactor

* Revert "Helper Refactor"

This reverts commit 4aca315593.

* Helper Refactor

* Жееесть, я не знал про setdefault у словарей

* Update localization_helper.py

* Ревёрт "Жееесть, я не знал про setdefault у словарей"

Лучше бы я продолжал не знать о них

* чтооооо

* Update yml_parser.py

* Update entities.ftl

---------

Co-authored-by: Ed <96445749+TheShuEd@users.noreply.github.com>
This commit is contained in:
comasqw
2024-09-23 16:43:02 +04:00
committed by GitHub
parent 0456d40bef
commit 8e27ef22ed
8 changed files with 685 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Tools/_CP14/LocalizationHelper/base_parser.py
View File

@@ -10,7 +10,7 @@ class BaseParser:
def __init__(self, paths: tuple):
self.path, self.errors_path = paths
def get_files_paths(self) -> list:
def _get_files_paths(self) -> list:
"""
The method gets the path to the yml folder of localization prototypes/files, e.g. "ftl",
then with the help of os library goes through each file in
@@ -31,7 +31,7 @@ class BaseParser:
json.dump(prototypes, json_file, indent=4)
@staticmethod
def check_file_extension(path: str, extension: str) -> bool:
def _check_file_extension(path: str, extension: str) -> bool:
if path.endswith(extension):
return True
return False

6
Tools/_CP14/LocalizationHelper/fluent/ftl_reader.py
View File

@@ -4,9 +4,7 @@ def read_ftl(paths: tuple) -> dict:
file and determines by the indentation in the line whether
it is a new prototype or an attribute of an old one.
"""
prototypes = {
}
prototypes = {}
last_prototype = ""
path, error_log_path = paths
@@ -36,4 +34,4 @@ def read_ftl(paths: tuple) -> dict:
with open(error_log_path, "a") as file:
file.write(f"FTL-ERROR:\nAn error occurred while reading a file {path}, error - {e}\n")
return prototypes
return prototypes

4
Tools/_CP14/LocalizationHelper/ftl_parser/ftl_parser.py
View File

@@ -15,9 +15,9 @@ class FTLParser(BaseParser):
"""
prototypes = {}
for path in self.get_files_paths():
for path in self._get_files_paths():
if not self.check_file_extension(path, ".ftl"):
if not self._check_file_extension(path, ".ftl"):
continue
file = ftl_reader.read_ftl((path, self.errors_path))

136
Tools/_CP14/LocalizationHelper/localization_helper.py Normal file
View File

@@ -0,0 +1,136 @@
import json
import os
from yml_parser import YMLParser
from ftl_parser import FTLParser
from fluent import ftl_writer
# Config constants
CONFIG_PATH = "config.json"
CONFIG_PATHS_KEY_NAME = "paths"
PROTOTYPES_PATH_IN_CONFIG = "prototypes"
FTL_PATH_IN_CONFIG = "localization"
ERRORS_LOG_PATH_IN_CONFIG = "error_log_path"
PARSED_PROTOTYPES_PATH_IN_LAST_LAUNCH = "yml_parser_last_launch"
LAST_LAUNCH_PROTOTYPES_DIR_NAME = "last_launch"
NAME_OF_FILE_TO_SAVE = "entities.ftl"
class LocalizationHelper:
if not os.path.isdir(LAST_LAUNCH_PROTOTYPES_DIR_NAME):
os.mkdir(LAST_LAUNCH_PROTOTYPES_DIR_NAME)
def __init__(self, config_path: str):
self._config = self._read_config(config_path)
self._prototypes_path, self._localization_path, self._errors_log_path, self._yml_parser_last_launch = self._get_paths()
self._clear_logs()
self.prototypes_dict_yml = YMLParser((self._prototypes_path, self._errors_log_path)).yml_parser()
self.prototypes_dict_ftl = FTLParser((self._localization_path, self._errors_log_path)).ftl_parser()
self._check_changed_attrs()
self.prototypes = {**self.prototypes_dict_yml, **self.prototypes_dict_ftl}
def _clear_logs(self):
with open(self._errors_log_path, "w") as file:
file.write("")
@staticmethod
def _read_config(config_path: str) -> dict:
with open(config_path, "r", encoding="utf-8") as file:
return json.load(file)
def _get_paths(self) -> tuple:
paths_dict = self._config[CONFIG_PATHS_KEY_NAME]
prototypes_path = paths_dict[PROTOTYPES_PATH_IN_CONFIG]
localization_path = paths_dict[FTL_PATH_IN_CONFIG]
errors_log_path = paths_dict[ERRORS_LOG_PATH_IN_CONFIG]
yml_parser_last_launch = paths_dict[PARSED_PROTOTYPES_PATH_IN_LAST_LAUNCH]
return prototypes_path, localization_path, errors_log_path, yml_parser_last_launch
def _check_changed_attrs(self):
"""
What error it fixes - without this function, changed attributes of prototypes that have not been changed in
localization files will simply not be added to the original ftl file, because the script first of all takes data
from localization files, if they exist, of course
The function gets the data received during the last run of the script, and checks if some attribute from
the last run has been changed,then simply replaces with this attribute the attribute
of the prototype received during parsing of localization files.
"""
if os.path.isfile(self._yml_parser_last_launch):
with open(self._yml_parser_last_launch, 'r', encoding='utf-8') as file:
last_launch_prototypes = json.load(file)
if last_launch_prototypes:
for prototype, last_launch_attrs in last_launch_prototypes.items():
if prototype in self.prototypes_dict_yml:
if prototype in self.prototypes_dict_ftl:
attrs = self.prototypes_dict_ftl[prototype]
proto_attrs_in_yml = self.prototypes_dict_yml[prototype]
for key, value in proto_attrs_in_yml.items():
if value != last_launch_attrs.get(key):
attrs[key] = value
self.prototypes_dict_ftl[prototype] = attrs
else:
if prototype in self.prototypes_dict_ftl:
del self.prototypes_dict_ftl[prototype]
@staticmethod
def _save_result(entities: str) -> None:
with open(NAME_OF_FILE_TO_SAVE, "w", encoding="utf-8") as file:
file.write(entities)
print(f"{NAME_OF_FILE_TO_SAVE} has been created\n")
@staticmethod
def _print_errors_log_info(errors_log_path: str, prototypes: dict) -> None:
with open(errors_log_path, "r") as file:
errors = file.read()
successful_count = len(prototypes) - errors.count("ERROR")
print(f"""Of the {len(prototypes)} prototypes, {successful_count} were successfully processed.
Errors can be found in {errors_log_path}
Number of errors during YML processing - {errors.count("YML-ERROR")}
Number of errors during FTL processing - {errors.count("FTL-ERROR")}
Number of errors during data extraction and creation of new FTL - {errors.count("RETRIEVING-ERROR")}""")
def main(self):
entities_ftl = ""
for prototype, prototype_attrs in self.prototypes.items():
try:
# This fragment is needed to restore some attributes after connecting the dictionary of
# prototypes parsed from ftl with the dictionary of prototypes parsed from yml.
if prototype in self.prototypes_dict_yml:
parent = self.prototypes_dict_yml[prototype]["parent"]
if parent and not isinstance(parent, list) and parent in self.prototypes_dict_yml:
if not prototype_attrs.get("name"):
prototype_attrs["name"] = f"{{ ent-{parent} }}"
if not prototype_attrs.get("desc"):
prototype_attrs["desc"] = f"{{ ent-{parent}.desc }}"
if not prototype_attrs.get("suffix"):
if self.prototypes_dict_yml[prototype].get("suffix"):
prototype_attrs["suffix"] = self.prototypes_dict_yml[prototype]["suffix"]
if any(prototype_attrs[attr] is not None for attr in ("name", "desc", "suffix")):
proto_ftl = ftl_writer.create_ftl(prototype, self.prototypes[prototype])
entities_ftl += proto_ftl
except Exception as e:
with open(self._errors_log_path, "a") as file:
print(prototype, prototype_attrs)
file.write(
f"RETRIEVING-ERROR:\nAn error occurred while retrieving data to be written to the file - {e}\n")
self._save_result(entities_ftl)
self._print_errors_log_info(self._errors_log_path, self.prototypes)
if __name__ == '__main__':
helper = LocalizationHelper(CONFIG_PATH)
helper.main()

139
Tools/_CP14/LocalizationHelper/main.py
View File

@@ -1,139 +0,0 @@
import json
import os
from yml_parser import YMLParser
from ftl_parser import FTLParser
from fluent import ftl_writer
def read_config():
with open("config.json", "r", encoding="utf-8") as file:
return json.load(file)
def get_paths(config: dict):
prototypes_path = config["paths"]["prototypes"]
localization_path = config["paths"]["localization"]
errors_log_path = config["paths"]["error_log_path"]
yml_parser_last_launch = config["paths"]["yml_parser_last_launch"]
return prototypes_path, localization_path, errors_log_path, yml_parser_last_launch
def print_errors_log_info(errors_log_path: str, all_prototypes: dict) -> None:
with open(errors_log_path, "r") as file:
errors = file.read()
successful_count = len(all_prototypes) - errors.count("ERROR")
print(f"""Of the {len(all_prototypes)} prototypes, {successful_count} were successfully processed.
Errors can be found in {errors_log_path}
Number of errors during YML processing - {errors.count("YML-ERROR")}
Number of errors during FTL processing - {errors.count("FTL-ERROR")}
Number of errors during data extraction and creation of new FTL - {errors.count("RETRIEVING-ERROR")}""")
def check_changed_attrs(yml_parser_last_launch: str, prototypes_dict: dict, localization_dict: dict):
"""
What error it fixes - without this function, changed attributes of prototypes that have not been changed in
localization files will simply not be added to the original ftl file, because the script first of all takes data
from localization files, if they exist, of course
The function gets the data received during the last run of the script, and checks if some attribute from
the last run has been changed,then simply replaces with this attribute the attribute
of the prototype received during parsing of localization files.
"""
if os.path.isfile(yml_parser_last_launch):
with open(yml_parser_last_launch, 'r', encoding='utf-8') as file:
last_launch_prototypes = json.load(file)
for prototype, proto_attrs_in_prototypes in prototypes_dict.items():
if prototype in last_launch_prototypes and prototype in localization_dict:
attrs = localization_dict[prototype]
last_launch_prototype_attrs = last_launch_prototypes[prototype]
for key, value in proto_attrs_in_prototypes.items():
if value != last_launch_prototype_attrs[key]:
attrs[key] = value
localization_dict[prototype] = attrs
def save_result(entities: str, file_name: str) -> None:
with open(file_name, "w", encoding="utf-8") as file:
file.write(entities)
print(f"{file_name} has been created\n")
def main():
"""
The function gets paths, creates dictionaries with the help of parsers,
performs various checks, and finally creates ftl file.
"""
config = read_config()
prototypes_path, localization_path, errors_log_path, yml_parser_last_launch = get_paths(config)
if not os.path.isdir("last_launch"):
os.mkdir("last_launch")
with open(errors_log_path, "w") as file:
file.write("")
yml_parser = YMLParser((prototypes_path, errors_log_path))
prototypes_dict = yml_parser.yml_parser()
ftl_parser = FTLParser((localization_path, errors_log_path))
localization_dict = ftl_parser.ftl_parser()
check_changed_attrs(yml_parser_last_launch, prototypes_dict, localization_dict)
with open(yml_parser_last_launch, 'w') as json_file:
json.dump(prototypes_dict, json_file, indent=4)
# This is where the two dictionaries are merged, and prototypes from
# the localization_dict dictionary are preferably selected.
all_prototypes = {**prototypes_dict, **localization_dict}
entities_ftl = ""
"""
The function traverses each prototype from the dictionary, checks if it has a parent,
and performs certain checks on the attributes of the parent if the prototype does not have its own attributes.
"""
for prototype in all_prototypes:
prototype_attrs = all_prototypes[prototype]
try:
if prototype in prototypes_dict:
prototype_attrs["parent"] = prototypes_dict[prototype]["parent"]
parent = prototype_attrs["parent"]
if not isinstance(parent, list) and parent in prototypes_dict:
if not prototype_attrs.get("name"):
prototype_attrs["name"] = f"{{ ent-{parent} }}"
if not prototype_attrs.get("desc"):
if parent and not isinstance(parent, list) and prototypes_dict.get(parent):
prototype_attrs["desc"] = f"{{ ent-{parent}.desc }}"
if not prototype_attrs.get("suffix"):
if prototypes_dict[prototype].get("suffix"):
prototype_attrs["suffix"] = prototypes_dict[prototype]["suffix"]
if any(prototype_attrs[attr] is not None for attr in ["name", "desc", "suffix"]):
proto_ftl = ftl_writer.create_ftl(prototype, all_prototypes[prototype])
entities_ftl += proto_ftl
except Exception as e:
with open(errors_log_path, "a") as file:
print(prototype, prototype_attrs)
file.write(f"RETRIEVING-ERROR:\nAn error occurred while retrieving data to be written to the file - {e}\n")
file_name = "entities.ftl"
save_result(entities_ftl, file_name)
print_errors_log_info(errors_log_path, all_prototypes)
if __name__ == '__main__':
main()

2
Tools/_CP14/LocalizationHelper/run.bat
View File

@@ -1,3 +1,3 @@
@echo off
python main.py
python localization_helper.py
pause

59
Tools/_CP14/LocalizationHelper/yml_parser/yml_parser.py
View File

@@ -1,5 +1,6 @@
import yaml
from base_parser import BaseParser
import re
class YMLParser(BaseParser):
@@ -8,7 +9,7 @@ class YMLParser(BaseParser):
"""
@staticmethod
def check_proto_attrs(prototype: dict) -> bool:
def _check_proto_attrs(prototype: dict) -> bool:
"""
The function checks that the prototype at least has some attribute from the "attrs_lst".
"""
@@ -21,7 +22,7 @@ class YMLParser(BaseParser):
return any(prototype.get(attr) is not None for attr in attrs_lst)
@staticmethod
def get_proto_attrs(prototypes: dict, prototype: dict) -> None:
def _get_proto_attrs(prototypes: dict, prototype: dict) -> None:
prototypes[prototype.get("id")] = {
"parent": prototype.get("parent"),
"name": prototype.get("name"),
@@ -29,18 +30,29 @@ class YMLParser(BaseParser):
"suffix": prototype.get("suffix")
}
@staticmethod
def create_proto(file) -> str:
proto = ""
for line in file.readlines():
# The PyYaml library cannot handle the following SpaceStation 14 prototype syntax - !type: ...
# We need to fix this :(
if "!type" in line:
continue
proto += line
def _load_proto(self, file, path) -> list[dict]:
content_str = file.read()
prototypes_lst = re.split(r"\n(?=- type:)", content_str)
return proto
prototypes = []
for proto in prototypes_lst:
try:
prototype_str = ""
for line in proto.splitlines():
if "components:" in line:
break
prototype_str += f"{line}\n"
prototype = yaml.safe_load(prototype_str)
if prototype is None:
continue
prototypes.append(prototype[0])
except Exception as e:
with open(self.errors_path, "a") as error_file:
error_file.write(
f"YML-ERROR:\nAn error occurred during prototype processing {path}, error - {e}\n")
return prototypes
def yml_parser(self) -> dict:
"""
The function gets the path, then with the help of the os library
@@ -49,21 +61,16 @@ class YMLParser(BaseParser):
"""
prototypes = {}
for path in self.get_files_paths():
if not self.check_file_extension(path, ".yml"):
for path in self._get_files_paths():
if not self._check_file_extension(path, ".yml"):
continue
try:
with open(path, encoding="utf-8") as file:
proto = self.create_proto(file)
data = yaml.safe_load(proto)
except Exception as e:
with open(self.errors_path, "a") as file:
file.write(f"YML-ERROR:\nAn error occurred during prototype processing {path}, error - {e}\n")
else:
if data is not None:
for prototype in data:
if self.check_proto_attrs(prototype):
self.get_proto_attrs(prototypes, prototype)
with open(path, encoding="utf-8") as file:
content = self._load_proto(file, path)
if content is not None:
for prototype in content:
if self._check_proto_attrs(prototype):
self._get_proto_attrs(prototypes, prototype)
return prototypes