refactor: convert all task-intensive functions to async.

chore: rename gui.py to cli.py
fix: internal downloader can resume download now.
feat: add verify_game, verify_from_pkg_version, clear_cache to installer.py.
feat: add clear_cache to patcher.py.
fix: linux now check for pkexec before executing it.
fix: add get_name to voicepack.py, latest.py, diff.py to get name from path (since the developer didn't set a name to these files in the sdk url)
chore: remove deprecation message in read_version_from_config in installer.py
misc: use chunk from self._download_chunk instead of being hardcoded to 8192.
fix: is_telemetry_blocked will only wait 15s for a connection.
chore: move appdirs to constants.py

This commit refactor almost all functions to be compatible with asyncio, also restructured CLI to use asyncio.run on main function instead of executing it randomly.
Also prioritize the use of asyncio.gather, sometimes making tasks faster
This commit is contained in:
2022-06-25 01:13:47 +07:00
parent 8b2d0cad8f
commit a5659f7ff3
14 changed files with 515 additions and 406 deletions

View File

@ -3,10 +3,9 @@ import re
import shutil
import platform
import aiohttp
import appdirs
import zipfile
import warnings
import json
import hashlib
from pathlib import Path
from configparser import ConfigParser
from aiopath import AsyncPath
@ -23,19 +22,21 @@ async def _download_file(file_url: str, file_name: str, file_path: Path | str, f
:param file_name:
:return:
"""
params = {}
headers = {}
file_path = AsyncPath(file_path).joinpath(file_name)
if overwrite:
await file_path.unlink(missing_ok=True)
if await file_path.exists():
cur_len = len(await file_path.read_bytes())
params |= {
cur_len = (await file_path.stat()).st_size
headers |= {
"Range": f"bytes={cur_len}-{file_len if file_len else ''}"
}
else:
await file_path.touch()
async with aiohttp.ClientSession() as session:
rsp = await session.get(file_url, params=params, timeout=None)
rsp = await session.get(file_url, headers=headers, timeout=None)
if rsp.status == 416:
return
rsp.raise_for_status()
while True:
chunk = await rsp.content.read(chunks)
@ -51,7 +52,7 @@ class HDiffPatch:
git_url = constants.HDIFFPATCH_GIT_URL
self._git_url = git_url
if not data_dir:
self._appdirs = appdirs.AppDirs(constants.APP_NAME, constants.APP_AUTHOR)
self._appdirs = constants.APPDIRS
self.temp_path = Path(self._appdirs.user_cache_dir).joinpath("HDiffPatch")
self.data_path = Path(self._appdirs.user_data_dir).joinpath("Tools/HDiffPatch")
else:
@ -98,7 +99,7 @@ class HDiffPatch:
hpatchz_name = "hpatchz" + (".exe" if platform.system() == "Windows" else "")
return self._get_hdiffpatch_exec(hpatchz_name)
async def patch_file(self, in_file, out_file, patch_file, wait=False):
async def patch_file(self, in_file, out_file, patch_file, error=False, wait=False):
hpatchz = self.get_hpatchz_executable()
if not hpatchz:
raise RuntimeError("hpatchz executable not found")
@ -106,6 +107,8 @@ class HDiffPatch:
if not wait:
return proc
await proc.wait()
if error and proc.returncode != 0:
raise RuntimeError(f"Patching failed, return code is {proc.returncode}")
return proc
def get_hdiffz_executable(self):
@ -141,91 +144,33 @@ class HDiffPatch:
await _download_file(url, name, self.temp_path, overwrite=True)
if not extract:
return
archive = zipfile.ZipFile(self.temp_path.joinpath(name))
archive.extractall(self.data_path)
archive.close()
with zipfile.ZipFile(self.temp_path.joinpath(name), 'r') as f:
await asyncio.to_thread(f.extractall, path=self.data_path)
class Installer:
def _read_version_from_config(self):
warnings.warn("This function is not reliable as upgrading game version from worthless\
doesn't write the config.", DeprecationWarning)
if not self._config_file.exists():
raise FileNotFoundError(f"Config file {self._config_file} not found")
cfg = ConfigParser()
cfg.read(str(self._config_file))
return cfg.get("General", "game_version")
@staticmethod
def read_version_from_game_file(globalgamemanagers: Path | bytes):
"""
Reads the version from the globalgamemanagers file. (Data/globalgamemanagers)
Uses `An Anime Game Launcher` method to read the version:
https://gitlab.com/KRypt0n_/an-anime-game-launcher/-/blob/main/src/ts/Game.ts#L26
:return: Game version (ex 1.0.0)
"""
if isinstance(globalgamemanagers, Path):
with globalgamemanagers.open("rb") as f:
data = f.read().decode("ascii", errors="ignore")
else:
data = globalgamemanagers.decode("ascii", errors="ignore")
result = re.search(r"([1-9]+\.[0-9]+\.[0-9]+)_[\d]+_[\d]+", data)
if not result:
raise ValueError("Could not find version in game file")
return result.group(1)
def get_game_data_name(self):
if self._overseas:
return "GenshinImpact_Data/"
else:
return "YuanShen_Data/"
def get_game_data_path(self):
return self._gamedir.joinpath(self.get_game_data_name())
def get_game_version(self):
globalgamemanagers = self.get_game_data_path().joinpath("./globalgamemanagers")
if not globalgamemanagers.exists():
return
return self.read_version_from_game_file(globalgamemanagers)
def get_installed_voiceovers(self):
"""
Returns a list of installed voiceovers.
:return: List of installed voiceovers
"""
voiceovers = []
for file in self.get_game_data_path().joinpath("StreamingAssets/Audio/GeneratedSoundBanks/Windows").iterdir():
if file.is_dir():
voiceovers.append(file.name)
return voiceovers
def __init__(self, gamedir: str | Path = Path.cwd(), overseas: bool = True, data_dir: str | Path = None):
if isinstance(gamedir, str):
gamedir = Path(gamedir)
def __init__(self, gamedir: str | Path | AsyncPath = AsyncPath.cwd(),
overseas: bool = True, data_dir: str | Path | AsyncPath = None):
if isinstance(gamedir, str | Path):
gamedir = AsyncPath(gamedir)
self._gamedir = gamedir
if not data_dir:
self._appdirs = appdirs.AppDirs(constants.APP_NAME, constants.APP_AUTHOR)
self.temp_path = Path(self._appdirs.user_cache_dir).joinpath("Installer")
self._appdirs = constants.APPDIRS
self.temp_path = AsyncPath(self._appdirs.user_cache_dir).joinpath("Installer")
else:
if not isinstance(data_dir, Path):
data_dir = Path(data_dir)
if isinstance(data_dir, str | AsyncPath):
data_dir = AsyncPath(data_dir)
self.temp_path = data_dir.joinpath("Temp/Installer/")
self.temp_path.mkdir(parents=True, exist_ok=True)
Path(self.temp_path).mkdir(parents=True, exist_ok=True)
config_file = self._gamedir.joinpath("config.ini")
self._config_file = config_file.resolve()
self._config_file = config_file
self._download_chunk = 8192
self._overseas = overseas
self._version = self.get_game_version()
self._version = None
self._launcher = Launcher(self._gamedir, overseas=self._overseas)
self._hdiffpatch = HDiffPatch(data_dir=data_dir)
self._config = LauncherConfig(self._config_file, self._version)
def set_download_chunk(self, chunk: int):
self._download_chunk = chunk
self._game_version_re = re.compile(r"([1-9]+\.[0-9]+\.[0-9]+)_\d+_\d+")
async def _download_file(self, file_url: str, file_name: str, file_len: int = None, overwrite=False):
"""
@ -234,76 +179,143 @@ class Installer:
:param file_name:
:return:
"""
await _download_file(file_url, file_name, self.temp_path, file_len=file_len, overwrite=overwrite)
await _download_file(file_url, file_name, self.temp_path, file_len=file_len, overwrite=overwrite,
chunks=self._download_chunk)
def get_game_archive_version(self, game_archive: str | Path):
if not game_archive.exists():
raise FileNotFoundError(f"Game archive {game_archive} not found")
archive = zipfile.ZipFile(game_archive, 'r')
return self.read_version_from_game_file(archive.read(self.get_game_data_name() + "globalgamemanagers"))
async def read_version_from_config(self):
if not await self._config_file.exists():
raise FileNotFoundError(f"Config file {self._config_file} not found")
cfg = ConfigParser()
await asyncio.to_thread(cfg.read, str(self._config_file))
return cfg.get("General", "game_version")
async def read_version_from_game_file(self, globalgamemanagers: AsyncPath | Path | bytes) -> str:
"""
Reads the version from the globalgamemanagers file. (Data/globalgamemanagers)
Uses `An Anime Game Launcher` method to read the version:
https://gitlab.com/KRypt0n_/an-anime-game-launcher/-/blob/main/src/ts/Game.ts#L26
:return: Game version (ex 1.0.0)
"""
if isinstance(globalgamemanagers, Path | AsyncPath):
globalgamemanagers = AsyncPath(globalgamemanagers)
data = await globalgamemanagers.read_text("ascii", errors="ignore")
else:
data = globalgamemanagers.decode("ascii", errors="ignore")
result = self._game_version_re.search(data)
if not result:
raise ValueError("Could not find version in game file")
return result.group(1)
@staticmethod
def voiceover_lang_translate(lang: str):
def voiceover_lang_translate(lang: str, base_language="game") -> str:
"""
Translates the voiceover language to the language code used by the game.
:param lang: Language to translate
:param base_language: Base language type (game/locale/both)
:return: Language code
"""
match lang:
case "English(US)":
return "en-us"
case "Japanese":
return "ja-jp"
case "Chinese":
return "zh-cn"
case "Korean":
return "ko-kr"
if base_language == "game" or base_language == "both":
match lang.lower():
case "english(us)":
return "en-us"
case "japanese":
return "ja-jp"
case "chinese":
return "zh-cn"
case "korean":
return "ko-kr"
if base_language == "locale" or base_language == "both":
match lang.lower().replace("_", "-"):
case "en-us":
return "English(US)"
case "ja-jp":
return "Japanese"
case "zh-cn":
return "Chinese"
case "ko-kr":
return "Korean"
# If nothing else matches
return lang
@staticmethod
def get_voiceover_archive_language(voiceover_archive: str | Path):
if isinstance(voiceover_archive, str):
async def get_voiceover_archive_language(voiceover_archive: str | Path | AsyncPath) -> str:
if isinstance(voiceover_archive, str | Path):
voiceover_archive = Path(voiceover_archive).resolve()
if not voiceover_archive.exists():
raise FileNotFoundError(f"Voiceover archive {voiceover_archive} not found")
archive = zipfile.ZipFile(voiceover_archive, 'r')
archive_path = zipfile.Path(archive)
for file in archive_path.iterdir():
if file.name.endswith("_pkg_version"):
return file.name.split("_")[1]
with zipfile.ZipFile(voiceover_archive, 'r') as f:
for file in zipfile.Path(f).iterdir():
if file.name.endswith("_pkg_version"):
return file.name.split("_")[1]
def get_voiceover_archive_type(self, voiceover_archive: str | Path):
vo_lang = self.get_voiceover_archive_language(voiceover_archive)
archive = zipfile.ZipFile(voiceover_archive, 'r')
archive_path = zipfile.Path(archive)
files = archive.read("Audio_{}_pkg_version".format(vo_lang)).decode().split("\n")
for file in files:
if file.strip() and not archive_path.joinpath(json.loads(file)["remoteName"]).exists():
return False
@staticmethod
async def get_voiceover_archive_type(voiceover_archive: str | Path) -> bool:
"""
Gets voiceover archive type.
:param voiceover_archive:
:return: True if this is a full archive, else False.
"""
vo_lang = Installer.get_voiceover_archive_language(voiceover_archive)
with zipfile.ZipFile(voiceover_archive, 'r') as f:
archive_path = zipfile.Path(f)
files = (await asyncio.to_thread(f.read, "Audio_{}_pkg_version".format(vo_lang))).decode().split("\n")
for file in files:
if file.strip() and not archive_path.joinpath(json.loads(file)["remoteName"]).exists():
return False
return True
def apply_voiceover(self, voiceover_archive: str | Path):
# Since Voiceover packages are unclear about diff package or full package
# we will try to extract the voiceover package and apply it to the game
# making this function universal for both cases
if not self.get_game_data_path().exists():
raise FileNotFoundError(f"Game not found in {self._gamedir}")
if isinstance(voiceover_archive, str):
voiceover_archive = Path(voiceover_archive).resolve()
if not voiceover_archive.exists():
raise FileNotFoundError(f"Voiceover archive {voiceover_archive} not found")
archive = zipfile.ZipFile(voiceover_archive, 'r')
archive.extractall(self._gamedir)
archive.close()
def set_download_chunk(self, chunk: int):
self._download_chunk = chunk
async def update_game(self, game_archive: str | Path):
if not self.get_game_data_path().exists():
def get_game_data_name(self):
if self._overseas:
return "GenshinImpact_Data/"
else:
return "YuanShen_Data/"
def get_game_data_path(self) -> AsyncPath:
return self._gamedir.joinpath(self.get_game_data_name())
async def get_game_archive_version(self, game_archive: str | Path):
if not game_archive.exists():
raise FileNotFoundError(f"Game archive {game_archive} not found")
with zipfile.ZipFile(game_archive, 'r') as f:
return await self.read_version_from_game_file(
await asyncio.to_thread(f.read, self.get_game_data_name() + "globalgamemanagers")
)
async def get_game_version(self) -> str | None:
globalgamemanagers = self.get_game_data_path().joinpath("./globalgamemanagers")
if not await globalgamemanagers.exists():
try:
return await self.read_version_from_config()
except FileNotFoundError:
return
return await self.read_version_from_game_file(globalgamemanagers)
async def get_installed_voiceovers(self) -> list[str]:
"""
Returns a list of installed voiceovers.
:return: List of installed voiceovers
"""
voiceovers = []
async for file in self.get_game_data_path()\
.joinpath("StreamingAssets/Audio/GeneratedSoundBanks/Windows").iterdir():
if await file.is_dir():
voiceovers.append(file.name)
return voiceovers
async def update_game(self, game_archive: str | Path | AsyncPath):
if not await self.get_game_data_path().exists():
raise FileNotFoundError(f"Game not found in {self._gamedir}")
if isinstance(game_archive, str):
if isinstance(game_archive, str | Path):
game_archive = Path(game_archive).resolve()
if not game_archive.exists():
raise FileNotFoundError(f"Update archive {game_archive} not found")
archive = zipfile.ZipFile(game_archive, 'r')
if not self._hdiffpatch.get_hpatchz_executable():
@ -320,13 +332,13 @@ class Installer:
# hdiffpatch implementation
hdifffiles = []
for x in archive.read("hdifffiles.txt").decode().split("\n"):
for x in (await asyncio.to_thread(archive.read, "hdifffiles.txt")).decode().split("\n"):
if x:
hdifffiles.append(json.loads(x)["remoteName"])
patch_jobs = []
for file in hdifffiles:
current_game_file = self._gamedir.joinpath(file)
if not current_game_file.exists():
if not await current_game_file.exists():
# Not patching since we don't have the file
continue
@ -341,10 +353,11 @@ class Installer:
patch_path, wait=True)
patch_path.unlink()
if proc.returncode != 0:
# Let the game redownload the file.
# Let the game download the file.
old_file.rename(old_file.with_suffix(old_suffix))
return
old_file.unlink()
files.remove(patch_file)
patch_jobs.append(extract_and_patch(current_game_file, patch_file))
@ -353,15 +366,16 @@ class Installer:
deletefiles = archive.read("deletefiles.txt").decode().split("\n")
for file in deletefiles:
current_game_file = self._gamedir.joinpath(file)
if not current_game_file.exists():
if not await current_game_file.exists():
continue
if current_game_file.is_file():
current_game_file.unlink(missing_ok=True)
archive.extractall(self._gamedir, members=files)
await asyncio.to_thread(archive.extractall, self._gamedir, members=files)
archive.close()
# Update game version on local variable.
self._version = self.get_game_version()
self._version = await self.get_game_version()
self.set_version_config()
def set_version_config(self, version: str = None):
if not version:
@ -370,11 +384,11 @@ class Installer:
self._config.save()
async def download_full_game(self):
archive = await self._launcher.get_resource_info()
if archive is None:
resource = await self._launcher.get_resource_info()
if resource is None:
raise RuntimeError("Failed to fetch game resource info.")
archive_name = archive.game.latest.path.split("/")[-1]
await self._download_file(archive.game.latest.path, archive_name, archive.game.latest.size)
archive_name = resource.game.latest.path.split("/")[-1]
await self._download_file(resource.game.latest.path, archive_name, resource.game.latest.size)
async def download_full_voiceover(self, language: str):
archive = await self._launcher.get_resource_info()
@ -383,19 +397,57 @@ class Installer:
translated_lang = self.voiceover_lang_translate(language)
for vo in archive.game.latest.voice_packs:
if vo.language == translated_lang:
await self._download_file(vo.path, vo.name, vo.size)
await self._download_file(vo.path, vo.get_name(), vo.size)
async def download_game_update(self, from_version: str = None):
async def uninstall_game(self):
await asyncio.to_thread(shutil.rmtree, self._gamedir, ignore_errors=True)
async def _extract_game_file(self, archive: str | Path | AsyncPath):
if isinstance(archive, str | AsyncPath):
archive = Path(archive).resolve()
if not archive.exists():
raise FileNotFoundError(f"'{archive}' not found")
with zipfile.ZipFile(archive, 'r') as f:
await asyncio.to_thread(f.extractall, path=self._gamedir)
async def apply_voiceover(self, voiceover_archive: str | Path):
# Since Voiceover packages are unclear about diff package or full package
# we will try to extract the voiceover package and apply it to the game
# making this function universal for both cases
if not await self.get_game_data_path().exists():
raise FileNotFoundError(f"Game not found in {self._gamedir}")
await self._extract_game_file(voiceover_archive)
async def install_game(self, game_archive: str | Path | AsyncPath, force_reinstall: bool = False):
"""Installs the game to the current directory
If `force_reinstall` is True, the game will be uninstalled then reinstalled.
"""
if await self.get_game_data_path().exists():
if not force_reinstall:
raise ValueError(f"Game is already installed in {self._gamedir}")
await self.uninstall_game()
await self._gamedir.mkdir(parents=True, exist_ok=True)
await self._extract_game_file(game_archive)
self._version = await self.get_game_version()
self.set_version_config()
async def _get_game_resource(self, from_version: str = None):
if not from_version:
if self._version:
from_version = self._version
else:
from_version = self._version = self.get_game_version()
from_version = self._version = await self.get_game_version()
if not from_version:
raise ValueError("No game version found")
version_info = await self._launcher.get_resource_info()
if version_info is None:
raise RuntimeError("Failed to fetch game resource info.")
game_resource = await self._launcher.get_resource_info()
if not game_resource:
raise ValueError("Could not fetch game resource")
return game_resource
async def download_game_update(self, from_version: str = None):
version_info = await self._get_game_resource()
if self._version == version_info.game.latest.version:
raise ValueError("Game is already up to date.")
diff_archive = await self.get_game_diff_archive(from_version)
@ -404,56 +456,17 @@ class Installer:
await self._download_file(diff_archive.path, diff_archive.name, diff_archive.size)
async def download_voiceover_update(self, language: str, from_version: str = None):
if not from_version:
if self._version:
from_version = self._version
else:
from_version = self._version = self.get_game_version()
if not from_version:
raise ValueError("No game version found")
version_info = await self._launcher.get_resource_info()
if version_info is None:
raise RuntimeError("Failed to fetch game resource info.")
diff_archive = await self.get_voiceover_diff_archive(language, from_version)
if diff_archive is None:
raise ValueError("Voiceover diff archive is not available for this version, please reinstall.")
await self._download_file(diff_archive.path, diff_archive.name, diff_archive.size)
def uninstall_game(self):
shutil.rmtree(self._gamedir)
def install_game(self, game_archive: str | Path, force_reinstall: bool = False):
"""Installs the game to the current directory
If `force_reinstall` is True, the game will be uninstalled then reinstalled.
"""
if self.get_game_data_path().exists():
if not force_reinstall:
raise ValueError(f"Game is already installed in {self._gamedir}")
self.uninstall_game()
self._gamedir.mkdir(parents=True, exist_ok=True)
if isinstance(game_archive, str):
game_archive = Path(game_archive).resolve()
if not game_archive.exists():
raise FileNotFoundError(f"Install archive {game_archive} not found")
archive = zipfile.ZipFile(game_archive, 'r')
archive.extractall(self._gamedir)
archive.close()
async def get_voiceover_diff_archive(self, lang: str, from_version: str = None):
"""Gets a diff archive from `from_version` to the latest one
If from_version is not specified, it will be taken from the game version.
"""
if not from_version:
if self._version:
from_version = self._version
else:
from_version = self._version = self.get_game_version()
if not from_version:
raise ValueError("No game version found")
game_resource = await self._launcher.get_resource_info()
game_resource = await self._get_game_resource()
if not game_resource:
raise ValueError("Could not fetch game resource")
translated_lang = self.voiceover_lang_translate(lang)
@ -470,16 +483,48 @@ class Installer:
If from_version is not specified, it will be taken from the game version.
"""
if not from_version:
if self._version:
from_version = self._version
else:
from_version = self._version = self.get_game_version()
if not from_version:
raise ValueError("No game version found")
game_resource = await self._launcher.get_resource_info()
if not game_resource:
raise ValueError("Could not fetch game resource")
game_resource = await self._get_game_resource()
for v in game_resource.game.diffs:
if v.version == from_version:
return v
async def verify_from_pkg_version(self, pkg_version: AsyncPath, ignore_mismatch=False):
contents = await pkg_version.read_text()
async def calculate_md5(file_to_calculate):
async with AsyncPath(file_to_calculate).open("rb") as f:
file_hash = hashlib.md5()
while chunk := await f.read(self._download_chunk):
file_hash.update(chunk)
return file_hash.hexdigest()
async def verify_file(file_to_verify, md5):
file_md5 = await calculate_md5(file_to_verify)
if file_md5 == md5:
return None
if ignore_mismatch:
return file_to_verify, md5, file_md5
raise ValueError(f"MD5 does not match for {file_to_verify}, expected md5: {md5}, actual md5: {file_md5}")
verify_jobs = []
for content in contents.split("\r\n"):
if not content.strip():
continue
info = json.loads(content)
verify_jobs.append(verify_file(self._gamedir.joinpath(info["remoteName"]), info["md5"]))
verify_result = await asyncio.gather(*verify_jobs)
failed_files = []
for file in verify_result:
if file is not None:
failed_files.append(file)
return None if not failed_files else failed_files
async def verify_game(self, pkg_version: str | Path | AsyncPath = None, ignore_mismatch=False):
if pkg_version is None:
pkg_version = self._gamedir.joinpath("pkg_version")
return await self.verify_from_pkg_version(pkg_version, ignore_mismatch)
async def clear_cache(self):
await asyncio.to_thread(shutil.rmtree, self.temp_path, ignore_errors=True)