Last active
June 23, 2024 18:36
-
-
Save terrapass/9294ec447ab931486f4c94267d14da60 to your computer and use it in GitHub Desktop.
Mass TGA compression Python script and pre-commit hook example
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @echo OFF | |
| python tools/compress_tga.py "mod/gfx/map/terrain/*.tga" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| import argparse | |
| import shutil | |
| import os | |
| from pathlib import Path | |
| from sys import stderr | |
| from enum import Enum | |
| import glob | |
| from PIL import Image | |
| from git import Repo, DiffIndex, InvalidGitRepositoryError | |
| # | |
| # Enums | |
| # | |
| class MainExitCode(int, Enum): | |
| SUCCESS = 0 | |
| UNKNOWN_ERROR = 1 | |
| MISSING_FILE = 2 | |
| OS_ERROR = 3 | |
| GIT_ERROR = 4 | |
| # | |
| # Constants | |
| # | |
| CLI_DESCRIPTION = f""" | |
| Applies lossless RLE compression to TGA image file(s) given as one or more command line arguments. | |
| Each argument can be a file path or a glob. | |
| If run with --git-hook, only compresses the files currently staged for commit and re-stages them. | |
| """ | |
| BACKUP_FILE_SUFFIX = '.bak' | |
| # | |
| # Command line arguments | |
| # | |
| def parse_cli_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=CLI_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument('raw_tga_paths', metavar='TGA_PATHS', nargs='+', type=str, help='TGA file paths or globs') | |
| parser.add_argument('-m', '--allow-missing', action='store_true', dest='allow_missing', help="don't fail on non-existing paths") | |
| parser.add_argument('-g', '--git-hook', action='store_true', dest='is_git_hook', help='only compress (and re-stage) files currently staged for commit') | |
| #parser.add_argument('-v', '--verbose', action='store_true', dest='is_verbose', help='print detailed logs to stdout') | |
| return parser.parse_args() | |
| # | |
| # Service | |
| # | |
| def extract_changed_paths(diff_index: DiffIndex) -> set[Path]: | |
| modified_diff_entries = filter(lambda diff_entry: diff_entry.change_type == 'M' or diff_entry.change_type == 'A', diff_index) | |
| return set(map(lambda diff_entry: Path(diff_entry.b_path), modified_diff_entries)) | |
| def backup_file(file_path: Path) -> Path: | |
| backup_file_path = file_path.with_suffix(BACKUP_FILE_SUFFIX) | |
| shutil.copyfile(file_path, backup_file_path) | |
| return backup_file_path | |
| # Main | |
| # | |
| def main(): | |
| args = parse_cli_args() | |
| try: | |
| repo = Repo() if args.is_git_hook else None | |
| if repo is not None and repo.bare: | |
| print(f"Error: cannot use this utility inside of a bare repo", file=stderr) | |
| exit(MainExitCode.GIT_ERROR) | |
| except InvalidGitRepositoryError: | |
| assert args.is_git_hook | |
| print(f"Error: cannot use --git-hook outside of a Git repo", file=stderr) | |
| exit(MainExitCode.GIT_ERROR) | |
| unstaged_changed_paths = None | |
| staged_changed_paths = None | |
| if args.is_git_hook: | |
| assert repo is not None | |
| unstaged_changed_paths = extract_changed_paths(repo.index.diff(None)) | |
| staged_changed_paths = extract_changed_paths(repo.index.diff("HEAD")) | |
| tga_paths = set() | |
| for raw_tga_path in args.raw_tga_paths: | |
| glob_tga_paths = set(map(lambda pathlike: Path(pathlike), glob.glob(raw_tga_path))) | |
| if len(glob_tga_paths) <= 0: | |
| if args.allow_missing: | |
| print(f"Warning: no files matched for {raw_tga_path}", file=stderr) | |
| continue | |
| else: | |
| print(f"Error: no files matched for {raw_tga_path}", file=stderr) | |
| exit(MainExitCode.MISSING_FILE) | |
| if args.is_git_hook: | |
| assert isinstance(unstaged_changed_paths, set) | |
| assert isinstance(staged_changed_paths, set) | |
| glob_tga_paths.intersection_update(staged_changed_paths) | |
| modified_staged_paths = glob_tga_paths.intersection(unstaged_changed_paths) | |
| if len(modified_staged_paths) > 0: | |
| modified_staged_path_example = next(iter(modified_staged_paths)) | |
| print(f"Error: {modified_staged_path_example} has both staged and unstaged changes", file=stderr) | |
| print("Already staged changes in this file would be overwritten.") | |
| print("Please either stage or discard unstaged changes, then try again.") | |
| exit(MainExitCode.GIT_ERROR) | |
| tga_paths.update(glob_tga_paths) | |
| if len(tga_paths) <= 0: | |
| assert args.allow_missing or args.is_git_hook | |
| print(f"No {'staged ' if args.is_git_hook else ''}files matched given paths, nothing to do") | |
| exit(MainExitCode.SUCCESS) | |
| tga_paths_count = len(tga_paths) | |
| print(f"RLE-compressing {tga_paths_count} {'staged ' if args.is_git_hook else ''}TGA file{'s' if tga_paths_count != 1 else ''} matching given path{'s' if len(args.raw_tga_paths) != 1 else ''}:") | |
| for tga_path in tga_paths: | |
| print(f"{tga_path} ... ", end='') | |
| try: | |
| backup_tga_path = backup_file(tga_path) | |
| with Image.open(backup_tga_path, formats=['TGA']) as input_tga: | |
| if "compression" not in input_tga.info or input_tga.info["compression"] != 'tga_rle': | |
| input_tga.save(tga_path, format='TGA', compression='tga_rle') | |
| print("DONE") | |
| else: | |
| print("SKIPPED (already compressed)") | |
| try: | |
| os.remove(backup_tga_path) | |
| except Exception as e: | |
| print(f"Warning: failed to remove backup {backup_tga_path}", file=stderr) | |
| except OSError as e: | |
| print("FAILED (OS error)") | |
| print(f"Error: {e}", file=stderr) | |
| exit(MainExitCode.OS_ERROR) | |
| except Exception as e: | |
| print("FAILED (unknown error)") | |
| print(f"Error: {e}", file=stderr) | |
| exit(MainExitCode.UNKNOWN_ERROR) | |
| if args.is_git_hook: | |
| assert repo is not None | |
| for restaged_path in map(lambda path: path.as_posix(), tga_paths): | |
| repo.git.add('-u', '--', str(restaged_path)) | |
| print(f"Re-staged {tga_paths_count} files{'s' if tga_paths_count != 1 else ''}") | |
| if __name__ == '__main__': | |
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| set -e # exit on the first error | |
| TOOLS_DIR=./tools | |
| MOD_DIR=./mod | |
| # A hack to get GitHub Desktop on Windows to behave (not necessary for Git CLI). | |
| # This is where cygpath.exe should be located on Windows, assuming default Git install directory. | |
| # Otherwise, GitHub Desktop users might need to manually add its actual location to PATH. | |
| CYGPATH_PATH_GUESS=/c/Program\ Files/Git/usr/bin | |
| export PATH=$PATH:$CYGPATH_PATH_GUESS | |
| # | |
| # Main | |
| # | |
| PYTHON_VENV_DIR=venv | |
| if [ -n $PYTHON_VENV_DIR ] | |
| then | |
| if [ -e $PYTHON_VENV_DIR/Scripts/activate ] # Windows | |
| then | |
| . $PYTHON_VENV_DIR/Scripts/activate | |
| trap deactivate EXIT | |
| elif [ -e $PYTHON_VENV_DIR/bin/activate ] # Linux or macOS | |
| then | |
| . $PYTHON_VENV_DIR/bin/activate | |
| trap deactivate EXIT | |
| fi | |
| fi | |
| which python > /dev/null | |
| if [ $? -ne 0 ] | |
| then | |
| >&2 printf "\nERROR: Python not installed or missing from PATH\n" | |
| exit 2 | |
| fi | |
| which pip > /dev/null | |
| if [ $? -ne 0 ] | |
| then | |
| >&2 printf "\nERROR: pip not installed or missing from PATH\n" | |
| exit 3 | |
| fi | |
| echo "Using $(python -V) at $(which python)" | |
| if pip freeze -r "$TOOLS_DIR/tools/requirements.txt" 2>&1 1>/dev/null | grep -P "^WARNING:" > /dev/null | |
| then | |
| >&2 printf "\nERROR: missing required Python dependencies\n" | |
| echo "To install them, run the following command from the root of the repo:" | |
| echo "pip install -r $TOOLS_DIR/requirements.txt" | |
| exit 4 | |
| fi | |
| if ! "$TOOLS_DIR/compress_tga.py" --git-hook "$MOD_DIR/gfx/map/terrain/*.tga" | |
| then | |
| >&2 printf "\nERROR: failed to compress map TGAs\n" | |
| exit 5 | |
| fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| pillow==10.3.0 | |
| GitPython==3.1.32 |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
compress_tga.pyis a Python script that (losslessly) RLE-compresses TGA files, which can be deployed as apre-commithook to compress certain files automatically whenever changes to them are being committed to Git.Running this script produces equivalent results to opening every given TGA file in Paint.NET and re-saving it with "Compress (RLE)" checkbox checked. It has no effect if a given TGA file is already RLE-compressed.
This is particularly helpful for
detail_index.tgaanddetail_intensity.tgapacked map texture files, used for terrain in CK3 and produced by CK3's map editor, as it cuts down their size from 128MB each to ~13MB and ~30MB respectively (measured for Godherja's map, the exact number will vary depending on map complexity).Since the compression is lossless, no data degradation in these textures occurs. The game also loads the map from RLE-compressed TGAs about as fast as from uncompressed ones, with no noticeable loading time increase in practice.
compress_tga.pySynopsisPython and Dependencies
The script was tested on Python 3.12, but should be compatible with any 3.10+ version.
It requires a couple of thirdparty modules to run, which are listed in
requirements.txtand can be installed into your Python environment using the following command from the directory where you've placedrequirements.txt:Running via a
.batFileAs can be seen from the synopsis, the script itself is universal - it takes its target file paths as arguments, so it can be used in any environment, and not necessarily for the purposes of CK3 modding.
In order to run it then, one needs to either run it from the command line and give it paths (or path masks with
*) as arguments, or store the command invocation in a.batfile such ascompress_map_tgas.batprovided above, so it's easily double-clickable.compress_map_tgas.batas given above assumes that you've placedcompress_tga.pyintools/and that your mod's root is inmod/folder, both relative to the location of the.bat. Feel free to change these paths incompress_map_tgas.batto reflect your actual setup.You can commit and push both the Python script and the
.battargeting yourgfx/map/terrain/*.tgato your Git repo. This way mappers, provided they have Python and the aforementioned dependencies installed, can just double-click it to compress the files before committing changes to them.Unless they forget to. Which brings us to...
Deploying as a
pre-commitHookLuckily, Git provides a mechanism, by which a certain script, say for example - an utility to mass-compress staged TGA files, can run automatically each time when (or rather, immediately before) a commit is made. This mechanism is known as a
pre-commithook.To make use of this,
compress_tga.pycomes with suport for a--git-hookswitch, given which it will only compress the files currently staged for commit, and then re-stage them. This allowscompress_tga.py --git-hook mod/gfx/map/terrain/*.tgato be run from inside apre-commithook and silently do the job it was made for - preventing mappers from pushing 128MB TGA files to the repo without being obtrusive and even without requiring them to remember to run a.bat.pre-commitfile above is an example of such hook script, a slightly modified version of the one used by Godherja: The Dying World mod team. It ensures that Python and the necessary dependencies are installed, then RLE-compresses any staged TGA files fromgfx/map/terrainin the mod.The given
pre-commithook code assumes that the mod files are insidemod/subdirectory of the repo, while thecompress_tga.pyis insidetools/. This can be changed by adjustingMOD_DIRandTOOLS_DIRvariables in its code accordingly. (Note that you can use.to denote current folder, aka repo root, if needed.)Git client-side hooks such as
pre-commitare opt-in (unfortunately), so you'll need to ask your team members to copy yourpre-commitfile into their local repo's.git/hooksdirectory (either manually, or by giving them another.batfile that performs such a copy). But this installation step only needs to be done once, after which it'll "just work".Note: GitHub Desktop users reported that they needed to manually install Git (the command line utility itself) to get GitHub Desktop to correctly invoke the hook. The simplest way is to ask your team members who use GHD to go to
Repository -> Open in command prompt. If they see a git-bash terminal, they're good to go. If not, GHD will guide them on how to install Git CLI.