Skip to content

Instantly share code, notes, and snippets.

@terrapass
Last active June 23, 2024 18:36
Show Gist options
  • Select an option

  • Save terrapass/9294ec447ab931486f4c94267d14da60 to your computer and use it in GitHub Desktop.

Select an option

Save terrapass/9294ec447ab931486f4c94267d14da60 to your computer and use it in GitHub Desktop.
Mass TGA compression Python script and pre-commit hook example
@echo OFF
python tools/compress_tga.py "mod/gfx/map/terrain/*.tga"
#!/usr/bin/env python
import argparse
import shutil
import os
from pathlib import Path
from sys import stderr
from enum import Enum
import glob
from PIL import Image
from git import Repo, DiffIndex, InvalidGitRepositoryError
#
# Enums
#
class MainExitCode(int, Enum):
SUCCESS = 0
UNKNOWN_ERROR = 1
MISSING_FILE = 2
OS_ERROR = 3
GIT_ERROR = 4
#
# Constants
#
CLI_DESCRIPTION = f"""
Applies lossless RLE compression to TGA image file(s) given as one or more command line arguments.
Each argument can be a file path or a glob.
If run with --git-hook, only compresses the files currently staged for commit and re-stages them.
"""
BACKUP_FILE_SUFFIX = '.bak'
#
# Command line arguments
#
def parse_cli_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=CLI_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('raw_tga_paths', metavar='TGA_PATHS', nargs='+', type=str, help='TGA file paths or globs')
parser.add_argument('-m', '--allow-missing', action='store_true', dest='allow_missing', help="don't fail on non-existing paths")
parser.add_argument('-g', '--git-hook', action='store_true', dest='is_git_hook', help='only compress (and re-stage) files currently staged for commit')
#parser.add_argument('-v', '--verbose', action='store_true', dest='is_verbose', help='print detailed logs to stdout')
return parser.parse_args()
#
# Service
#
def extract_changed_paths(diff_index: DiffIndex) -> set[Path]:
modified_diff_entries = filter(lambda diff_entry: diff_entry.change_type == 'M' or diff_entry.change_type == 'A', diff_index)
return set(map(lambda diff_entry: Path(diff_entry.b_path), modified_diff_entries))
def backup_file(file_path: Path) -> Path:
backup_file_path = file_path.with_suffix(BACKUP_FILE_SUFFIX)
shutil.copyfile(file_path, backup_file_path)
return backup_file_path
# Main
#
def main():
args = parse_cli_args()
try:
repo = Repo() if args.is_git_hook else None
if repo is not None and repo.bare:
print(f"Error: cannot use this utility inside of a bare repo", file=stderr)
exit(MainExitCode.GIT_ERROR)
except InvalidGitRepositoryError:
assert args.is_git_hook
print(f"Error: cannot use --git-hook outside of a Git repo", file=stderr)
exit(MainExitCode.GIT_ERROR)
unstaged_changed_paths = None
staged_changed_paths = None
if args.is_git_hook:
assert repo is not None
unstaged_changed_paths = extract_changed_paths(repo.index.diff(None))
staged_changed_paths = extract_changed_paths(repo.index.diff("HEAD"))
tga_paths = set()
for raw_tga_path in args.raw_tga_paths:
glob_tga_paths = set(map(lambda pathlike: Path(pathlike), glob.glob(raw_tga_path)))
if len(glob_tga_paths) <= 0:
if args.allow_missing:
print(f"Warning: no files matched for {raw_tga_path}", file=stderr)
continue
else:
print(f"Error: no files matched for {raw_tga_path}", file=stderr)
exit(MainExitCode.MISSING_FILE)
if args.is_git_hook:
assert isinstance(unstaged_changed_paths, set)
assert isinstance(staged_changed_paths, set)
glob_tga_paths.intersection_update(staged_changed_paths)
modified_staged_paths = glob_tga_paths.intersection(unstaged_changed_paths)
if len(modified_staged_paths) > 0:
modified_staged_path_example = next(iter(modified_staged_paths))
print(f"Error: {modified_staged_path_example} has both staged and unstaged changes", file=stderr)
print("Already staged changes in this file would be overwritten.")
print("Please either stage or discard unstaged changes, then try again.")
exit(MainExitCode.GIT_ERROR)
tga_paths.update(glob_tga_paths)
if len(tga_paths) <= 0:
assert args.allow_missing or args.is_git_hook
print(f"No {'staged ' if args.is_git_hook else ''}files matched given paths, nothing to do")
exit(MainExitCode.SUCCESS)
tga_paths_count = len(tga_paths)
print(f"RLE-compressing {tga_paths_count} {'staged ' if args.is_git_hook else ''}TGA file{'s' if tga_paths_count != 1 else ''} matching given path{'s' if len(args.raw_tga_paths) != 1 else ''}:")
for tga_path in tga_paths:
print(f"{tga_path} ... ", end='')
try:
backup_tga_path = backup_file(tga_path)
with Image.open(backup_tga_path, formats=['TGA']) as input_tga:
if "compression" not in input_tga.info or input_tga.info["compression"] != 'tga_rle':
input_tga.save(tga_path, format='TGA', compression='tga_rle')
print("DONE")
else:
print("SKIPPED (already compressed)")
try:
os.remove(backup_tga_path)
except Exception as e:
print(f"Warning: failed to remove backup {backup_tga_path}", file=stderr)
except OSError as e:
print("FAILED (OS error)")
print(f"Error: {e}", file=stderr)
exit(MainExitCode.OS_ERROR)
except Exception as e:
print("FAILED (unknown error)")
print(f"Error: {e}", file=stderr)
exit(MainExitCode.UNKNOWN_ERROR)
if args.is_git_hook:
assert repo is not None
for restaged_path in map(lambda path: path.as_posix(), tga_paths):
repo.git.add('-u', '--', str(restaged_path))
print(f"Re-staged {tga_paths_count} files{'s' if tga_paths_count != 1 else ''}")
if __name__ == '__main__':
main()
#!/bin/sh
set -e # exit on the first error
TOOLS_DIR=./tools
MOD_DIR=./mod
# A hack to get GitHub Desktop on Windows to behave (not necessary for Git CLI).
# This is where cygpath.exe should be located on Windows, assuming default Git install directory.
# Otherwise, GitHub Desktop users might need to manually add its actual location to PATH.
CYGPATH_PATH_GUESS=/c/Program\ Files/Git/usr/bin
export PATH=$PATH:$CYGPATH_PATH_GUESS
#
# Main
#
PYTHON_VENV_DIR=venv
if [ -n $PYTHON_VENV_DIR ]
then
if [ -e $PYTHON_VENV_DIR/Scripts/activate ] # Windows
then
. $PYTHON_VENV_DIR/Scripts/activate
trap deactivate EXIT
elif [ -e $PYTHON_VENV_DIR/bin/activate ] # Linux or macOS
then
. $PYTHON_VENV_DIR/bin/activate
trap deactivate EXIT
fi
fi
which python > /dev/null
if [ $? -ne 0 ]
then
>&2 printf "\nERROR: Python not installed or missing from PATH\n"
exit 2
fi
which pip > /dev/null
if [ $? -ne 0 ]
then
>&2 printf "\nERROR: pip not installed or missing from PATH\n"
exit 3
fi
echo "Using $(python -V) at $(which python)"
if pip freeze -r "$TOOLS_DIR/tools/requirements.txt" 2>&1 1>/dev/null | grep -P "^WARNING:" > /dev/null
then
>&2 printf "\nERROR: missing required Python dependencies\n"
echo "To install them, run the following command from the root of the repo:"
echo "pip install -r $TOOLS_DIR/requirements.txt"
exit 4
fi
if ! "$TOOLS_DIR/compress_tga.py" --git-hook "$MOD_DIR/gfx/map/terrain/*.tga"
then
>&2 printf "\nERROR: failed to compress map TGAs\n"
exit 5
fi
pillow==10.3.0
GitPython==3.1.32
@terrapass
Copy link
Author

terrapass commented Apr 23, 2024

compress_tga.py is a Python script that (losslessly) RLE-compresses TGA files, which can be deployed as a pre-commit hook to compress certain files automatically whenever changes to them are being committed to Git.

Running this script produces equivalent results to opening every given TGA file in Paint.NET and re-saving it with "Compress (RLE)" checkbox checked. It has no effect if a given TGA file is already RLE-compressed.

This is particularly helpful for detail_index.tga and detail_intensity.tga packed map texture files, used for terrain in CK3 and produced by CK3's map editor, as it cuts down their size from 128MB each to ~13MB and ~30MB respectively (measured for Godherja's map, the exact number will vary depending on map complexity).
Since the compression is lossless, no data degradation in these textures occurs. The game also loads the map from RLE-compressed TGAs about as fast as from uncompressed ones, with no noticeable loading time increase in practice.

compress_tga.py Synopsis

usage: compress_tga.py [-h] [-m] [-g] TGA_PATHS [TGA_PATHS ...]

Applies lossless RLE compression to TGA image file(s) given as one or more command line arguments.
Each argument can be a file path or a glob.

If run with --git-hook, only compresses the files currently staged for commit and re-stages them.

positional arguments:
  TGA_PATHS            TGA file paths or globs

options:
  -h, --help           show this help message and exit
  -m, --allow-missing  don't fail on non-existing paths
  -g, --git-hook       only compress (and re-stage) files currently staged for
                       commit

Python and Dependencies

The script was tested on Python 3.12, but should be compatible with any 3.10+ version.

It requires a couple of thirdparty modules to run, which are listed in requirements.txt and can be installed into your Python environment using the following command from the directory where you've placed requirements.txt:

pip install -r requirements.txt

Running via a .bat File

As can be seen from the synopsis, the script itself is universal - it takes its target file paths as arguments, so it can be used in any environment, and not necessarily for the purposes of CK3 modding.

In order to run it then, one needs to either run it from the command line and give it paths (or path masks with *) as arguments, or store the command invocation in a .bat file such as compress_map_tgas.bat provided above, so it's easily double-clickable.

compress_map_tgas.bat as given above assumes that you've placed compress_tga.py in tools/ and that your mod's root is in mod/ folder, both relative to the location of the .bat. Feel free to change these paths in compress_map_tgas.bat to reflect your actual setup.

You can commit and push both the Python script and the .bat targeting your gfx/map/terrain/*.tga to your Git repo. This way mappers, provided they have Python and the aforementioned dependencies installed, can just double-click it to compress the files before committing changes to them.

Unless they forget to. Which brings us to...

Deploying as a pre-commit Hook

Luckily, Git provides a mechanism, by which a certain script, say for example - an utility to mass-compress staged TGA files, can run automatically each time when (or rather, immediately before) a commit is made. This mechanism is known as a pre-commit hook.

To make use of this, compress_tga.py comes with suport for a --git-hook switch, given which it will only compress the files currently staged for commit, and then re-stage them. This allows compress_tga.py --git-hook mod/gfx/map/terrain/*.tga to be run from inside a pre-commit hook and silently do the job it was made for - preventing mappers from pushing 128MB TGA files to the repo without being obtrusive and even without requiring them to remember to run a .bat.

pre-commit file above is an example of such hook script, a slightly modified version of the one used by Godherja: The Dying World mod team. It ensures that Python and the necessary dependencies are installed, then RLE-compresses any staged TGA files from gfx/map/terrain in the mod.

The given pre-commit hook code assumes that the mod files are inside mod/ subdirectory of the repo, while the compress_tga.py is inside tools/. This can be changed by adjusting MOD_DIR and TOOLS_DIR variables in its code accordingly. (Note that you can use . to denote current folder, aka repo root, if needed.)

Git client-side hooks such as pre-commit are opt-in (unfortunately), so you'll need to ask your team members to copy your pre-commit file into their local repo's .git/hooks directory (either manually, or by giving them another .bat file that performs such a copy). But this installation step only needs to be done once, after which it'll "just work".

Note: GitHub Desktop users reported that they needed to manually install Git (the command line utility itself) to get GitHub Desktop to correctly invoke the hook. The simplest way is to ask your team members who use GHD to go to Repository -> Open in command prompt. If they see a git-bash terminal, they're good to go. If not, GHD will guide them on how to install Git CLI.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment