Created
November 5, 2020 19:15
-
-
Save webstrand/f98877f8255ad30c46c09b0e5af8d9cc to your computer and use it in GitHub Desktop.
A tutorial script explaining how to safely inline remote repositories.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ################################################################################ | |
| # Introduction | |
| ################################################################################ | |
| # This is an attempt at explaining how to vendor git repositories by storing | |
| # their _entire_ commit history inside of the local repository. By vendoring | |
| # dependencies using this technique, the remote repository can entirely | |
| # recreated from the local repository if the original were to be lost. | |
| # Additionally, clones of the local repository also satisfy that property. | |
| # | |
| # You're meant to read through this script and understand what it's doing. I | |
| # don't recommend using this script for any other purpose. | |
| # | |
| ################################################################################ | |
| # What this script does | |
| ################################################################################ | |
| # 1. Creates a bare repository at `./bare-repo`. | |
| # 2. Adds remotes for `zimfw` and it's supporting submodules in such a way that | |
| # a copy of the remote repository is stored locally. | |
| # 3. Demonstrates how to carry a local patch against one of the remote repos. | |
| # 4. Create a branch `etc-zsh` as a demonstration of how to reference locally | |
| # stored remotes as submodules. Also installs zimfw into that branch. | |
| # 5. Creates a master branch which includes a script to restore the remotes | |
| # configuration. Also installs the branch `etc-zsh` into /etc/zsh. | |
| ################################################################################ | |
| set -euo pipefail; # Strict Mode | |
| IFS=$'\n'; | |
| ################################################################################ | |
| ## 1. Setup | |
| ################################################################################ | |
| if [[ -e bare-repo ]]; then | |
| echo 'fatal: A file or folder called `bare-repo` already exists. Refusing to overwrite.'; | |
| exit 1; | |
| fi; | |
| # Set up the bare repository. We're using a bare repository here for example | |
| # purposes, because it makes working on multiple branches in one script easier | |
| # to follow. The techniques below will work on on-bare repos, too. | |
| git init --bare bare-repo; | |
| cd bare-repo; | |
| # We need an empty commit to checkout when creating worktrees from new branches, | |
| # otherwise git refuses to create the worktree. Once this script is done, this | |
| # commit will eventually be cleaned up by `git gc`. | |
| empty_commit="$(git commit-tree -m 'placeholder commit' "$(git mktree < /dev/null)")"; | |
| ################################################################################ | |
| ## 2. Install remote repositories | |
| ################################################################################ | |
| # We add the various repositories we're interest in. In this case `zimfw` and | |
| # the submodules that it requires to function. | |
| # | |
| # The important part below is the three `fetch` rules. The remotes we add below | |
| # are a little bit different than those created automatically by git: | |
| # | |
| # 1. The first `fetch` rule copies branches from the remote repository into the | |
| # local repositories remotes. This is identical to what git normally does. | |
| # | |
| # 2. The second `fetch` rule is the most important. It copies branches from the | |
| # remote repository into our `heads`. This means that remote branches will show | |
| # up as local branches in our repository. We do this because `git clone` does | |
| # not ever clone remotes; it only clones local heads. Thus, without this rule, | |
| # cloning this repository wouldn't preserve all of the remote commits, only | |
| # commits referenced by local branches. | |
| # | |
| # 3. The third `fetch` rule copies tags from the remote repository into our local | |
| # local repository, but namespaces them so that they will not collide with local tags | |
| # tags or tags from other remotes. | |
| # Here we add them directly to .git/config, but they can also be added through | |
| # conventional `git config --add remote.zimfw.fetch ...`. | |
| cat << 'EOF' >> config; | |
| [remote "zimfw"] | |
| url = https://github.com/zimfw/zimfw.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw/* | |
| fetch = +refs/tags/*:refs/tags/zimfw/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-history-substring-search"] | |
| url = https://github.com/zsh-users/zsh-history-substring-search.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-history-substring-search/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-history-substring-search/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-history-substring-search/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-completions"] | |
| url = https://github.com/zsh-users/zsh-completions.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-completions/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-completions/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-completions/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-syntax-highlighting"] | |
| url = https://github.com/zsh-users/zsh-syntax-highlighting.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-syntax-highlighting/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-syntax-highlighting/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-syntax-highlighting/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-pure"] | |
| url = https://github.com/sindresorhus/pure.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-pure/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-pure/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-pure/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-liquidprompt"] | |
| url = https://github.com/nojhan/liquidprompt.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-liquidprompt/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-liquidprompt/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-liquidprompt/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-lean"] | |
| url = https://github.com/miekg/lean | |
| fetch = +refs/heads/*:refs/remotes/zimfw-lean/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-lean/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-lean/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-autosuggestions"] | |
| url = https://github.com/zsh-users/zsh-autosuggestions.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-autosuggestions/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-autosuggestions/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-autosuggestions/* | |
| tagOpt = --no-tags | |
| EOF | |
| # Load the data from all of the remotes into our repository. Now, even if the | |
| # remotes we defined above get deleted, we'll still have a full copy of their | |
| # commit history even if the remote repositories get deleted. | |
| git fetch --all; | |
| ################################################################################ | |
| ## 3. Patch zimfw to reference local repository for submodules | |
| ################################################################################ | |
| # We need to carry a patch against zimfw that changes it's .gitmodules so that | |
| # they refer to the local repository rather than a remote one. This way they're | |
| # cloned directly from this repositories local history rather than over the | |
| # network. | |
| # Here we create a temporary worktree and create a new local branch `zimfw`. | |
| git worktree add -b zimfw zimfw-worktree zimfw/master; | |
| git branch -u zimfw/master zimfw; # Set the branch to pull from zimfw/master | |
| git config --add branch.zimfw.rebase true; # rebase by default on pull | |
| pushd zimfw-worktree; | |
| # Unfortunately, zimfw went through a major rewrite and cannot be installed | |
| # system-wide anymore. 4f6ae96b1241caf1b55e8393fb9df2e45a1657fb is the last | |
| # known good commit in the https://github.com/zimfw/zimfw.git before the | |
| # rewrite. We simply rollback the branch to the last known good commit. | |
| git reset --hard "4f6ae96b1241caf1b55e8393fb9df2e45a1657fb"; | |
| # This patch replaces the repository url for each submodule with ./ so that the | |
| # local repository is referenced, not the remote one. | |
| git am << 'EOF'; | |
| From: nobody <nobody@example.com> | |
| Date: Mon, 21 Jan 2019 23:09:55 +0000 | |
| Subject: [PATCH] Point .gitmodules <name>.url to local remotes | |
| --- | |
| .gitmodules | 21 ++++++++++++++------- | |
| 1 file changed, 14 insertions(+), 7 deletions(-) | |
| diff --git a/.gitmodules b/.gitmodules | |
| index 9b65f59b..a43f0501 100644 | |
| --- a/.gitmodules | |
| +++ b/.gitmodules | |
| @@ -1,28 +1,35 @@ | |
| [submodule "modules/history-substring-search/external"] | |
| path = modules/history-substring-search/external | |
| - url = https://github.com/zsh-users/zsh-history-substring-search.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-zsh-history-substring-search/master | |
| [submodule "modules/completion/external"] | |
| path = modules/completion/external | |
| - url = https://github.com/zsh-users/zsh-completions.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-zsh-completions/master | |
| [submodule "modules/syntax-highlighting/external"] | |
| path = modules/syntax-highlighting/external | |
| - url = https://github.com/zsh-users/zsh-syntax-highlighting.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-zsh-syntax-highlighting/master | |
| [submodule "modules/prompt/external-themes/pure"] | |
| path = modules/prompt/external-themes/pure | |
| - url = https://github.com/sindresorhus/pure.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-pure/master | |
| [submodule "modules/prompt/external-themes/liquidprompt"] | |
| path = modules/prompt/external-themes/liquidprompt | |
| - url = https://github.com/nojhan/liquidprompt.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-liquidprompt/master | |
| [submodule "modules/prompt/external-themes/lean"] | |
| path = modules/prompt/external-themes/lean | |
| - url = https://github.com/miekg/lean | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-lean/master | |
| [submodule "modules/autosuggestions/external"] | |
| path = modules/autosuggestions/external | |
| - url = https://github.com/zsh-users/zsh-autosuggestions.git | |
| + url = ./ | |
| ignore = untracked | |
| + branch = tracking-zimfw-zsh-autosuggestions/master | |
| \ No newline at end of file | |
| -- | |
| 2.26.2 | |
| EOF | |
| popd; | |
| # We remove the temporary worktree, but the branch `zimfw` still exists. | |
| git worktree remove zimfw-worktree; | |
| ################################################################################ | |
| ## 4. Create etc-zsh branch, configure zsh, and install zimfw | |
| ################################################################################ | |
| # Now we want to actually use zimfw somehow. We could simply expose it through | |
| # a `git worktree add /etc/zsh/zimfw zimfw` but wouldn't it be nice to keep | |
| # user configuration and zimfw version in sync? So we'll use zimfw as a | |
| # submodule in a new branch and then expose that branch as /etc/zsh! | |
| # Here we create a temporary worktree and create a new local branch `etc-zsh`. | |
| git worktree add -b etc-zsh etc-zsh-worktree "$empty_commit"; | |
| git update-ref -d refs/heads/etc-zsh; # reset branch to be an orphan | |
| pushd etc-zsh-worktree; | |
| # Install zimfw by referencing our local, patched, branch `zimfw`. If we didn't | |
| # need to patch it, we would reference `tracking-zimfw/master` instead. | |
| # We install our patched version of zimfw as a submodule. If we didn't need to | |
| # patch zimfw, we could reference `tracking-zimfw/master` instead. The submodule | |
| # is attached to a specific commit sha1 and will not automatically update until | |
| # we call `git submodule update --remote --merge`. | |
| git submodule add -b zimfw ./ zimfw; | |
| # Configure zshrc | |
| cat << 'EOF' > zshrc; | |
| export ZIM_HOME="/etc/zsh/zimfw"; | |
| [[ -s /etc/zsh/zimrc ]] && source /etc/zsh/zimrc; | |
| [[ -s ${ZIM_HOME}/init.zsh ]] && source ${ZIM_HOME}/init.zsh; | |
| EOF | |
| # Copy the default configuration out of zimfw. | |
| cp ./zimfw/templates/{zimrc,zlogin} ./; | |
| # Commit everything to the branch: etc-zsh. | |
| git add -A ./; | |
| git commit -m "Set up zimfw for zsh"; | |
| # Worktrees containing initialized submodules cannot be moved or deleted. | |
| git submodule deinit --all; | |
| popd; | |
| # TODO: Why do we need to force this? | |
| git worktree remove etc-zsh-worktree -f; | |
| ################################################################################ | |
| ## 5. Create master branch and install script | |
| ################################################################################ | |
| # When this repository gets cloned, the urls of the remote repositories are | |
| # lost. If we were using submodules directly, the remote repository url would be | |
| # stored in .gitmodules. However, since any submodules we use reference the | |
| # local repository we need to keep track of the remotes ourselves. | |
| # | |
| # We create a file in the master branch called .gitremotes and we create | |
| # a Makefile target `install-remotes` to re-install the remotes configuration | |
| # into the current repository. | |
| # Here we create a temporary worktree and create a new local branch `master`. | |
| git worktree add -b master master-worktree "$empty_commit"; | |
| git update-ref -d refs/heads/master; # reset branch to be an orphan | |
| pushd master-worktree; | |
| # We just copy the remotes configuration we used above into the .gitremotes | |
| # file. | |
| cat << 'EOF' > .gitremotes; | |
| [remote "zimfw"] | |
| url = https://github.com/zimfw/zimfw.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw/* | |
| fetch = +refs/tags/*:refs/tags/zimfw/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-history-substring-search"] | |
| url = https://github.com/zsh-users/zsh-history-substring-search.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-history-substring-search/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-history-substring-search/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-history-substring-search/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-completions"] | |
| url = https://github.com/zsh-users/zsh-completions.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-completions/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-completions/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-completions/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-syntax-highlighting"] | |
| url = https://github.com/zsh-users/zsh-syntax-highlighting.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-syntax-highlighting/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-syntax-highlighting/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-syntax-highlighting/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-pure"] | |
| url = https://github.com/sindresorhus/pure.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-pure/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-pure/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-pure/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-liquidprompt"] | |
| url = https://github.com/nojhan/liquidprompt.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-liquidprompt/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-liquidprompt/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-liquidprompt/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-lean"] | |
| url = https://github.com/miekg/lean | |
| fetch = +refs/heads/*:refs/remotes/zimfw-lean/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-lean/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-lean/* | |
| tagOpt = --no-tags | |
| [remote "zimfw-zsh-autosuggestions"] | |
| url = https://github.com/zsh-users/zsh-autosuggestions.git | |
| fetch = +refs/heads/*:refs/remotes/zimfw-zsh-autosuggestions/* | |
| fetch = +refs/heads/*:refs/heads/tracking-zimfw-zsh-autosuggestions/* | |
| fetch = +refs/tags/*:refs/tags/zimfw-zsh-autosuggestions/* | |
| tagOpt = --no-tags | |
| EOF | |
| # The target `install-remotes` figures out which lines of configuration are | |
| # missing from our local .git/config and installs them. No duplication. | |
| cat << 'EOF' > Makefile; | |
| default: | |
| echo 'See targets `install-remotes` or `install-worktrees`.'; | |
| # Configure the remotes, since remotes configuration isn't copied via git clone. | |
| install-remotes: .gitremotes | |
| (git config -lzf .gitremotes; git config -lz --local | sed -z 'p;p') | \ | |
| sort -z | uniq -zu | tr '\n' '\0' | xargs -0r -n2 git config; | |
| demo-install-worktrees: etc-zsh | |
| etc-zsh: | |
| git worktree add ./etc-zsh etc-zsh; | |
| cd ./etc-zsh; git submodule update --init --recursive; | |
| install-worktrees: /etc/zsh | |
| /etc/zsh: | |
| git worktree add /etc/zsh etc-zsh; | |
| cd /etc/zsh; git submodule update --init --recursive; | |
| .PHONY: default install-remotes install-worktrees demo-install-worktrees | |
| EOF | |
| # Commit everything to the branch: master. | |
| git add -A ./; | |
| git commit -m "Add initial configuration"; | |
| popd; | |
| git worktree remove master-worktree; | |
| echo '################################################################################'; | |
| echo '## Creation of repository complete. Now try `git clone bare-repo somerepo` ##'; | |
| echo '## Every commit belonging to bare-repo'"'"'s remotes has been safely cloned ##'; | |
| echo '## into somerepo. ##'; | |
| echo '################################################################################'; | |
| echo '## Try running `make demo-install-worktrees` inside of the cloned repository ##'; | |
| echo '## to see how worktrees & submodules interact. ##'; | |
| echo '################################################################################'; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment