Skip to content

Instantly share code, notes, and snippets.

@CodyKochmann

CodyKochmann/Makefile

Last active September 16, 2023 16:35
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save CodyKochmann/b6ea601ec0f95e395b7439bd2b4c2eb7 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save CodyKochmann/b6ea601ec0f95e395b7439bd2b4c2eb7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. CodyKochmann revised this gist Sep 16, 2023. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -49,7 +49,7 @@ LOCK_MAGNET_FILE = flock --exclusive --nonblock $(GET_MAGNET_FILE_FROM_LOG_FILE)
    ARIA2C_DOWNLOAD = aria2c --check-integrity=true --allow-overwrite=true --bt-force-encryption=true

    # string template to add further context to the log files
    LOGGER = logger --rfc5424=notq --socket-errors=off -t 'aria2-$@' --stderr
    LOGGER = logger --rfc5424=notq --socket-errors=off -t 'aria2-$(GET_MAGNET_FILE_FROM_LOG_FILE)' --stderr

    # target to refresh the log files
    clean-logs:
    @@ -66,5 +66,5 @@ _torrents: $(MAGNET_LOG_FILES)
    torrent:
    $(MAKE) -j $(MAGNET_FILE_COUNT) _torrents

    # define targets that dont actually result in a file being created
    .PHONY: _torrents torrent clean-logs
    # define phony targets for make to know about
    .PHONY: _torrents torrent clean-logs $(MAGNET_LOG_FILES)
  2. CodyKochmann revised this gist Sep 16, 2023. 1 changed file with 6 additions and 0 deletions.
    6 changes: 6 additions & 0 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -22,6 +22,9 @@
    SHELL := /bin/bash
    SHELLFLAGS := -euxo pipefail

    # set the default target so simply calling "make" does the job
    .DEFAULT_GOAL := torrent

    # command to find all of the local magnet files in this directory
    FIND_MAGNETS = find . -type f -name '*.magnet' -maxdepth 1 2>/dev/null

    @@ -62,3 +65,6 @@ _torrents: $(MAGNET_LOG_FILES)
    # primary target for the user to operate with that forks make into as many parallel jobs as it needs to host every magnet
    torrent:
    $(MAKE) -j $(MAGNET_FILE_COUNT) _torrents

    # define targets that dont actually result in a file being created
    .PHONY: _torrents torrent clean-logs
  3. CodyKochmann revised this gist Sep 16, 2023. 1 changed file with 16 additions and 4 deletions.
    20 changes: 16 additions & 4 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -18,35 +18,47 @@
    # specific directories are always being torrented from.
    #

    # tweaks to ensure correct shell is used
    SHELL := /bin/bash
    SHELLFLAGS := -euxo pipefail

    # command to find all of the local magnet files in this directory
    FIND_MAGNETS = find . -type f -name '*.magnet' -maxdepth 1 2>/dev/null

    # generate metadata needed to guide make
    MAGNET_FILES = $(shell $(FIND_MAGNETS))
    MAGNET_FILE_COUNT = $(shell $(FIND_MAGNETS) | grep . -c)
    MAGNET_LOG_FILES = $(subst .magnet,.magnet.log,$(MAGNET_FILES))

    # log the metadata for debug purposes
    $(info MAGNET_FILES=$(MAGNET_FILES))
    $(info MAGNET_FILE_COUNT=$(MAGNET_FILE_COUNT))
    $(info MAGNET_LOG_FILES=$(MAGNET_LOG_FILES))

    # string templates to get data from magnet files using the target log files as reference
    GET_MAGNET_FILE_FROM_LOG_FILE = $(subst .magnet.log,.magnet,$@)
    GET_MAGNET_FROM_LOG_FILE = $(shell grep $(GET_MAGNET_FILE_FROM_LOG_FILE) -e '^magnet' | tr -d ' \n\t')

    # string template to flock the log file to ensure duplicate downloads are not started
    LOCK_MAGNET_FILE = flock --exclusive --nonblock $(GET_MAGNET_FILE_FROM_LOG_FILE)

    # string template to configure what options aria2c will be executed with
    ARIA2C_DOWNLOAD = aria2c --check-integrity=true --allow-overwrite=true --bt-force-encryption=true

    # string template to add further context to the log files
    LOGGER = logger --rfc5424=notq --socket-errors=off -t 'aria2-$@' --stderr

    clean:
    rm -fv $(MAGNET_LOG_FILES) torrent.log
    # target to refresh the log files
    clean-logs:
    rm -fv $(MAGNET_LOG_FILES)

    # primary mechanism that fires aria2c by making the log files for each torrent the easy to manage make target
    $(MAGNET_LOG_FILES):
    $(LOCK_MAGNET_FILE) $(ARIA2C_DOWNLOAD) '$(GET_MAGNET_FROM_LOG_FILE)' 2>&1 | $(LOGGER) 2>&1
    $(LOCK_MAGNET_FILE) $(ARIA2C_DOWNLOAD) '$(GET_MAGNET_FROM_LOG_FILE)' 2>&1 | $(LOGGER) 2>&1 | tee '$@'

    # singleton make target that puts all the log files into a single target for make to plan around
    _torrents: $(MAGNET_LOG_FILES)

    # primary target for the user to operate with that forks make into as many parallel jobs as it needs to host every magnet
    torrent:
    $(MAKE) -j $(MAGNET_FILE_COUNT) _torrents | tee '$@.log'
    $(MAKE) -j $(MAGNET_FILE_COUNT) _torrents
  4. CodyKochmann created this gist Sep 16, 2023.
    52 changes: 52 additions & 0 deletions Makefile
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,52 @@
    # Author: Cody Kochmann
    # Date Created: 2023-09-16
    # Last Modified: 2023-09-16
    # License: MIT
    #
    # This makefile spins up an aria2c process for every .magnet file in the current directory.
    #
    # It assumes it will be only one magnet link per each .magnet file.
    #
    # The benefit of this over just having a single aria2c call pull multiple magnet links from
    # a single file is each magnet becomes individually monitorable both through the enhanced
    # log files and through individual pid inspection. It also allows linux to handle any type
    # of scaling, to fully utilize all cpu cores if needed, and keeps each download balanced
    # using the linux congestion algorithms and process scheduler.
    #
    # As a last note, it also uses flocks to ensure two aria2c processes are never managing the
    # same torrent. This allows the user to use mechanisms as simple as cronjobs to ensure
    # specific directories are always being torrented from.
    #

    SHELL := /bin/bash
    SHELLFLAGS := -euxo pipefail

    FIND_MAGNETS = find . -type f -name '*.magnet' -maxdepth 1 2>/dev/null

    MAGNET_FILES = $(shell $(FIND_MAGNETS))
    MAGNET_FILE_COUNT = $(shell $(FIND_MAGNETS) | grep . -c)
    MAGNET_LOG_FILES = $(subst .magnet,.magnet.log,$(MAGNET_FILES))

    $(info MAGNET_FILES=$(MAGNET_FILES))
    $(info MAGNET_FILE_COUNT=$(MAGNET_FILE_COUNT))
    $(info MAGNET_LOG_FILES=$(MAGNET_LOG_FILES))

    GET_MAGNET_FILE_FROM_LOG_FILE = $(subst .magnet.log,.magnet,$@)
    GET_MAGNET_FROM_LOG_FILE = $(shell grep $(GET_MAGNET_FILE_FROM_LOG_FILE) -e '^magnet' | tr -d ' \n\t')

    LOCK_MAGNET_FILE = flock --exclusive --nonblock $(GET_MAGNET_FILE_FROM_LOG_FILE)

    ARIA2C_DOWNLOAD = aria2c --check-integrity=true --allow-overwrite=true --bt-force-encryption=true

    LOGGER = logger --rfc5424=notq --socket-errors=off -t 'aria2-$@' --stderr

    clean:
    rm -fv $(MAGNET_LOG_FILES) torrent.log

    $(MAGNET_LOG_FILES):
    $(LOCK_MAGNET_FILE) $(ARIA2C_DOWNLOAD) '$(GET_MAGNET_FROM_LOG_FILE)' 2>&1 | $(LOGGER) 2>&1

    _torrents: $(MAGNET_LOG_FILES)

    torrent:
    $(MAKE) -j $(MAGNET_FILE_COUNT) _torrents | tee '$@.log'